/
omptarget.cpp
2361 lines (2031 loc) · 82.9 KB
/
omptarget.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//===------ omptarget.cpp - Target independent OpenMP target RTL -- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//
//
// Implementation of the interface to be used by Clang during the codegen of a
// target region.
//
//===----------------------------------------------------------------------===//
#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <list>
#include <map>
#include <mutex>
#include <string>
#include <vector>
// Header file global to this project
#include "omptarget.h"
#ifdef OMPTARGET_DEBUG
static int DebugLevel = 0;
#define DP(...) \
do { \
if (DebugLevel > 0) { \
DEBUGP("Libomptarget", __VA_ARGS__); \
} \
} while (false)
#else // OMPTARGET_DEBUG
#define DP(...) {}
#endif // OMPTARGET_DEBUG
#define INF_REF_CNT (LONG_MAX>>1) // leave room for additions/subtractions
#define CONSIDERED_INF(x) (x > (INF_REF_CNT>>1))
// List of all plugins that can support offloading.
static const char *RTLNames[] = {
/* PowerPC target */ "libomptarget.rtl.ppc64.so",
/* x86_64 target */ "libomptarget.rtl.x86_64.so",
/* CUDA target */ "libomptarget.rtl.cuda.so",
/* AArch64 target */ "libomptarget.rtl.aarch64.so"};
// forward declarations
struct RTLInfoTy;
static int target(int64_t device_id, void *host_ptr, int32_t arg_num,
void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
int32_t team_num, int32_t thread_limit, int IsTeamConstruct);
/// Map between host data and target data.
struct HostDataToTargetTy {
uintptr_t HstPtrBase; // host info.
uintptr_t HstPtrBegin;
uintptr_t HstPtrEnd; // non-inclusive.
uintptr_t TgtPtrBegin; // target info.
long RefCount;
HostDataToTargetTy()
: HstPtrBase(0), HstPtrBegin(0), HstPtrEnd(0),
TgtPtrBegin(0), RefCount(0) {}
HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E, uintptr_t TB)
: HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E),
TgtPtrBegin(TB), RefCount(1) {}
HostDataToTargetTy(uintptr_t BP, uintptr_t B, uintptr_t E, uintptr_t TB,
long RF)
: HstPtrBase(BP), HstPtrBegin(B), HstPtrEnd(E),
TgtPtrBegin(TB), RefCount(RF) {}
};
typedef std::list<HostDataToTargetTy> HostDataToTargetListTy;
struct LookupResult {
struct {
unsigned IsContained : 1;
unsigned ExtendsBefore : 1;
unsigned ExtendsAfter : 1;
} Flags;
HostDataToTargetListTy::iterator Entry;
LookupResult() : Flags({0,0,0}), Entry() {}
};
/// Map for shadow pointers
struct ShadowPtrValTy {
void *HstPtrVal;
void *TgtPtrAddr;
void *TgtPtrVal;
};
typedef std::map<void *, ShadowPtrValTy> ShadowPtrListTy;
///
struct PendingCtorDtorListsTy {
std::list<void *> PendingCtors;
std::list<void *> PendingDtors;
};
typedef std::map<__tgt_bin_desc *, PendingCtorDtorListsTy>
PendingCtorsDtorsPerLibrary;
struct DeviceTy {
int32_t DeviceID;
RTLInfoTy *RTL;
int32_t RTLDeviceID;
bool IsInit;
std::once_flag InitFlag;
bool HasPendingGlobals;
HostDataToTargetListTy HostDataToTargetMap;
PendingCtorsDtorsPerLibrary PendingCtorsDtors;
ShadowPtrListTy ShadowPtrMap;
std::mutex DataMapMtx, PendingGlobalsMtx, ShadowMtx;
uint64_t loopTripCnt;
DeviceTy(RTLInfoTy *RTL)
: DeviceID(-1), RTL(RTL), RTLDeviceID(-1), IsInit(false), InitFlag(),
HasPendingGlobals(false), HostDataToTargetMap(),
PendingCtorsDtors(), ShadowPtrMap(), DataMapMtx(), PendingGlobalsMtx(),
ShadowMtx(), loopTripCnt(0) {}
// The existence of mutexes makes DeviceTy non-copyable. We need to
// provide a copy constructor and an assignment operator explicitly.
DeviceTy(const DeviceTy &d)
: DeviceID(d.DeviceID), RTL(d.RTL), RTLDeviceID(d.RTLDeviceID),
IsInit(d.IsInit), InitFlag(), HasPendingGlobals(d.HasPendingGlobals),
HostDataToTargetMap(d.HostDataToTargetMap),
PendingCtorsDtors(d.PendingCtorsDtors), ShadowPtrMap(d.ShadowPtrMap),
DataMapMtx(), PendingGlobalsMtx(),
ShadowMtx(), loopTripCnt(d.loopTripCnt) {}
DeviceTy& operator=(const DeviceTy &d) {
DeviceID = d.DeviceID;
RTL = d.RTL;
RTLDeviceID = d.RTLDeviceID;
IsInit = d.IsInit;
HasPendingGlobals = d.HasPendingGlobals;
HostDataToTargetMap = d.HostDataToTargetMap;
PendingCtorsDtors = d.PendingCtorsDtors;
ShadowPtrMap = d.ShadowPtrMap;
loopTripCnt = d.loopTripCnt;
return *this;
}
long getMapEntryRefCnt(void *HstPtrBegin);
LookupResult lookupMapping(void *HstPtrBegin, int64_t Size);
void *getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase, int64_t Size,
bool &IsNew, bool IsImplicit, bool UpdateRefCount = true);
void *getTgtPtrBegin(void *HstPtrBegin, int64_t Size);
void *getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast,
bool UpdateRefCount);
int deallocTgtPtr(void *TgtPtrBegin, int64_t Size, bool ForceDelete);
int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
int disassociatePtr(void *HstPtrBegin);
// calls to RTL
int32_t initOnce();
__tgt_target_table *load_binary(void *Img);
int32_t data_submit(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size);
int32_t data_retrieve(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
int32_t run_region(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, int32_t TgtVarsSize);
int32_t run_team_region(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, int32_t TgtVarsSize, int32_t NumTeams,
int32_t ThreadLimit, uint64_t LoopTripCount);
private:
// Call to RTL
void init(); // To be called only via DeviceTy::initOnce()
};
/// Map between Device ID (i.e. openmp device id) and its DeviceTy.
typedef std::vector<DeviceTy> DevicesTy;
static DevicesTy Devices;
struct RTLInfoTy {
typedef int32_t(is_valid_binary_ty)(void *);
typedef int32_t(number_of_devices_ty)();
typedef int32_t(init_device_ty)(int32_t);
typedef __tgt_target_table *(load_binary_ty)(int32_t, void *);
typedef void *(data_alloc_ty)(int32_t, int64_t, void *);
typedef int32_t(data_submit_ty)(int32_t, void *, void *, int64_t);
typedef int32_t(data_retrieve_ty)(int32_t, void *, void *, int64_t);
typedef int32_t(data_delete_ty)(int32_t, void *);
typedef int32_t(run_region_ty)(int32_t, void *, void **, ptrdiff_t *,
int32_t);
typedef int32_t(run_team_region_ty)(int32_t, void *, void **, ptrdiff_t *,
int32_t, int32_t, int32_t, uint64_t);
int32_t Idx; // RTL index, index is the number of devices
// of other RTLs that were registered before,
// i.e. the OpenMP index of the first device
// to be registered with this RTL.
int32_t NumberOfDevices; // Number of devices this RTL deals with.
std::vector<DeviceTy *> Devices; // one per device (NumberOfDevices in total).
void *LibraryHandler;
#ifdef OMPTARGET_DEBUG
std::string RTLName;
#endif
// Functions implemented in the RTL.
is_valid_binary_ty *is_valid_binary;
number_of_devices_ty *number_of_devices;
init_device_ty *init_device;
load_binary_ty *load_binary;
data_alloc_ty *data_alloc;
data_submit_ty *data_submit;
data_retrieve_ty *data_retrieve;
data_delete_ty *data_delete;
run_region_ty *run_region;
run_team_region_ty *run_team_region;
// Are there images associated with this RTL.
bool isUsed;
// Mutex for thread-safety when calling RTL interface functions.
// It is easier to enforce thread-safety at the libomptarget level,
// so that developers of new RTLs do not have to worry about it.
std::mutex Mtx;
// The existence of the mutex above makes RTLInfoTy non-copyable.
// We need to provide a copy constructor explicitly.
RTLInfoTy()
: Idx(-1), NumberOfDevices(-1), Devices(), LibraryHandler(0),
#ifdef OMPTARGET_DEBUG
RTLName(),
#endif
is_valid_binary(0), number_of_devices(0), init_device(0),
load_binary(0), data_alloc(0), data_submit(0), data_retrieve(0),
data_delete(0), run_region(0), run_team_region(0), isUsed(false),
Mtx() {}
RTLInfoTy(const RTLInfoTy &r) : Mtx() {
Idx = r.Idx;
NumberOfDevices = r.NumberOfDevices;
Devices = r.Devices;
LibraryHandler = r.LibraryHandler;
#ifdef OMPTARGET_DEBUG
RTLName = r.RTLName;
#endif
is_valid_binary = r.is_valid_binary;
number_of_devices = r.number_of_devices;
init_device = r.init_device;
load_binary = r.load_binary;
data_alloc = r.data_alloc;
data_submit = r.data_submit;
data_retrieve = r.data_retrieve;
data_delete = r.data_delete;
run_region = r.run_region;
run_team_region = r.run_team_region;
isUsed = r.isUsed;
}
};
/// RTLs identified in the system.
class RTLsTy {
private:
// Mutex-like object to guarantee thread-safety and unique initialization
// (i.e. the library attempts to load the RTLs (plugins) only once).
std::once_flag initFlag;
void LoadRTLs(); // not thread-safe
public:
// List of the detected runtime libraries.
std::list<RTLInfoTy> AllRTLs;
// Array of pointers to the detected runtime libraries that have compatible
// binaries.
std::vector<RTLInfoTy *> UsedRTLs;
explicit RTLsTy() {}
// Load all the runtime libraries (plugins) if not done before.
void LoadRTLsOnce();
};
void RTLsTy::LoadRTLs() {
#ifdef OMPTARGET_DEBUG
if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) {
DebugLevel = std::stoi(envStr);
}
#endif // OMPTARGET_DEBUG
// Parse environment variable OMP_TARGET_OFFLOAD (if set)
char *envStr = getenv("OMP_TARGET_OFFLOAD");
if (envStr && !strcmp(envStr, "DISABLED")) {
DP("Target offloading disabled by environment\n");
return;
}
DP("Loading RTLs...\n");
// Attempt to open all the plugins and, if they exist, check if the interface
// is correct and if they are supporting any devices.
for (auto *Name : RTLNames) {
DP("Loading library '%s'...\n", Name);
void *dynlib_handle = dlopen(Name, RTLD_NOW);
if (!dynlib_handle) {
// Library does not exist or cannot be found.
DP("Unable to load library '%s': %s!\n", Name, dlerror());
continue;
}
DP("Successfully loaded library '%s'!\n", Name);
// Retrieve the RTL information from the runtime library.
RTLInfoTy R;
R.LibraryHandler = dynlib_handle;
R.isUsed = false;
#ifdef OMPTARGET_DEBUG
R.RTLName = Name;
#endif
if (!(*((void**) &R.is_valid_binary) = dlsym(
dynlib_handle, "__tgt_rtl_is_valid_binary")))
continue;
if (!(*((void**) &R.number_of_devices) = dlsym(
dynlib_handle, "__tgt_rtl_number_of_devices")))
continue;
if (!(*((void**) &R.init_device) = dlsym(
dynlib_handle, "__tgt_rtl_init_device")))
continue;
if (!(*((void**) &R.load_binary) = dlsym(
dynlib_handle, "__tgt_rtl_load_binary")))
continue;
if (!(*((void**) &R.data_alloc) = dlsym(
dynlib_handle, "__tgt_rtl_data_alloc")))
continue;
if (!(*((void**) &R.data_submit) = dlsym(
dynlib_handle, "__tgt_rtl_data_submit")))
continue;
if (!(*((void**) &R.data_retrieve) = dlsym(
dynlib_handle, "__tgt_rtl_data_retrieve")))
continue;
if (!(*((void**) &R.data_delete) = dlsym(
dynlib_handle, "__tgt_rtl_data_delete")))
continue;
if (!(*((void**) &R.run_region) = dlsym(
dynlib_handle, "__tgt_rtl_run_target_region")))
continue;
if (!(*((void**) &R.run_team_region) = dlsym(
dynlib_handle, "__tgt_rtl_run_target_team_region")))
continue;
// No devices are supported by this RTL?
if (!(R.NumberOfDevices = R.number_of_devices())) {
DP("No devices supported in this RTL\n");
continue;
}
DP("Registering RTL %s supporting %d devices!\n",
R.RTLName.c_str(), R.NumberOfDevices);
// The RTL is valid! Will save the information in the RTLs list.
AllRTLs.push_back(R);
}
DP("RTLs loaded!\n");
return;
}
void RTLsTy::LoadRTLsOnce() {
// RTL.LoadRTLs() is called only once in a thread-safe fashion.
std::call_once(initFlag, &RTLsTy::LoadRTLs, this);
}
static RTLsTy RTLs;
static std::mutex RTLsMtx;
/// Map between the host entry begin and the translation table. Each
/// registered library gets one TranslationTable. Use the map from
/// __tgt_offload_entry so that we may quickly determine whether we
/// are trying to (re)register an existing lib or really have a new one.
struct TranslationTable {
__tgt_target_table HostTable;
// Image assigned to a given device.
std::vector<__tgt_device_image *> TargetsImages; // One image per device ID.
// Table of entry points or NULL if it was not already computed.
std::vector<__tgt_target_table *> TargetsTable; // One table per device ID.
};
typedef std::map<__tgt_offload_entry *, TranslationTable>
HostEntriesBeginToTransTableTy;
static HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable;
static std::mutex TrlTblMtx;
/// Map between the host ptr and a table index
struct TableMap {
TranslationTable *Table; // table associated with the host ptr.
uint32_t Index; // index in which the host ptr translated entry is found.
TableMap() : Table(0), Index(0) {}
TableMap(TranslationTable *table, uint32_t index)
: Table(table), Index(index) {}
};
typedef std::map<void *, TableMap> HostPtrToTableMapTy;
static HostPtrToTableMapTy HostPtrToTableMap;
static std::mutex TblMapMtx;
/// Check whether a device has an associated RTL and initialize it if it's not
/// already initialized.
static bool device_is_ready(int device_num) {
DP("Checking whether device %d is ready.\n", device_num);
// Devices.size() can only change while registering a new
// library, so try to acquire the lock of RTLs' mutex.
RTLsMtx.lock();
size_t Devices_size = Devices.size();
RTLsMtx.unlock();
if (Devices_size <= (size_t)device_num) {
DP("Device ID %d does not have a matching RTL\n", device_num);
return false;
}
// Get device info
DeviceTy &Device = Devices[device_num];
DP("Is the device %d (local ID %d) initialized? %d\n", device_num,
Device.RTLDeviceID, Device.IsInit);
// Init the device if not done before
if (!Device.IsInit && Device.initOnce() != OFFLOAD_SUCCESS) {
DP("Failed to init device %d\n", device_num);
return false;
}
DP("Device %d is ready to use.\n", device_num);
return true;
}
////////////////////////////////////////////////////////////////////////////////
// Target API functions
//
EXTERN int omp_get_num_devices(void) {
RTLsMtx.lock();
size_t Devices_size = Devices.size();
RTLsMtx.unlock();
DP("Call to omp_get_num_devices returning %zd\n", Devices_size);
return Devices_size;
}
EXTERN int omp_get_initial_device(void) {
DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE);
return HOST_DEVICE;
}
EXTERN void *omp_target_alloc(size_t size, int device_num) {
DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
device_num, size);
if (size <= 0) {
DP("Call to omp_target_alloc with non-positive length\n");
return NULL;
}
void *rc = NULL;
if (device_num == omp_get_initial_device()) {
rc = malloc(size);
DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
return rc;
}
if (!device_is_ready(device_num)) {
DP("omp_target_alloc returns NULL ptr\n");
return NULL;
}
DeviceTy &Device = Devices[device_num];
rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL);
DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
return rc;
}
EXTERN void omp_target_free(void *device_ptr, int device_num) {
DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
device_num, DPxPTR(device_ptr));
if (!device_ptr) {
DP("Call to omp_target_free with NULL ptr\n");
return;
}
if (device_num == omp_get_initial_device()) {
free(device_ptr);
DP("omp_target_free deallocated host ptr\n");
return;
}
if (!device_is_ready(device_num)) {
DP("omp_target_free returns, nothing to do\n");
return;
}
DeviceTy &Device = Devices[device_num];
Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr);
DP("omp_target_free deallocated device ptr\n");
}
EXTERN int omp_target_is_present(void *ptr, int device_num) {
DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
device_num, DPxPTR(ptr));
if (!ptr) {
DP("Call to omp_target_is_present with NULL ptr, returning false\n");
return false;
}
if (device_num == omp_get_initial_device()) {
DP("Call to omp_target_is_present on host, returning true\n");
return true;
}
RTLsMtx.lock();
size_t Devices_size = Devices.size();
RTLsMtx.unlock();
if (Devices_size <= (size_t)device_num) {
DP("Call to omp_target_is_present with invalid device ID, returning "
"false\n");
return false;
}
DeviceTy& Device = Devices[device_num];
bool IsLast; // not used
int rc = (Device.getTgtPtrBegin(ptr, 0, IsLast, false) != NULL);
DP("Call to omp_target_is_present returns %d\n", rc);
return rc;
}
EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
DP("Call to omp_target_memcpy, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
"src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
DPxPTR(src), dst_offset, src_offset, length);
if (!dst || !src || length <= 0) {
DP("Call to omp_target_memcpy with invalid arguments\n");
return OFFLOAD_FAIL;
}
if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
return OFFLOAD_FAIL;
}
if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
return OFFLOAD_FAIL;
}
int rc = OFFLOAD_SUCCESS;
void *srcAddr = (char *)src + src_offset;
void *dstAddr = (char *)dst + dst_offset;
if (src_device == omp_get_initial_device() &&
dst_device == omp_get_initial_device()) {
DP("copy from host to host\n");
const void *p = memcpy(dstAddr, srcAddr, length);
if (p == NULL)
rc = OFFLOAD_FAIL;
} else if (src_device == omp_get_initial_device()) {
DP("copy from host to device\n");
DeviceTy& DstDev = Devices[dst_device];
rc = DstDev.data_submit(dstAddr, srcAddr, length);
} else if (dst_device == omp_get_initial_device()) {
DP("copy from device to host\n");
DeviceTy& SrcDev = Devices[src_device];
rc = SrcDev.data_retrieve(dstAddr, srcAddr, length);
} else {
DP("copy from device to device\n");
void *buffer = malloc(length);
DeviceTy& SrcDev = Devices[src_device];
DeviceTy& DstDev = Devices[dst_device];
rc = SrcDev.data_retrieve(buffer, srcAddr, length);
if (rc == OFFLOAD_SUCCESS)
rc = DstDev.data_submit(dstAddr, buffer, length);
}
DP("omp_target_memcpy returns %d\n", rc);
return rc;
}
EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
int num_dims, const size_t *volume, const size_t *dst_offsets,
const size_t *src_offsets, const size_t *dst_dimensions,
const size_t *src_dimensions, int dst_device, int src_device) {
DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
"src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
"volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device,
src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
DPxPTR(volume), element_size, num_dims);
if (!(dst || src)) {
DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
INT_MAX);
return INT_MAX;
}
if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
!dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
DP("Call to omp_target_memcpy_rect with invalid arguments\n");
return OFFLOAD_FAIL;
}
int rc;
if (num_dims == 1) {
rc = omp_target_memcpy(dst, src, element_size * volume[0],
element_size * dst_offsets[0], element_size * src_offsets[0],
dst_device, src_device);
} else {
size_t dst_slice_size = element_size;
size_t src_slice_size = element_size;
for (int i=1; i<num_dims; ++i) {
dst_slice_size *= dst_dimensions[i];
src_slice_size *= src_dimensions[i];
}
size_t dst_off = dst_offsets[0] * dst_slice_size;
size_t src_off = src_offsets[0] * src_slice_size;
for (size_t i=0; i<volume[0]; ++i) {
rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i,
(char *) src + src_off + src_slice_size * i, element_size,
num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
if (rc) {
DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
return rc;
}
}
}
DP("omp_target_memcpy_rect returns %d\n", rc);
return rc;
}
EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
size_t size, size_t device_offset, int device_num) {
DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
"device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
if (!host_ptr || !device_ptr || size <= 0) {
DP("Call to omp_target_associate_ptr with invalid arguments\n");
return OFFLOAD_FAIL;
}
if (device_num == omp_get_initial_device()) {
DP("omp_target_associate_ptr: no association possible on the host\n");
return OFFLOAD_FAIL;
}
if (!device_is_ready(device_num)) {
DP("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
return OFFLOAD_FAIL;
}
DeviceTy& Device = Devices[device_num];
void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
int rc = Device.associatePtr(host_ptr, device_addr, size);
DP("omp_target_associate_ptr returns %d\n", rc);
return rc;
}
EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
"device_num %d\n", DPxPTR(host_ptr), device_num);
if (!host_ptr) {
DP("Call to omp_target_associate_ptr with invalid host_ptr\n");
return OFFLOAD_FAIL;
}
if (device_num == omp_get_initial_device()) {
DP("omp_target_disassociate_ptr: no association possible on the host\n");
return OFFLOAD_FAIL;
}
if (!device_is_ready(device_num)) {
DP("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
return OFFLOAD_FAIL;
}
DeviceTy& Device = Devices[device_num];
int rc = Device.disassociatePtr(host_ptr);
DP("omp_target_disassociate_ptr returns %d\n", rc);
return rc;
}
////////////////////////////////////////////////////////////////////////////////
// functionality for device
int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
DataMapMtx.lock();
// Check if entry exists
for (auto &HT : HostDataToTargetMap) {
if ((uintptr_t)HstPtrBegin == HT.HstPtrBegin) {
// Mapping already exists
bool isValid = HT.HstPtrBegin == (uintptr_t) HstPtrBegin &&
HT.HstPtrEnd == (uintptr_t) HstPtrBegin + Size &&
HT.TgtPtrBegin == (uintptr_t) TgtPtrBegin;
DataMapMtx.unlock();
if (isValid) {
DP("Attempt to re-associate the same device ptr+offset with the same "
"host ptr, nothing to do\n");
return OFFLOAD_SUCCESS;
} else {
DP("Not allowed to re-associate a different device ptr+offset with the "
"same host ptr\n");
return OFFLOAD_FAIL;
}
}
}
// Mapping does not exist, allocate it
HostDataToTargetTy newEntry;
// Set up missing fields
newEntry.HstPtrBase = (uintptr_t) HstPtrBegin;
newEntry.HstPtrBegin = (uintptr_t) HstPtrBegin;
newEntry.HstPtrEnd = (uintptr_t) HstPtrBegin + Size;
newEntry.TgtPtrBegin = (uintptr_t) TgtPtrBegin;
// refCount must be infinite
newEntry.RefCount = INF_REF_CNT;
DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", HstEnd="
DPxMOD ", TgtBegin=" DPxMOD "\n", DPxPTR(newEntry.HstPtrBase),
DPxPTR(newEntry.HstPtrBegin), DPxPTR(newEntry.HstPtrEnd),
DPxPTR(newEntry.TgtPtrBegin));
HostDataToTargetMap.push_front(newEntry);
DataMapMtx.unlock();
return OFFLOAD_SUCCESS;
}
int DeviceTy::disassociatePtr(void *HstPtrBegin) {
DataMapMtx.lock();
// Check if entry exists
for (HostDataToTargetListTy::iterator ii = HostDataToTargetMap.begin();
ii != HostDataToTargetMap.end(); ++ii) {
if ((uintptr_t)HstPtrBegin == ii->HstPtrBegin) {
// Mapping exists
if (CONSIDERED_INF(ii->RefCount)) {
DP("Association found, removing it\n");
HostDataToTargetMap.erase(ii);
DataMapMtx.unlock();
return OFFLOAD_SUCCESS;
} else {
DP("Trying to disassociate a pointer which was not mapped via "
"omp_target_associate_ptr\n");
break;
}
}
}
// Mapping not found
DataMapMtx.unlock();
DP("Association not found\n");
return OFFLOAD_FAIL;
}
// Get ref count of map entry containing HstPtrBegin
long DeviceTy::getMapEntryRefCnt(void *HstPtrBegin) {
uintptr_t hp = (uintptr_t)HstPtrBegin;
long RefCnt = -1;
DataMapMtx.lock();
for (auto &HT : HostDataToTargetMap) {
if (hp >= HT.HstPtrBegin && hp < HT.HstPtrEnd) {
DP("DeviceTy::getMapEntry: requested entry found\n");
RefCnt = HT.RefCount;
break;
}
}
DataMapMtx.unlock();
if (RefCnt < 0) {
DP("DeviceTy::getMapEntry: requested entry not found\n");
}
return RefCnt;
}
LookupResult DeviceTy::lookupMapping(void *HstPtrBegin, int64_t Size) {
uintptr_t hp = (uintptr_t)HstPtrBegin;
LookupResult lr;
DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%ld)...\n", DPxPTR(hp),
Size);
for (lr.Entry = HostDataToTargetMap.begin();
lr.Entry != HostDataToTargetMap.end(); ++lr.Entry) {
auto &HT = *lr.Entry;
// Is it contained?
lr.Flags.IsContained = hp >= HT.HstPtrBegin && hp < HT.HstPtrEnd &&
(hp+Size) <= HT.HstPtrEnd;
// Does it extend into an already mapped region?
lr.Flags.ExtendsBefore = hp < HT.HstPtrBegin && (hp+Size) > HT.HstPtrBegin;
// Does it extend beyond the mapped region?
lr.Flags.ExtendsAfter = hp < HT.HstPtrEnd && (hp+Size) > HT.HstPtrEnd;
if (lr.Flags.IsContained || lr.Flags.ExtendsBefore ||
lr.Flags.ExtendsAfter) {
break;
}
}
if (lr.Flags.ExtendsBefore) {
DP("WARNING: Pointer is not mapped but section extends into already "
"mapped data\n");
}
if (lr.Flags.ExtendsAfter) {
DP("WARNING: Pointer is already mapped but section extends beyond mapped "
"region\n");
}
return lr;
}
// Used by target_data_begin
// Return the target pointer begin (where the data will be moved).
// Allocate memory if this is the first occurrence if this mapping.
// Increment the reference counter.
// If NULL is returned, then either data allocation failed or the user tried
// to do an illegal mapping.
void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
int64_t Size, bool &IsNew, bool IsImplicit, bool UpdateRefCount) {
void *rc = NULL;
DataMapMtx.lock();
LookupResult lr = lookupMapping(HstPtrBegin, Size);
// Check if the pointer is contained.
if (lr.Flags.IsContained ||
((lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) && IsImplicit)) {
auto &HT = *lr.Entry;
IsNew = false;
if (UpdateRefCount)
++HT.RefCount;
uintptr_t tp = HT.TgtPtrBegin + ((uintptr_t)HstPtrBegin - HT.HstPtrBegin);
DP("Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
"Size=%ld,%s RefCount=%s\n", (IsImplicit ? " (implicit)" : ""),
DPxPTR(HstPtrBegin), DPxPTR(tp), Size,
(UpdateRefCount ? " updated" : ""),
(CONSIDERED_INF(HT.RefCount)) ? "INF" :
std::to_string(HT.RefCount).c_str());
rc = (void *)tp;
} else if ((lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) && !IsImplicit) {
// Explicit extension of mapped data - not allowed.
DP("Explicit extension of mapping is not allowed.\n");
} else if (Size) {
// If it is not contained and Size > 0 we should create a new entry for it.
IsNew = true;
uintptr_t tp = (uintptr_t)RTL->data_alloc(RTLDeviceID, Size, HstPtrBegin);
DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
"HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n", DPxPTR(HstPtrBase),
DPxPTR(HstPtrBegin), DPxPTR((uintptr_t)HstPtrBegin + Size), DPxPTR(tp));
HostDataToTargetMap.push_front(HostDataToTargetTy((uintptr_t)HstPtrBase,
(uintptr_t)HstPtrBegin, (uintptr_t)HstPtrBegin + Size, tp));
rc = (void *)tp;
}
DataMapMtx.unlock();
return rc;
}
// Used by target_data_begin, target_data_end, target_data_update and target.
// Return the target pointer begin (where the data will be moved).
// Decrement the reference counter if called from target_data_end.
void *DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast,
bool UpdateRefCount) {
void *rc = NULL;
DataMapMtx.lock();
LookupResult lr = lookupMapping(HstPtrBegin, Size);
if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) {
auto &HT = *lr.Entry;
IsLast = !(HT.RefCount > 1);
if (HT.RefCount > 1 && UpdateRefCount)
--HT.RefCount;
uintptr_t tp = HT.TgtPtrBegin + ((uintptr_t)HstPtrBegin - HT.HstPtrBegin);
DP("Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
"Size=%ld,%s RefCount=%s\n", DPxPTR(HstPtrBegin), DPxPTR(tp), Size,
(UpdateRefCount ? " updated" : ""),
(CONSIDERED_INF(HT.RefCount)) ? "INF" :
std::to_string(HT.RefCount).c_str());
rc = (void *)tp;
} else {
IsLast = false;
}
DataMapMtx.unlock();
return rc;
}
// Return the target pointer begin (where the data will be moved).
// Lock-free version called when loading global symbols from the fat binary.
void *DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size) {
uintptr_t hp = (uintptr_t)HstPtrBegin;
LookupResult lr = lookupMapping(HstPtrBegin, Size);
if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) {
auto &HT = *lr.Entry;
uintptr_t tp = HT.TgtPtrBegin + (hp - HT.HstPtrBegin);
return (void *)tp;
}
return NULL;
}
int DeviceTy::deallocTgtPtr(void *HstPtrBegin, int64_t Size, bool ForceDelete) {
// Check if the pointer is contained in any sub-nodes.
int rc;
DataMapMtx.lock();
LookupResult lr = lookupMapping(HstPtrBegin, Size);
if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) {
auto &HT = *lr.Entry;
if (ForceDelete)
HT.RefCount = 1;
if (--HT.RefCount <= 0) {
assert(HT.RefCount == 0 && "did not expect a negative ref count");
DP("Deleting tgt data " DPxMOD " of size %ld\n",
DPxPTR(HT.TgtPtrBegin), Size);
RTL->data_delete(RTLDeviceID, (void *)HT.TgtPtrBegin);
DP("Removing%s mapping with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
", Size=%ld\n", (ForceDelete ? " (forced)" : ""),
DPxPTR(HT.HstPtrBegin), DPxPTR(HT.TgtPtrBegin), Size);
HostDataToTargetMap.erase(lr.Entry);
}
rc = OFFLOAD_SUCCESS;
} else {
DP("Section to delete (hst addr " DPxMOD ") does not exist in the allocated"
" memory\n", DPxPTR(HstPtrBegin));
rc = OFFLOAD_FAIL;
}
DataMapMtx.unlock();
return rc;
}
/// Init device, should not be called directly.
void DeviceTy::init() {
int32_t rc = RTL->init_device(RTLDeviceID);
if (rc == OFFLOAD_SUCCESS) {
IsInit = true;
}
}
/// Thread-safe method to initialize the device only once.
int32_t DeviceTy::initOnce() {
std::call_once(InitFlag, &DeviceTy::init, this);
// At this point, if IsInit is true, then either this thread or some other
// thread in the past successfully initialized the device, so we can return
// OFFLOAD_SUCCESS. If this thread executed init() via call_once() and it
// failed, return OFFLOAD_FAIL. If call_once did not invoke init(), it means
// that some other thread already attempted to execute init() and if IsInit
// is still false, return OFFLOAD_FAIL.
if (IsInit)
return OFFLOAD_SUCCESS;
else
return OFFLOAD_FAIL;
}
// Load binary to device.
__tgt_target_table *DeviceTy::load_binary(void *Img) {
RTL->Mtx.lock();
__tgt_target_table *rc = RTL->load_binary(RTLDeviceID, Img);
RTL->Mtx.unlock();