/
ib_iface.c
1883 lines (1583 loc) · 67.6 KB
/
ib_iface.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2021. ALL RIGHTS RESERVED.
* Copyright (C) 2021 Broadcom. ALL RIGHTS RESERVED. The term "Broadcom"
* refers to Broadcom Inc. and/or its subsidiaries.
* Copyright (C) Huawei Technologies Co., Ltd. 2020. ALL RIGHTS RESERVED.
*
* See file LICENSE for terms.
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include "ib_iface.h"
#include "ib_log.h"
#include <uct/base/uct_md.h>
#include <ucs/arch/bitops.h>
#include <ucs/arch/cpu.h>
#include <ucs/profile/profile.h>
#include <ucs/type/class.h>
#include <ucs/type/cpu_set.h>
#include <ucs/type/serialize.h>
#include <ucs/debug/log.h>
#include <ucs/time/time.h>
#include <ucs/sys/sock.h>
#include <string.h>
#include <stdlib.h>
#include <poll.h>
static UCS_CONFIG_DEFINE_ARRAY(path_bits_spec,
sizeof(ucs_range_spec_t),
UCS_CONFIG_TYPE_RANGE_SPEC);
const char *uct_ib_mtu_values[] = {
[UCT_IB_MTU_DEFAULT] = "default",
[UCT_IB_MTU_512] = "512",
[UCT_IB_MTU_1024] = "1024",
[UCT_IB_MTU_2048] = "2048",
[UCT_IB_MTU_4096] = "4096",
[UCT_IB_MTU_LAST] = NULL
};
enum {
UCT_IB_ADDRESS_TYPE_LINK_LOCAL,
UCT_IB_ADDRESS_TYPE_SITE_LOCAL,
UCT_IB_ADDRESS_TYPE_GLOBAL,
UCT_IB_ADDRESS_TYPE_ETH,
UCT_IB_ADDRESS_TYPE_LAST,
UCT_IB_IFACE_ADDRESS_TYPE_AUTO = UCT_IB_ADDRESS_TYPE_LAST,
UCT_IB_IFACE_ADDRESS_TYPE_LAST
};
static const char *uct_ib_iface_addr_types[] = {
[UCT_IB_ADDRESS_TYPE_LINK_LOCAL] = "ib_local",
[UCT_IB_ADDRESS_TYPE_SITE_LOCAL] = "ib_site_local",
[UCT_IB_ADDRESS_TYPE_GLOBAL] = "ib_global",
[UCT_IB_ADDRESS_TYPE_ETH] = "eth",
[UCT_IB_IFACE_ADDRESS_TYPE_AUTO] = "auto",
[UCT_IB_IFACE_ADDRESS_TYPE_LAST] = NULL
};
ucs_config_field_t uct_ib_iface_config_table[] = {
{"", "ALLOC=thp,mmap,heap", NULL,
ucs_offsetof(uct_ib_iface_config_t, super), UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)},
{"SEG_SIZE", "8192",
"Size of bounce buffers used for post_send and post_recv.",
ucs_offsetof(uct_ib_iface_config_t, seg_size), UCS_CONFIG_TYPE_MEMUNITS},
{"TX_QUEUE_LEN", "256",
"Length of send queue in the QP.",
ucs_offsetof(uct_ib_iface_config_t, tx.queue_len), UCS_CONFIG_TYPE_UINT},
{"TX_MAX_BATCH", "16",
"Number of send WQEs to batch in one post-send list. Larger values reduce\n"
"the CPU usage, but increase the latency and pipelining between sender and\n"
"receiver.",
ucs_offsetof(uct_ib_iface_config_t, tx.max_batch), UCS_CONFIG_TYPE_UINT},
{"TX_MAX_POLL", "16",
"Max number of receive completions to pick during TX poll",
ucs_offsetof(uct_ib_iface_config_t, tx.max_poll), UCS_CONFIG_TYPE_UINT},
{"TX_MIN_INLINE", "64",
"Bytes to reserve in send WQE for inline data. Messages which are small\n"
"enough will be sent inline.",
ucs_offsetof(uct_ib_iface_config_t, tx.min_inline), UCS_CONFIG_TYPE_MEMUNITS},
{"TX_INLINE_RESP", "0",
"Bytes to reserve in send WQE for inline response. Responses which are small\n"
"enough, such as of atomic operations and small reads, will be received inline.",
ucs_offsetof(uct_ib_iface_config_t, inl[UCT_IB_DIR_TX]), UCS_CONFIG_TYPE_MEMUNITS},
{"TX_MIN_SGE", "5",
"Number of SG entries to reserve in the send WQE.",
ucs_offsetof(uct_ib_iface_config_t, tx.min_sge), UCS_CONFIG_TYPE_UINT},
UCT_IFACE_MPOOL_CONFIG_FIELDS("TX_", -1, 1024, 128m, 1.0, "send",
ucs_offsetof(uct_ib_iface_config_t, tx.mp),
"\nAttention: Setting this param with value != -1 is a dangerous thing\n"
"in RC/DC and could cause deadlock or performance degradation."),
{"RX_QUEUE_LEN", "4096",
"Length of receive queue in the QPs.",
ucs_offsetof(uct_ib_iface_config_t, rx.queue_len), UCS_CONFIG_TYPE_UINT},
{"RX_MAX_BATCH", "16",
"How many post-receives to perform in one batch.",
ucs_offsetof(uct_ib_iface_config_t, rx.max_batch), UCS_CONFIG_TYPE_UINT},
{"RX_MAX_POLL", "16",
"Max number of receive completions to pick during RX poll",
ucs_offsetof(uct_ib_iface_config_t, rx.max_poll), UCS_CONFIG_TYPE_UINT},
{"RX_INLINE", "0",
"Number of bytes to request for inline receive. If the maximal supported size\n"
"is smaller, it will be used instead. If it is possible to support a larger\n"
"size than requested with the same hardware resources, it will be used instead.",
ucs_offsetof(uct_ib_iface_config_t, inl[UCT_IB_DIR_RX]), UCS_CONFIG_TYPE_MEMUNITS},
UCT_IFACE_MPOOL_CONFIG_FIELDS("RX_", -1, 0, 128m, 1.0, "receive",
ucs_offsetof(uct_ib_iface_config_t, rx.mp), ""),
{"ADDR_TYPE", "auto",
"Set the interface address type. \"auto\" mode detects the type according to\n"
"link layer type and IB subnet prefix.\n"
"Deprecated. To force use of global routing use IS_GLOBAL.",
ucs_offsetof(uct_ib_iface_config_t, addr_type),
UCS_CONFIG_TYPE_ENUM(uct_ib_iface_addr_types)},
{"IS_GLOBAL", "n",
"Force interface to use global routing.",
ucs_offsetof(uct_ib_iface_config_t, is_global), UCS_CONFIG_TYPE_BOOL},
{"FLID_ROUTE", "y",
"Enable FLID based routing with site-local GIDs.",
ucs_offsetof(uct_ib_iface_config_t, flid_enabled), UCS_CONFIG_TYPE_BOOL},
{"SL", "auto",
"InfiniBand: Service level. 'auto' will select a value matching UCX_IB_AR configuration.\n"
"RoCEv2: Ethernet Priority. 'auto' will select 0 by default.",
ucs_offsetof(uct_ib_iface_config_t, sl), UCS_CONFIG_TYPE_ULUNITS},
{"TRAFFIC_CLASS", "auto",
"IB Traffic Class / RoCEv2 Differentiated Services Code Point (DSCP).\n"
"\"auto\" option selects 106 on RoCEv2 and 0 otherwise.",
ucs_offsetof(uct_ib_iface_config_t, traffic_class), UCS_CONFIG_TYPE_ULUNITS},
{"HOP_LIMIT", "255",
"IB Hop limit / RoCEv2 Time to Live. Should be between 0 and 255.\n",
ucs_offsetof(uct_ib_iface_config_t, hop_limit), UCS_CONFIG_TYPE_UINT},
{"NUM_PATHS", "auto",
"Number of connections that should be created between a pair of communicating\n"
"endpoints for optimal performance. The default value 'auto' behaves according\n"
"to the port link layer:\n"
" RoCE - "UCS_PP_MAKE_STRING(UCT_IB_DEV_MAX_PORTS) " for LAG port, otherwise - 1.\n"
" InfiniBand - As the number of path bits enabled by fabric's LMC value and selected\n"
" by "UCS_DEFAULT_ENV_PREFIX UCT_IB_CONFIG_PREFIX"LID_PATH_BITS configuration.",
ucs_offsetof(uct_ib_iface_config_t, num_paths), UCS_CONFIG_TYPE_ULUNITS},
{"ROCE_LOCAL_SUBNET", "n",
"Use the local IP address and subnet mask of each network device to route RoCEv2 packets.\n"
"If set to 'y', only addresses within the interface's subnet will be assumed as reachable.\n"
"If set to 'n', every remote RoCEv2 IP address is assumed to be reachable from any port.",
ucs_offsetof(uct_ib_iface_config_t, rocev2_local_subnet), UCS_CONFIG_TYPE_BOOL},
{"ROCE_SUBNET_PREFIX_LEN", "auto",
"Length, in bits, of the subnet prefix to be used for reachability check\n"
"when UCX_IB_ROCE_LOCAL_SUBNET is enabled.\n"
" - auto - Detect the subnet prefix length automatically from device address\n"
" - inf - Allow connections only within the same machine and same device\n"
" - <num> - Specify a numeric bit-length value for the subnet prefix",
ucs_offsetof(uct_ib_iface_config_t, rocev2_subnet_pfx_len), UCS_CONFIG_TYPE_ULUNITS},
{"ROCE_PATH_FACTOR", "1",
"Multiplier for RoCE LAG UDP source port calculation. The UDP source port\n"
"is typically used by switches and network adapters to select a different\n"
"path for the same pair of endpoints.",
ucs_offsetof(uct_ib_iface_config_t, roce_path_factor), UCS_CONFIG_TYPE_UINT},
{"LID_PATH_BITS", "0",
"List of IB Path bits separated by comma (a,b,c) "
"which will be the low portion of the LID, according to the LMC in the fabric.",
ucs_offsetof(uct_ib_iface_config_t, lid_path_bits), UCS_CONFIG_TYPE_ARRAY(path_bits_spec)},
{"PKEY", "auto",
"Which pkey value to use. Should be between 0 and 0x7fff.\n"
"\"auto\" option selects a first valid pkey value with full membership.",
ucs_offsetof(uct_ib_iface_config_t, pkey), UCS_CONFIG_TYPE_HEX},
{"PATH_MTU", "default",
"Path MTU. \"default\" will select the best MTU for the device.",
ucs_offsetof(uct_ib_iface_config_t, path_mtu),
UCS_CONFIG_TYPE_ENUM(uct_ib_mtu_values)},
{"COUNTER_SET_ID", "auto",
"Counter set ID to use for performance counters. A value of 'auto' will try to\n"
"detect the default value by creating a dummy QP." ,
ucs_offsetof(uct_ib_iface_config_t, counter_set_id), UCS_CONFIG_TYPE_ULUNITS},
{"REVERSE_SL", "auto",
"Reverse Service level. 'auto' will set the same value of sl\n",
ucs_offsetof(uct_ib_iface_config_t, reverse_sl), UCS_CONFIG_TYPE_ULUNITS},
{"SEND_OVERHEAD", UCT_IB_SEND_OVERHEAD_VALUE(0),
"Estimated overhead of preparing a work request, posting it to the NIC,\n"
"and finalizing an operation",
0, UCS_CONFIG_TYPE_KEY_VALUE(UCS_CONFIG_TYPE_TIME,
{"bcopy", "estimated overhead of allocating a tx buffer",
ucs_offsetof(uct_ib_iface_config_t, send_overhead.bcopy)},
{"cqe", "estimated overhead of processing a work request completion",
ucs_offsetof(uct_ib_iface_config_t, send_overhead.cqe)},
{"db", "estimated overhead of writing a doorbell to PCI",
ucs_offsetof(uct_ib_iface_config_t, send_overhead.db)},
{"wqe_fetch", "estimated overhead of fetching a wqe",
ucs_offsetof(uct_ib_iface_config_t, send_overhead.wqe_fetch)},
{"wqe_post", "estimated overhead of posting a wqe",
ucs_offsetof(uct_ib_iface_config_t, send_overhead.wqe_post)},
{NULL})},
{NULL}
};
#ifdef ENABLE_STATS
static ucs_stats_class_t uct_ib_iface_stats_class = {
.name = "ib_iface",
.num_counters = UCT_IB_IFACE_STAT_LAST,
.class_id = UCS_STATS_CLASS_ID_INVALID,
.counter_names = {
[UCT_IB_IFACE_STAT_RX_COMPLETION] = "rx_completion",
[UCT_IB_IFACE_STAT_TX_COMPLETION] = "tx_completion",
[UCT_IB_IFACE_STAT_RX_COMPLETION_ZIPPED] = "rx_completion_zipped",
[UCT_IB_IFACE_STAT_TX_COMPLETION_ZIPPED] = "tx_completion_zipped"
}
};
#endif /* ENABLE_STATS */
int uct_ib_iface_is_roce(uct_ib_iface_t *iface)
{
return uct_ib_device_is_port_roce(uct_ib_iface_device(iface),
iface->config.port_num);
}
int uct_ib_iface_is_ib(uct_ib_iface_t *iface)
{
return uct_ib_device_is_port_ib(uct_ib_iface_device(iface),
iface->config.port_num);
}
static void
uct_ib_iface_recv_desc_init(uct_iface_h tl_iface, void *obj, uct_mem_h memh)
{
uct_ib_iface_recv_desc_t *desc = obj;
desc->lkey = uct_ib_memh_get_lkey(memh);
}
ucs_status_t uct_ib_iface_recv_mpool_init(uct_ib_iface_t *iface,
const uct_ib_iface_config_t *config,
const uct_iface_params_t *params,
const char *name, ucs_mpool_t *mp)
{
size_t align_offset, alignment;
ucs_status_t status;
unsigned grow;
if (config->rx.queue_len < 1024) {
grow = 1024;
} else {
/* We want to have some free (+10%) elements to avoid mem pool expansion */
grow = ucs_min( (int)(1.1 * config->rx.queue_len + 0.5),
config->rx.mp.max_bufs);
}
/* Preserve the default alignment by UCT header if user does not request
* specific alignment.
* TODO: Analyze how to keep UCT header aligned by cache line even when
* user requested specific alignment for payload.
*/
status = uct_iface_param_am_alignment(params, iface->config.seg_size,
iface->config.rx_hdr_offset,
iface->config.rx_payload_offset,
&alignment, &align_offset);
if (status != UCS_OK) {
return status;
}
return uct_iface_mpool_init(&iface->super, mp,
iface->config.rx_hdr_offset +
iface->config.seg_size,
align_offset, alignment, &config->rx.mp, grow,
uct_ib_iface_recv_desc_init, name);
}
void uct_ib_iface_release_desc(uct_recv_desc_t *self, void *desc)
{
uct_ib_iface_t *iface = ucs_container_of(self, uct_ib_iface_t, release_desc);
void *ib_desc;
ib_desc = UCS_PTR_BYTE_OFFSET(desc, -(ptrdiff_t)iface->config.rx_headroom_offset);
ucs_mpool_put_inline(ib_desc);
}
static inline uct_ib_roce_version_t
uct_ib_address_flags_get_roce_version(uint8_t flags)
{
ucs_assert(flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH);
return (uct_ib_roce_version_t)(flags >> ucs_ilog2(UCT_IB_ADDRESS_FLAG_ETH_LAST));
}
static inline sa_family_t
uct_ib_address_flags_get_roce_af(uint8_t flags)
{
ucs_assert(flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH);
return (flags & UCT_IB_ADDRESS_FLAG_ROCE_IPV6) ?
AF_INET6 : AF_INET;
}
size_t uct_ib_address_size(const uct_ib_address_pack_params_t *params)
{
size_t size = sizeof(uct_ib_address_t);
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
/* Ethernet: address contains only raw GID */
size += sizeof(union ibv_gid);
} else {
/* InfiniBand: address always contains LID */
size += sizeof(uint16_t); /* lid */
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID) {
/* Add GUID */
UCS_STATIC_ASSERT(sizeof(params->gid.global.interface_id) == sizeof(uint64_t));
size += sizeof(uint64_t);
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX) {
if ((params->gid.global.subnet_prefix & UCT_IB_SITE_LOCAL_MASK) ==
UCT_IB_SITE_LOCAL_PREFIX) {
/* 16-bit subnet prefix */
size += sizeof(uint16_t);
} else if (params->gid.global.subnet_prefix != UCT_IB_LINK_LOCAL_PREFIX) {
/* 64-bit subnet prefix */
size += sizeof(uint64_t);
}
/* Note: if subnet prefix is LINK_LOCAL, no need to pack it because
* it's a well-known value defined by IB specification.
*/
}
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU) {
size += sizeof(uint8_t);
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX) {
size += sizeof(uint8_t);
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_PKEY) {
size += sizeof(uint16_t);
}
return size;
}
static int uct_ib_address_gid_is_site_local(const union ibv_gid *gid)
{
return (gid->global.subnet_prefix & UCT_IB_SITE_LOCAL_MASK) ==
UCT_IB_SITE_LOCAL_PREFIX;
}
static int uct_ib_address_gid_is_global(const union ibv_gid *gid)
{
return !uct_ib_address_gid_is_site_local(gid) &&
(gid->global.subnet_prefix != UCT_IB_LINK_LOCAL_PREFIX);
}
void uct_ib_address_pack(const uct_ib_address_pack_params_t *params,
uct_ib_address_t *ib_addr)
{
void *ptr = ib_addr + 1;
union ibv_gid *gid;
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
/* RoCE, in this case we don't use the lid, we pack the gid, the RoCE
* version, address family and set the ETH flag */
ib_addr->flags = UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH |
(params->roce_info.ver <<
ucs_ilog2(UCT_IB_ADDRESS_FLAG_ETH_LAST));
if (params->roce_info.addr_family == AF_INET6) {
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_ROCE_IPV6;
}
/* uint8_t raw[16]; */
gid = ucs_serialize_next(&ptr, union ibv_gid);
memcpy(gid->raw, params->gid.raw, sizeof(params->gid.raw));
} else {
/* IB, LID */
ib_addr->flags = 0;
*ucs_serialize_next(&ptr, uint16_t) = params->lid;
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID) {
/* Pack GUID */
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_IF_ID;
*ucs_serialize_next(&ptr,
uint64_t) = params->gid.global.interface_id;
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX) {
if (uct_ib_address_gid_is_site_local(¶ms->gid)) {
/* Site-local */
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_SUBNET16;
*ucs_serialize_next(&ptr, uint16_t) =
params->gid.global.subnet_prefix >> 48;
} else if (uct_ib_address_gid_is_global(¶ms->gid)) {
/* Global or site local GID with non-zero FLID */
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_SUBNET64;
*ucs_serialize_next(&ptr, uint64_t) =
params->gid.global.subnet_prefix;
}
}
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU) {
ucs_assert((int)params->path_mtu < UINT8_MAX);
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_PATH_MTU;
*ucs_serialize_next(&ptr, uint8_t) = params->path_mtu;
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX) {
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_GID_INDEX;
*ucs_serialize_next(&ptr, uint8_t) = params->gid_index;
}
if (params->flags & UCT_IB_ADDRESS_PACK_FLAG_PKEY) {
ucs_assert(params->pkey != UCT_IB_ADDRESS_DEFAULT_PKEY);
ib_addr->flags |= UCT_IB_ADDRESS_FLAG_PKEY;
*ucs_serialize_next(&ptr, uint16_t) = params->pkey;
}
}
unsigned uct_ib_iface_address_pack_flags(uct_ib_iface_t *iface)
{
unsigned pack_flags = 0;
if (iface->pkey != UCT_IB_ADDRESS_DEFAULT_PKEY) {
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_PKEY;
}
if (uct_ib_iface_is_roce(iface)) {
/* pack Ethernet address */
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_ETH;
} else if (iface->config.force_global_addr) {
/* pack full IB address */
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX |
UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID;
} else {
/* pack only subnet prefix for reachability test */
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX;
}
if (iface->config.path_mtu != IBV_MTU_4096) {
pack_flags |= UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU;
}
return pack_flags;
}
size_t uct_ib_iface_address_size(uct_ib_iface_t *iface)
{
uct_ib_address_pack_params_t params;
params.flags = uct_ib_iface_address_pack_flags(iface);
params.gid = iface->gid_info.gid;
params.roce_info = iface->gid_info.roce_info;
return uct_ib_address_size(¶ms);
}
void uct_ib_iface_address_pack(uct_ib_iface_t *iface, uct_ib_address_t *ib_addr)
{
uct_ib_address_pack_params_t params;
params.flags = uct_ib_iface_address_pack_flags(iface);
params.gid = iface->gid_info.gid;
params.lid = uct_ib_iface_port_attr(iface)->lid;
params.roce_info = iface->gid_info.roce_info;
params.path_mtu = iface->config.path_mtu;
/* to suppress gcc 4.3.4 warning */
params.gid_index = UCT_IB_ADDRESS_INVALID_GID_INDEX;
params.pkey = iface->pkey;
uct_ib_address_pack(¶ms, ib_addr);
}
void uct_ib_address_unpack(const uct_ib_address_t *ib_addr,
uct_ib_address_pack_params_t *params_p)
{
const void *ptr = ib_addr + 1;
/* silence cppcheck warning */
uct_ib_address_pack_params_t params = {0};
uint64_t site_local_subnet;
const union ibv_gid *gid;
params.gid_index = UCT_IB_ADDRESS_INVALID_GID_INDEX;
params.path_mtu = UCT_IB_ADDRESS_INVALID_PATH_MTU;
params.pkey = UCT_IB_ADDRESS_DEFAULT_PKEY;
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH) {
/* uint8_t raw[16]; */
gid = ucs_serialize_next(&ptr, const union ibv_gid);
memcpy(params.gid.raw, gid->raw, sizeof(params.gid.raw));
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_ETH;
params.roce_info.addr_family =
uct_ib_address_flags_get_roce_af(ib_addr->flags);
params.roce_info.ver =
uct_ib_address_flags_get_roce_version(ib_addr->flags);
} else {
/* Default prefix */
params.gid.global.subnet_prefix = UCT_IB_LINK_LOCAL_PREFIX;
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX;
/* If the link layer is not ETHERNET, then it is IB and a lid
* must be present */
params.lid = *ucs_serialize_next(&ptr, const uint16_t);
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_IF_ID) {
params.gid.global.interface_id =
*ucs_serialize_next(&ptr, const uint64_t);
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID;
}
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_SUBNET16) {
site_local_subnet = *ucs_serialize_next(&ptr, const uint16_t);
params.gid.global.subnet_prefix = UCT_IB_SITE_LOCAL_PREFIX |
(site_local_subnet << 48);
ucs_assert(!(ib_addr->flags & UCT_IB_ADDRESS_FLAG_SUBNET64));
}
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_SUBNET64) {
params.gid.global.subnet_prefix =
*ucs_serialize_next(&ptr, const uint64_t);
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX;
}
}
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_PATH_MTU) {
params.path_mtu = (enum ibv_mtu) *
ucs_serialize_next(&ptr, const uint8_t);
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU;
}
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_GID_INDEX) {
params.gid_index = *ucs_serialize_next(&ptr, const uint8_t);
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX;
}
if (ib_addr->flags & UCT_IB_ADDRESS_FLAG_PKEY) {
params.pkey = *ucs_serialize_next(&ptr, const uint16_t);
}
/* PKEY is always in params */
params.flags |= UCT_IB_ADDRESS_PACK_FLAG_PKEY;
*params_p = params;
}
const char *uct_ib_address_str(const uct_ib_address_t *ib_addr, char *buf,
size_t max)
{
uct_ib_address_pack_params_t params;
char *p, *endp;
uct_ib_address_unpack(ib_addr, ¶ms);
p = buf;
endp = buf + max;
if (params.lid != 0) {
snprintf(p, endp - p, "lid %d ", params.lid);
p += strlen(p);
}
uct_ib_gid_str(¶ms.gid, p, endp - p);
p += strlen(p);
if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX) {
ucs_assert(params.gid_index != UCT_IB_ADDRESS_INVALID_GID_INDEX);
snprintf(p, endp - p, "gid index %u ", params.gid_index);
p += strlen(p);
}
if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU) {
ucs_assert(params.path_mtu != UCT_IB_ADDRESS_INVALID_PATH_MTU);
snprintf(p, endp - p, "mtu %zu ", uct_ib_mtu_value(params.path_mtu));
p += strlen(p);
}
ucs_assert((params.flags & UCT_IB_ADDRESS_PACK_FLAG_PKEY) &&
(params.flags != UCT_IB_ADDRESS_INVALID_PKEY));
snprintf(p, endp - p, "pkey 0x%x ", params.pkey);
return buf;
}
ucs_status_t uct_ib_iface_get_device_address(uct_iface_h tl_iface,
uct_device_addr_t *dev_addr)
{
uct_ib_iface_t *iface = ucs_derived_of(tl_iface, uct_ib_iface_t);
uct_ib_iface_address_pack(iface, (void*)dev_addr);
return UCS_OK;
}
static int
uct_ib_iface_roce_is_reachable(const uct_ib_device_gid_info_t *local_gid_info,
const uct_ib_address_t *remote_ib_addr,
unsigned prefix_bits)
{
sa_family_t local_ib_addr_af = local_gid_info->roce_info.addr_family;
uct_ib_roce_version_t local_roce_ver = local_gid_info->roce_info.ver;
uint8_t remote_ib_addr_flags = remote_ib_addr->flags;
uct_ib_roce_version_t remote_roce_ver;
sa_family_t remote_ib_addr_af;
char local_str[128], remote_str[128];
uint8_t *local_addr, *remote_addr;
ucs_status_t status;
size_t addr_offset;
size_t addr_size;
int ret;
/* check for wildcards in the RoCE version (RDMACM or non-RoCE cases) */
if ((uct_ib_address_flags_get_roce_version(remote_ib_addr_flags)) ==
UCT_IB_DEVICE_ROCE_ANY) {
return 1;
}
/* check for zero-sized netmask */
if (prefix_bits == 0) {
return 1;
}
/* check the address family */
remote_ib_addr_af = uct_ib_address_flags_get_roce_af(remote_ib_addr_flags);
if (local_ib_addr_af != remote_ib_addr_af) {
ucs_assert(local_ib_addr_af != 0);
ucs_debug("different addr_family detected. local %s remote %s",
ucs_sockaddr_address_family_str(local_ib_addr_af),
ucs_sockaddr_address_family_str(remote_ib_addr_af));
return 0;
}
/* check the RoCE version */
ucs_assert(local_roce_ver != UCT_IB_DEVICE_ROCE_ANY);
remote_roce_ver = uct_ib_address_flags_get_roce_version(remote_ib_addr_flags);
if (local_roce_ver != remote_roce_ver) {
ucs_debug("different RoCE versions detected. local %s (gid=%s)"
"remote %s (gid=%s)",
uct_ib_roce_version_str(local_roce_ver),
uct_ib_gid_str(&local_gid_info->gid, local_str,
sizeof(local_str)),
uct_ib_roce_version_str(remote_roce_ver),
uct_ib_gid_str((union ibv_gid*)(remote_ib_addr + 1),
remote_str, sizeof(remote_str)));
return 0;
}
if (local_gid_info->roce_info.ver != UCT_IB_DEVICE_ROCE_V2) {
return 1; /* We assume it is, but actually there's no good test */
}
status = ucs_sockaddr_inet_addr_size(local_ib_addr_af, &addr_size);
if (status != UCS_OK) {
ucs_error("failed to detect RoCE address size");
return 0;
}
addr_offset = sizeof(union ibv_gid) - addr_size;
local_addr = UCS_PTR_BYTE_OFFSET(&local_gid_info->gid, addr_offset);
remote_addr = UCS_PTR_BYTE_OFFSET(&remote_ib_addr->flags + 1, addr_offset);
/* sanity check on the subnet mask size (bits belonging to the prefix) */
ucs_assert((prefix_bits / 8) <= addr_size);
/* check if the addresses have matching prefixes */
ret = ucs_bitwise_is_equal(local_addr, remote_addr, prefix_bits);
ucs_debug(ret ? "IP addresses match with a %u-bit prefix: local IP is %s,"
" remote IP is %s" :
"IP addresses do not match with a %u-bit prefix. local IP"
" is %s, remote IP is %s",
prefix_bits,
inet_ntop(local_ib_addr_af, local_addr, local_str, 128),
inet_ntop(remote_ib_addr_af, remote_addr, remote_str, 128));
return ret;
}
int uct_ib_iface_is_same_device(const uct_ib_address_t *ib_addr, uint16_t dlid,
const union ibv_gid *dgid)
{
uct_ib_address_pack_params_t params;
uct_ib_address_unpack(ib_addr, ¶ms);
if (!(params.flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) &&
(dlid != params.lid)) {
return 0;
}
if (dgid == NULL) {
return !(params.flags & (UCT_IB_ADDRESS_PACK_FLAG_ETH |
UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID));
}
if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
return !memcmp(dgid->raw, params.gid.raw, sizeof(params.gid.raw));
}
return !(params.flags & UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID) ||
(params.gid.global.interface_id == dgid->global.interface_id);
}
static int uct_ib_iface_gid_extract_flid(const union ibv_gid *gid)
{
if ((gid->global.subnet_prefix & UCT_IB_SITE_LOCAL_FLID_MASK) !=
UCT_IB_SITE_LOCAL_PREFIX) {
return 0;
}
return ntohs(*((uint16_t*)UCS_PTR_BYTE_OFFSET(gid->raw, 4)));
}
static int uct_ib_iface_is_flid_enabled(uct_ib_iface_t *iface)
{
return iface->config.flid_enabled &&
(uct_ib_iface_gid_extract_flid(&iface->gid_info.gid) != 0);
}
static int uct_ib_iface_dev_addr_is_reachable(uct_ib_iface_t *iface,
const uct_ib_address_t *ib_addr)
{
int is_local_eth = uct_ib_iface_is_roce(iface);
uct_ib_address_pack_params_t params;
uct_ib_address_unpack(ib_addr, ¶ms);
if (/* at least one PKEY has to be with full membership */
!((params.pkey | iface->pkey) & UCT_IB_PKEY_MEMBERSHIP_MASK) ||
/* PKEY values have to be equal */
((params.pkey ^ iface->pkey) & UCT_IB_PKEY_PARTITION_MASK)) {
return 0;
}
if (!is_local_eth && !(ib_addr->flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH)) {
if (params.gid.global.subnet_prefix ==
iface->gid_info.gid.global.subnet_prefix) {
return 1;
}
/* Check FLID route: is enabled locally, and remote GID has it */
return (uct_ib_iface_is_flid_enabled(iface) &&
uct_ib_iface_gid_extract_flid(¶ms.gid) != 0);
} else if (is_local_eth && (ib_addr->flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH)) {
/* there shouldn't be a lid and the UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH
* flag should be on. If reachable, the remote and local RoCE versions
* and address families have to be the same */
return uct_ib_iface_roce_is_reachable(&iface->gid_info, ib_addr,
iface->addr_prefix_bits);
} else {
/* local and remote have different link layers and therefore are unreachable */
return 0;
}
}
int uct_ib_iface_is_reachable_v2(const uct_iface_h tl_iface,
const uct_iface_is_reachable_params_t *params)
{
uct_ib_iface_t *iface = ucs_derived_of(tl_iface, uct_ib_iface_t);
const uct_ib_address_t *device_addr;
uct_iface_reachability_scope_t scope;
if (!uct_iface_is_reachable_params_valid(
params, UCT_IFACE_IS_REACHABLE_FIELD_DEVICE_ADDR)) {
return 0;
}
device_addr = (const uct_ib_address_t*)
UCS_PARAM_VALUE(UCT_IFACE_IS_REACHABLE_FIELD, params, device_addr,
DEVICE_ADDR, NULL);
if (device_addr == NULL) {
return 0;
}
if (!uct_ib_iface_dev_addr_is_reachable(iface, device_addr)) {
return 0;
}
scope = UCS_PARAM_VALUE(UCT_IFACE_IS_REACHABLE_FIELD, params, scope, SCOPE,
UCT_IFACE_REACHABILITY_SCOPE_NETWORK);
return (scope == UCT_IFACE_REACHABILITY_SCOPE_NETWORK) ||
uct_ib_iface_is_same_device(device_addr,
uct_ib_iface_port_attr(iface)->lid,
&iface->gid_info.gid);
}
ucs_status_t uct_ib_iface_create_ah(uct_ib_iface_t *iface,
struct ibv_ah_attr *ah_attr,
const char *usage, struct ibv_ah **ah_p)
{
return uct_ib_device_create_ah_cached(uct_ib_iface_device(iface), ah_attr,
uct_ib_iface_md(iface)->pd, usage,
ah_p);
}
void uct_ib_iface_fill_ah_attr_from_gid_lid(uct_ib_iface_t *iface, uint16_t lid,
const union ibv_gid *gid,
uint8_t gid_index,
unsigned path_index,
struct ibv_ah_attr *ah_attr)
{
uint8_t path_bits;
char buf[128];
memset(ah_attr, 0, sizeof(*ah_attr));
ucs_assert(iface->config.sl < UCT_IB_SL_NUM);
ah_attr->sl = iface->config.sl;
ah_attr->port_num = iface->config.port_num;
ah_attr->grh.traffic_class = iface->config.traffic_class;
if (uct_ib_iface_is_roce(iface)) {
ah_attr->dlid = UCT_IB_ROCE_UDP_SRC_PORT_BASE |
(iface->config.roce_path_factor * path_index);
/* Workaround rdma-core flow label to udp sport conversion */
ah_attr->grh.flow_label = ~(iface->config.roce_path_factor * path_index);
} else {
/* TODO iface->path_bits should be removed and replaced by path_index */
path_bits = iface->path_bits[path_index %
iface->path_bits_count];
ah_attr->dlid = lid | path_bits;
ah_attr->src_path_bits = path_bits;
}
if ((gid != NULL) &&
(iface->config.force_global_addr ||
(iface->gid_info.gid.global.subnet_prefix !=
gid->global.subnet_prefix))) {
ucs_assert_always(gid->global.interface_id != 0);
ah_attr->is_global = 1;
ah_attr->grh.dgid = *gid;
ah_attr->grh.sgid_index = gid_index;
ah_attr->grh.hop_limit = iface->config.hop_limit;
} else {
ah_attr->is_global = 0;
}
ucs_debug("iface %p: ah_attr %s", iface,
uct_ib_ah_attr_str(buf, sizeof(buf), ah_attr));
}
static uint16_t uct_ib_gid_site_local_subnet_prefix(const union ibv_gid *gid)
{
return be64toh(gid->global.subnet_prefix) & 0xffff;
}
uint16_t uct_ib_iface_resolve_remote_flid(uct_ib_iface_t *iface,
const union ibv_gid *gid)
{
if (!uct_ib_iface_is_flid_enabled(iface)) {
return 0;
}
if (uct_ib_gid_site_local_subnet_prefix(gid) ==
uct_ib_gid_site_local_subnet_prefix(&iface->gid_info.gid)) {
/* On the same subnet, no need to use FLID*/
return 0;
}
return uct_ib_iface_gid_extract_flid(gid);
}
void uct_ib_iface_fill_ah_attr_from_addr(uct_ib_iface_t *iface,
const uct_ib_address_t *ib_addr,
unsigned path_index,
struct ibv_ah_attr *ah_attr,
enum ibv_mtu *path_mtu)
{
union ibv_gid *gid = NULL;
uint16_t lid, flid = 0;
uct_ib_address_pack_params_t params;
ucs_assert(!uct_ib_iface_is_roce(iface) ==
!(ib_addr->flags & UCT_IB_ADDRESS_FLAG_LINK_LAYER_ETH));
uct_ib_address_unpack(ib_addr, ¶ms);
if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_PATH_MTU) {
ucs_assert(params.path_mtu != UCT_IB_ADDRESS_INVALID_PATH_MTU);
*path_mtu = params.path_mtu;
} else {
*path_mtu = iface->config.path_mtu;
}
if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_GID_INDEX) {
ucs_assert(params.gid_index != UCT_IB_ADDRESS_INVALID_GID_INDEX);
} else {
params.gid_index = iface->gid_info.gid_index;
}
if (ucs_test_all_flags(params.flags,
UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID |
UCT_IB_ADDRESS_PACK_FLAG_SUBNET_PREFIX) ||
params.flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
gid = ¶ms.gid;
flid = uct_ib_iface_resolve_remote_flid(iface, gid);
}
lid = (flid == 0) ? params.lid : flid;
uct_ib_iface_fill_ah_attr_from_gid_lid(iface, lid, gid, params.gid_index,
path_index, ah_attr);
}
static ucs_status_t uct_ib_iface_init_pkey(uct_ib_iface_t *iface,
const uct_ib_iface_config_t *config)
{
uct_ib_device_t *dev = uct_ib_iface_device(iface);
uint16_t pkey_tbl_len = uct_ib_iface_port_attr(iface)->pkey_tbl_len;
uint16_t lim_pkey = UCT_IB_ADDRESS_INVALID_PKEY;
uint16_t lim_pkey_index = UINT16_MAX;
uint16_t pkey_index, port_pkey, pkey;
if (uct_ib_iface_is_roce(iface)) {
/* RoCE: use PKEY index 0, which contains the default PKEY: 0xffff */
iface->pkey_index = 0;
iface->pkey = UCT_IB_PKEY_DEFAULT;
goto out_pkey_found;
}
if ((config->pkey != UCS_HEXUNITS_AUTO) &&
(config->pkey > UCT_IB_PKEY_PARTITION_MASK)) {
ucs_error("requested pkey 0x%x is invalid, should be in the range 0..0x%x",
config->pkey, UCT_IB_PKEY_PARTITION_MASK);
return UCS_ERR_INVALID_PARAM;
}
/* get the user's pkey value and find its index in the port's pkey table */
for (pkey_index = 0; pkey_index < pkey_tbl_len; ++pkey_index) {
/* get the pkey values from the port's pkeys table */
if (ibv_query_pkey(dev->ibv_context, iface->config.port_num, pkey_index,
&port_pkey))
{
ucs_debug("ibv_query_pkey("UCT_IB_IFACE_FMT", index=%d) failed: %m",
UCT_IB_IFACE_ARG(iface), pkey_index);
continue;
}
pkey = ntohs(port_pkey);
/* if pkey = 0x0, just skip it w/o debug trace, because 0x0
* means that there is no real pkey configured at this index */
if (pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
continue;
}
if ((config->pkey == UCS_HEXUNITS_AUTO) ||
/* take only the lower 15 bits for the comparison */
((pkey & UCT_IB_PKEY_PARTITION_MASK) == config->pkey)) {
if (pkey & UCT_IB_PKEY_MEMBERSHIP_MASK) {
iface->pkey_index = pkey_index;
iface->pkey = pkey;
goto out_pkey_found;
} else if (lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
/* limited PKEY has not yet been found */
lim_pkey_index = pkey_index;
lim_pkey = pkey;
}
}
}
if (lim_pkey == UCT_IB_ADDRESS_INVALID_PKEY) {
/* PKEY neither with full nor with limited membership was found */
if (config->pkey == UCS_HEXUNITS_AUTO) {
ucs_error("there is no valid pkey to use on " UCT_IB_IFACE_FMT,
UCT_IB_IFACE_ARG(iface));
} else {
ucs_error("unable to find specified pkey 0x%x on "UCT_IB_IFACE_FMT,
config->pkey, UCT_IB_IFACE_ARG(iface));
}
return UCS_ERR_NO_ELEM;
}
ucs_assertv(lim_pkey_index < pkey_tbl_len, "lim_pkey_index=%u"
" pkey_tbl_len=%u", lim_pkey_index, pkey_tbl_len);