/
dpdk_setup_ports.py
executable file
·1369 lines (1187 loc) · 60.3 KB
/
dpdk_setup_ports.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#! /usr/bin/python
# hhaim
import sys
import os
python_ver = 'python%s' % sys.version_info[0]
yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)
if yaml_path not in sys.path:
sys.path.append(yaml_path)
import yaml
import dpdk_nic_bind
import re
import argparse
import copy
import shlex
import traceback
from collections import defaultdict, OrderedDict
from distutils.util import strtobool
import subprocess
import platform
import stat
import time
# exit code is Important should be
# -1 : don't continue
# 0 : no errors - no need to load mlx share object
# 32 : no errors - mlx share object should be loaded
# 64 : no errors - napatech 3GD should be running
MLX_EXIT_CODE = 32
NTACC_EXIT_CODE = 64
class VFIOBindErr(Exception): pass
PATH_ARR = os.getenv('PATH', '').split(':')
for path in ['/usr/local/sbin', '/usr/sbin', '/sbin']:
if path not in PATH_ARR:
PATH_ARR.append(path)
os.environ['PATH'] = ':'.join(PATH_ARR)
def if_list_remove_sub_if(if_list):
return if_list
class ConfigCreator(object):
mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
_2hex_re = '[\da-fA-F]{2}'
mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
MAC_LCORE_NUM = 63 # current bitmask is 64 bit
# cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
# interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
self.cpu_topology = copy.deepcopy(cpu_topology)
self.interfaces = copy.deepcopy(interfaces)
del cpu_topology
del interfaces
assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
if only_first_thread:
for cores in self.cpu_topology.values():
for core in cores.keys():
cores[core] = cores[core][:1]
include_lcores = [int(x) for x in include_lcores]
exclude_lcores = [int(x) for x in exclude_lcores]
self.has_zero_lcore = False
self.lcores_per_numa = {}
total_lcores = 0
for numa, cores in self.cpu_topology.items():
self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
for core, lcores in cores.items():
total_lcores += len(lcores)
for lcore in list(lcores):
if include_lcores and lcore not in include_lcores:
cores[core].remove(lcore)
if exclude_lcores and lcore in exclude_lcores:
cores[core].remove(lcore)
if lcore > self.MAC_LCORE_NUM:
cores[core].remove(lcore)
if 0 in lcores:
self.has_zero_lcore = True
lcores.remove(0)
self.lcores_per_numa[numa]['siblings'].extend(lcores)
else:
self.lcores_per_numa[numa]['main'].extend(lcores[:1])
self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
self.lcores_per_numa[numa]['all'].extend(lcores)
for interface in self.interfaces:
for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
if mandatory_interface_field not in interface:
raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
Device_str = self._verify_devices_same_type(self.interfaces)
if '40Gb' in Device_str:
self.speed = 40
else:
self.speed = 10
minimum_required_lcores = len(self.interfaces) // 2 + 2
if total_lcores < minimum_required_lcores:
raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
(minimum_required_lcores, len(self.interfaces), total_lcores))
interfaces_per_numa = defaultdict(int)
for i in range(0, len(self.interfaces), 2):
numa = self.interfaces[i]['NUMA']
if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
(numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
interfaces_per_numa[numa] += 2
self.interfaces_per_numa = interfaces_per_numa
self.prefix = prefix
self.zmq_pub_port = zmq_pub_port
self.zmq_rpc_port = zmq_rpc_port
self.ignore_numa = ignore_numa
@staticmethod
def verify_mac(mac_string):
if not ConfigCreator.mac_re.match(mac_string):
raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
return mac_string.lower()
@staticmethod
def _exit_if_bad_ip(ip):
if not ConfigCreator._verify_ip(ip):
raise DpdkSetup("Got bad IP %s" % ip)
@staticmethod
def _verify_ip(ip):
a = ip.split('.')
if len(a) != 4:
return False
for x in a:
if not x.isdigit():
return False
i = int(x)
if i < 0 or i > 255:
return False
return True
@staticmethod
def _verify_devices_same_type(interfaces_list):
Device_str = interfaces_list[0]['Device_str']
for interface in interfaces_list:
if Device_str != interface['Device_str']:
raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
return Device_str
def create_config(self, filename = None, print_config = False):
config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
config_str += '- port_limit: %s\n' % len(self.interfaces)
config_str += ' version: 2\n'
config_str += " interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
if self.speed > 10:
config_str += ' port_bandwidth_gb: %s\n' % self.speed
if self.prefix:
config_str += ' prefix: %s\n' % self.prefix
if self.zmq_pub_port:
config_str += ' zmq_pub_port: %s\n' % self.zmq_pub_port
if self.zmq_rpc_port:
config_str += ' zmq_rpc_port: %s\n' % self.zmq_rpc_port
config_str += ' port_info:\n'
for index, interface in enumerate(self.interfaces):
if 'ip' in interface:
self._exit_if_bad_ip(interface['ip'])
self._exit_if_bad_ip(interface['def_gw'])
config_str += ' '*6 + '- ip: %s\n' % interface['ip']
config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
else:
config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
if interface.get('loopback_dest'):
config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
else:
config_str += '\n'
config_str += ' '*8 + 'src_mac: %s\n' % self.verify_mac(interface['src_mac'])
if index % 2:
config_str += '\n' # dual if barrier
if not self.ignore_numa:
config_str += ' platform:\n'
if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
config_str += ' '*6 + 'dual_if:\n'
for i in range(0, len(self.interfaces), 2):
lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
lcores_pool = lcores_pool[lcores_per_dual_if:]
if not lcores_for_this_dual_if:
raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
config_str += ' '*8 + '- socket: 0\n'
config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
else:
# we will take common minimum among all NUMAs, to satisfy all
lcores_per_dual_if = 99
extra_lcores = 1 if self.has_zero_lcore else 2
# worst case 3 iterations, to ensure master and "rx" have cores left
while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
lcores_per_dual_if -= 1
for numa, lcores_dict in self.lcores_per_numa.items():
if not self.interfaces_per_numa[numa]:
continue
lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
lcores_pool = copy.deepcopy(self.lcores_per_numa)
# first, allocate lcores for dual_if section
dual_if_section = ' '*6 + 'dual_if:\n'
for i in range(0, len(self.interfaces), 2):
numa = self.interfaces[i]['NUMA']
dual_if_section += ' '*8 + '- socket: %s\n' % numa
lcores_for_this_dual_if = lcores_pool[numa]['all'][:lcores_per_dual_if]
lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
for lcore in lcores_for_this_dual_if:
if lcore in lcores_pool[numa]['main']:
lcores_pool[numa]['main'].remove(lcore)
elif lcore in lcores_pool[numa]['siblings']:
lcores_pool[numa]['siblings'].remove(lcore)
else:
raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
if not lcores_for_this_dual_if:
raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))
# take the cores left to master and rx
mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
if mains_left:
rx_core = mains_left.pop(0)
else:
rx_core = siblings_left.pop(0)
if self.has_zero_lcore:
master_core = 0
elif mains_left:
master_core = mains_left.pop(0)
else:
master_core = siblings_left.pop(0)
config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
# add the dual_if section
config_str += dual_if_section
# verify config is correct YAML format
try:
yaml.safe_load(config_str)
except Exception as e:
raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
if print_config:
print(config_str)
if filename:
if os.path.exists(filename):
if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
print('Skipping.')
return config_str
with open(filename, 'w') as f:
f.write(config_str)
print('Saved to %s.' % filename)
return config_str
# only load igb_uio if it's available
def load_igb_uio():
loaded_mods = dpdk_nic_bind.get_loaded_modules()
if 'igb_uio' in loaded_mods:
return True
if 'uio' not in loaded_mods:
ret = os.system('modprobe uio')
if ret:
return False
km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
if os.path.exists(km):
return os.system('insmod %s' % km) == 0
# try to compile igb_uio if it's missing
def compile_and_load_igb_uio():
loaded_mods = dpdk_nic_bind.get_loaded_modules()
if 'igb_uio' in loaded_mods:
return
if 'uio' not in loaded_mods:
ret = os.system('modprobe uio')
if ret:
print('Failed inserting uio module, please check if it is installed')
sys.exit(-1)
km = './ko/%s/igb_uio.ko' % dpdk_nic_bind.kernel_ver
if not os.path.exists(km):
print("ERROR: We don't have precompiled igb_uio.ko module for your kernel version")
print('Will try compiling automatically - make sure you have file-system read/write permission')
try:
subprocess.check_output('make', cwd = './ko/src', stderr = subprocess.STDOUT)
subprocess.check_output(['make', 'install'], cwd = './ko/src', stderr = subprocess.STDOUT)
print('\nSuccess.')
except Exception as e:
print('\n ERROR: Automatic compilation failed: (%s)' % e)
print('Make sure you have file-system read/write permission')
print('You can try compiling yourself, using the following commands:')
print(' $cd ko/src')
print(' $make')
print(' $make install')
print(' $cd -')
print('Then, try to run TRex again.')
print('Note: you might need additional Linux packages for that:')
print(' * yum based (Fedora, CentOS, RedHat):')
print(' sudo yum install kernel-devel-`uname -r`')
print(' sudo yum group install "Development tools"')
print(' * apt based (Ubuntu):')
print(' sudo apt install linux-headers-`uname -r`')
print(' sudo apt install build-essential')
sys.exit(-1)
ret = os.system('insmod %s' % km)
if ret:
print('Failed inserting igb_uio module')
sys.exit(-1)
class map_driver(object):
args=None;
cfg_file='/etc/trex_cfg.yaml'
parent_args = None
class DpdkSetup(Exception):
pass
class CIfMap:
def __init__(self, cfg_file):
self.m_cfg_file =cfg_file;
self.m_cfg_dict={};
self.m_devices={};
self.m_is_mellanox_mode=False;
def dump_error (self,err):
s="""%s
From this TRex version a configuration file must exist in /etc/ folder "
The name of the configuration file should be /etc/trex_cfg.yaml "
The minimum configuration file should include something like this
- version : 2 # version 2 of the configuration file
interfaces : ["03:00.0","03:00.1","13:00.1","13:00.0"] # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
port_limit : 2 # number of ports to use valid is 2,4,6,8,10,12
example of already bind devices
$ ./dpdk_nic_bind.py --status
Network devices using DPDK-compatible driver
============================================
0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
Network devices using kernel driver
===================================
0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
Other network devices
=====================
""" % (err);
return s;
def raise_error (self,err):
s= self.dump_error (err)
raise DpdkSetup(s)
def set_only_mellanox_nics(self):
self.m_is_mellanox_mode=True;
def get_only_mellanox_nics(self):
return self.m_is_mellanox_mode
def read_pci (self,pci_id,reg_id):
out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])
out=out.decode(errors='replace');
return (out.strip());
def write_pci (self,pci_id,reg_id,val):
out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])
out=out.decode(errors='replace');
return (out.strip());
def tune_mlx_device (self,pci_id):
# set PCIe Read to 4K and not 512 ... need to add it to startup s
val=self.read_pci (pci_id,68)
if val[0]=='0':
#hypervisor does not give the right to write to this register
return;
if val[0]!='5':
val='5'+val[1:]
self.write_pci (pci_id,68,val)
assert(self.read_pci (pci_id,68)==val);
def get_mtu_mlx (self,dev_id):
if len(dev_id)>0:
try:
out=subprocess.check_output(['ifconfig', dev_id])
except Exception as e:
raise DpdkSetup(' "ifconfig %s" utility does not works, try to install it using "$yum install net-tools -y" on CentOS system' %(dev_id) )
out=out.decode(errors='replace');
obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);
if obj:
return int(obj.group(1));
else:
obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);
if obj:
return int(obj.group(1));
else:
return -1
def set_mtu_mlx (self,dev_id,new_mtu):
if len(dev_id)>0:
out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])
out=out.decode(errors='replace');
def set_max_mtu_mlx_device(self,dev_id):
mtu=9*1024+22
dev_mtu=self.get_mtu_mlx (dev_id);
if (dev_mtu>0) and (dev_mtu!=mtu):
self.set_mtu_mlx(dev_id,mtu);
if self.get_mtu_mlx(dev_id) != mtu:
print("Could not set MTU to %d" % mtu)
sys.exit(-1);
def disable_flow_control_mlx_device (self,dev_id):
if len(dev_id)>0:
my_stderr = open("/dev/null","wb")
cmd ='ethtool -A '+dev_id + ' rx off tx off '
subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)
my_stderr.close();
def check_ofed_version (self):
ofed_info='/usr/bin/ofed_info'
ofed_ver_re = re.compile('.*[-](\d)[.](\d)[-].*')
ofed_ver= 42
ofed_ver_show= '4.2'
if not os.path.isfile(ofed_info):
print("OFED %s is not installed on this setup" % ofed_info)
sys.exit(-1);
try:
out = subprocess.check_output([ofed_info])
except Exception as e:
print("OFED %s can't run " % (ofed_info))
sys.exit(-1);
lines=out.splitlines();
if len(lines)>1:
m= ofed_ver_re.match(str(lines[0]))
if m:
ver=int(m.group(1))*10+int(m.group(2))
if ver < ofed_ver:
print("installed OFED version is '%s' should be at least '%s' and up" % (lines[0],ofed_ver_show))
sys.exit(-1);
else:
print("not found valid OFED version '%s' " % (lines[0]))
sys.exit(-1);
def verify_ofed_os(self):
err_msg = 'Warning: Mellanox NICs where tested only with RedHat/CentOS 7.4\n'
err_msg += 'Correct usage with other Linux distributions is not guaranteed.'
try:
dist = platform.dist()
if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.4'):
print(err_msg)
except Exception as e:
print('Error while determining OS type: %s' % e)
def load_config_file (self):
fcfg=self.m_cfg_file
if not os.path.isfile(fcfg) :
self.raise_error ("There is no valid configuration file %s\n" % fcfg)
try:
stream = open(fcfg, 'r')
self.m_cfg_dict= yaml.safe_load(stream)
except Exception as e:
print(e);
raise e
stream.close();
cfg_dict = self.m_cfg_dict[0]
if 'version' not in cfg_dict:
raise DpdkSetup("Configuration file %s is old, it should include version field\n" % fcfg )
if int(cfg_dict['version'])<2 :
raise DpdkSetup("Configuration file %s is old, expected version 2, got: %s\n" % (fcfg, cfg_dict['version']))
if 'interfaces' not in self.m_cfg_dict[0]:
raise DpdkSetup("Configuration file %s is old, it should include interfaces field with even number of elements" % fcfg)
if_list= if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces']);
l=len(if_list);
if l > 16:
raise DpdkSetup("Configuration file %s should include interfaces field with maximum 16 elements, got: %s." % (fcfg,l))
if l % 2:
raise DpdkSetup("Configuration file %s should include even number of interfaces, got: %s" % (fcfg,l))
if 'port_limit' in cfg_dict:
if cfg_dict['port_limit'] > len(if_list):
raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
if cfg_dict['port_limit'] % 2:
raise DpdkSetup('Error: port_limit in config file must be even number, got: %s\n' % cfg_dict['port_limit'])
if cfg_dict['port_limit'] <= 0:
raise DpdkSetup('Error: port_limit in config file must be positive number, got: %s\n' % cfg_dict['port_limit'])
if map_driver.parent_args and map_driver.parent_args.limit_ports is not None:
if map_driver.parent_args.limit_ports > len(if_list):
raise DpdkSetup('Error: --limit-ports CLI argument (%s) must not be higher than number of interfaces (%s) in config file: %s\n' % (map_driver.parent_args.limit_ports, len(if_list), fcfg))
def do_bind_all(self, drv, pci, force = False):
assert type(pci) is list
cmd = '{ptn} dpdk_nic_bind.py --bind={drv} {pci} {frc}'.format(
ptn = sys.executable,
drv = drv,
pci = ' '.join(pci),
frc = '--force' if force else '')
print(cmd)
return os.system(cmd)
# pros: no need to compile .ko per Kernel version
# cons: need special config/hw (not always works)
def try_bind_to_vfio_pci(self, to_bind_list):
krnl_params_file = '/proc/cmdline'
if not os.path.exists(krnl_params_file):
raise VFIOBindErr('Could not find file with Kernel boot parameters: %s' % krnl_params_file)
with open(krnl_params_file) as f:
krnl_params = f.read()
if 'iommu=' not in krnl_params:
raise VFIOBindErr('vfio-pci is not an option here')
if 'vfio_pci' not in dpdk_nic_bind.get_loaded_modules():
ret = os.system('modprobe vfio_pci')
if ret:
raise VFIOBindErr('Could not load vfio_pci')
ret = self.do_bind_all('vfio-pci', to_bind_list)
if ret:
raise VFIOBindErr('Binding to vfio_pci failed')
def pci_name_to_full_name (self,pci_name):
if pci_name == 'dummy':
return pci_name
c='[0-9A-Fa-f]';
sp='[:]'
s_short=c+c+sp+c+c+'[.]'+c;
s_full=c+c+c+c+sp+s_short
re_full = re.compile(s_full)
re_short = re.compile(s_short)
if re_short.match(pci_name):
return '0000:'+pci_name
if re_full.match(pci_name):
return pci_name
err=" %s is not a valid pci address \n" %pci_name;
raise DpdkSetup(err)
def run_dpdk_lspci (self):
dpdk_nic_bind.get_nic_details()
self.m_devices= dpdk_nic_bind.devices
def preprocess_astf_file_is_needed(self):
""" check if we are in astf mode, in case we are convert the profile to json in tmp"""
is_astf_mode = map_driver.parent_args and map_driver.parent_args.astf
if is_astf_mode:
input_file = map_driver.parent_args.file
if not input_file:
return
just_copy=False
extension = os.path.splitext(input_file)[1]
if extension == '.json':
just_copy = True;
else:
if extension != '.py':
raise DpdkSetup('ERROR when running with --astf mode, you need to have a new python profile format (.py) and not YAML')
instance_name = ""
if map_driver.parent_args.prefix is not '':
instance_name = "-" + map_driver.parent_args.prefix
elif 'prefix' in self.m_cfg_dict[0]:
instance_name = '-' + self.m_cfg_dict[0]['prefix']
json_file = "/tmp/astf{instance}.json".format(instance=instance_name)
msg="converting astf profile {file} to json {out}".format(file = input_file, out=json_file)
print(msg);
tunable='';
if map_driver.parent_args.tunable:
tunable="-t "+map_driver.parent_args.tunable+" "
if just_copy:
cmd = 'cp {file} {json_file}'.format(file=input_file, json_file=json_file)
else:
cmd = './astf-sim -f {file} {tun} --json > {json_file}'.format(file=input_file, tun=tunable, json_file=json_file)
print(cmd)
ret = os.system(cmd)
os.chmod(json_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
if ret:
raise DpdkSetup('ERROR could not convert astf profile to JSON try to debug it using the command above.')
def config_hugepages(self, wanted_count = None):
huge_mnt_dir = '/mnt/huge'
if not os.path.isdir(huge_mnt_dir):
print("Creating huge node")
os.makedirs(huge_mnt_dir)
mount_output = subprocess.check_output('mount', stderr = subprocess.STDOUT).decode(errors='replace')
if 'hugetlbfs' not in mount_output:
os.system('mount -t hugetlbfs nodev %s' % huge_mnt_dir)
for socket_id in range(2):
filename = '/sys/devices/system/node/node%d/hugepages/hugepages-2048kB/nr_hugepages' % socket_id
if not os.path.isfile(filename):
if socket_id == 0:
print('WARNING: hugepages config file (%s) does not exist!' % filename)
continue
if wanted_count is None:
if self.m_cfg_dict[0].get('low_end', False):
if socket_id == 0:
if map_driver.parent_args and map_driver.parent_args.limit_ports:
if_count = map_driver.parent_args.limit_ports
else:
if_count = self.m_cfg_dict[0].get('port_limit', len(self.m_cfg_dict[0]['interfaces']))
wanted_count = 20 + 40 * if_count
else:
wanted_count = 1 # otherwise, DPDK will not be able to see the device
else:
wanted_count = 2048
with open(filename) as f:
configured_hugepages = int(f.read())
if configured_hugepages < wanted_count:
os.system('echo %d > %s' % (wanted_count, filename))
time.sleep(0.1)
with open(filename) as f: # verify
configured_hugepages = int(f.read())
if configured_hugepages < wanted_count:
print('WARNING: tried to configure %d hugepages for socket %d, but result is: %d' % (wanted_count, socket_id, configured_hugepages))
def run_scapy_server(self):
if map_driver.parent_args and map_driver.parent_args.stl and not map_driver.parent_args.no_scapy_server:
try:
master_core = self.m_cfg_dict[0]['platform']['master_thread_id']
except:
master_core = 0
ret = os.system('%s scapy_daemon_server restart -c %s' % (sys.executable, master_core))
if ret:
print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
sys.exit(-1)
# check vdev Linux interfaces status
# return True if interfaces are vdev
def check_vdev(self, if_list):
if not if_list:
return
af_names = []
ifname_re = re.compile('iface\s*=\s*([^\s,]+)')
found_vdev = False
found_pdev = False
for iface in if_list:
if iface == 'dummy':
continue
elif '--vdev' in iface:
found_vdev = True
if 'net_af_packet' in iface:
res = ifname_re.search(iface)
if res:
af_names.append(res.group(1))
elif ':' not in iface: # no PCI => assume af_packet
found_vdev = True
af_names.append(iface)
else:
found_pdev = True
if found_vdev:
if found_pdev:
raise DpdkSetup('You have mix of vdev and pdev interfaces in config file!')
for name in af_names:
if not os.path.exists('/sys/class/net/%s' % name):
raise DpdkSetup('ERROR: Could not find Linux interface %s.' % name)
oper_state = '/sys/class/net/%s/operstate' % name
if os.path.exists(oper_state):
with open(oper_state) as f:
f_cont = f.read().strip()
if f_cont in ('down', 'DOWN'):
raise DpdkSetup('ERROR: Requested Linux interface %s is DOWN.' % name)
return found_vdev
def check_trex_running(self, if_list):
if if_list and map_driver.args.parent and self.m_cfg_dict[0].get('enable_zmq_pub', True):
publisher_port = self.m_cfg_dict[0].get('zmq_pub_port', 4500)
pid = dpdk_nic_bind.get_tcp_port_usage(publisher_port)
if pid:
cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
print('ZMQ port is used by following process:\npid: %s, cmd: %s' % (pid, cmdline))
sys.exit(-1)
def do_run (self,only_check_all_mlx=False):
""" return the number of mellanox drivers"""
self.run_dpdk_lspci ()
self.load_config_file()
self.preprocess_astf_file_is_needed()
if (map_driver.parent_args is None or
map_driver.parent_args.dump_interfaces is None or
(map_driver.parent_args.dump_interfaces == [] and
map_driver.parent_args.cfg)):
if_list=if_list_remove_sub_if(self.m_cfg_dict[0]['interfaces'])
else:
if_list = map_driver.parent_args.dump_interfaces
if not if_list:
for dev in self.m_devices.values():
if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel:
if_list.append(dev['Slot'])
if self.check_vdev(if_list):
self.check_trex_running(if_list)
self.run_scapy_server()
# no need to config hugepages
return
if_list = list(map(self.pci_name_to_full_name, if_list))
# check how many mellanox cards we have
Mellanox_cnt=0;
for key in if_list:
if key == 'dummy':
continue
key = self.split_pci_key(key)
if key not in self.m_devices:
err=" %s does not exist " %key;
raise DpdkSetup(err)
if 'Vendor_str' not in self.m_devices[key]:
err=" %s does not have Vendor_str " %key;
raise DpdkSetup(err)
if 'Mellanox' in self.m_devices[key]['Vendor_str']:
Mellanox_cnt += 1
if not (map_driver.parent_args and map_driver.parent_args.dump_interfaces):
if (Mellanox_cnt > 0) and (Mellanox_cnt != len(if_list)):
err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
raise DpdkSetup(err)
if Mellanox_cnt > 0:
self.set_only_mellanox_nics()
if self.get_only_mellanox_nics():
if not map_driver.parent_args.no_ofed_check:
self.verify_ofed_os()
self.check_ofed_version()
for key in if_list:
if key == 'dummy':
continue
key = self.split_pci_key(key)
if 'Virtual' not in self.m_devices[key]['Device_str']:
pci_id = self.m_devices[key]['Slot_str']
self.tune_mlx_device(pci_id)
if 'Interface' in self.m_devices[key]:
dev_ids = self.m_devices[key]['Interface'].split(",")
for dev_id in dev_ids:
self.disable_flow_control_mlx_device (dev_id)
self.set_max_mtu_mlx_device(dev_id)
if only_check_all_mlx:
if Mellanox_cnt > 0:
sys.exit(MLX_EXIT_CODE);
else:
sys.exit(0);
self.check_trex_running(if_list)
self.config_hugepages() # should be after check of running TRex
self.run_scapy_server()
Napatech_cnt=0;
to_bind_list = []
for key in if_list:
if key == 'dummy':
continue
key = self.split_pci_key(key)
if key not in self.m_devices:
err=" %s does not exist " %key;
raise DpdkSetup(err)
if 'Napatech' in self.m_devices[key]['Vendor_str']:
# These adapters doesn't need binding
Napatech_cnt += 1
continue
if self.m_devices[key].get('Driver_str') not in (dpdk_nic_bind.dpdk_drivers + dpdk_nic_bind.dpdk_and_kernel):
to_bind_list.append(key)
if Napatech_cnt:
# This is currently a hack needed until the DPDK NTACC PMD can do proper
# cleanup.
os.system("ipcs | grep 2117a > /dev/null && ipcrm shm `ipcs | grep 2117a | cut -d' ' -f2` > /dev/null")
if to_bind_list:
if Mellanox_cnt:
ret = self.do_bind_all('mlx5_core', to_bind_list)
if ret:
ret = self.do_bind_all('mlx4_core', to_bind_list)
if ret:
raise DpdkSetup('Unable to bind interfaces to driver mlx5_core/mlx4_core.')
return MLX_EXIT_CODE
else:
# if igb_uio is ready, use it as safer choice, afterwards try vfio-pci
if load_igb_uio():
print('Trying to bind to igb_uio ...')
ret = self.do_bind_all('igb_uio', to_bind_list)
if ret:
raise DpdkSetup('Unable to bind interfaces to driver igb_uio.') # module present, loaded, but unable to bind
return
try:
print('Trying to bind to vfio-pci ...')
self.try_bind_to_vfio_pci(to_bind_list)
return
except VFIOBindErr as e:
pass
#print(e)
print('Trying to compile and bind to igb_uio ...')
compile_and_load_igb_uio()
ret = self.do_bind_all('igb_uio', to_bind_list)
if ret:
raise DpdkSetup('Unable to bind interfaces to driver igb_uio.')
elif Mellanox_cnt:
return MLX_EXIT_CODE
elif Napatech_cnt:
return NTACC_EXIT_CODE
def do_return_to_linux(self):
if not self.m_devices:
self.run_dpdk_lspci()
dpdk_interfaces = []
check_drivers = set()
for device in self.m_devices.values():
if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
dpdk_interfaces.append(device['Slot'])
check_drivers.add(device['Driver_str'])
if not dpdk_interfaces:
print('No DPDK bound interfaces.')
return
any_driver_used = False
for driver in check_drivers:
if dpdk_nic_bind.is_module_used(driver):
any_driver_used = True
if any_driver_used:
pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
if pid:
cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
if not dpdk_nic_bind.confirm('Confirm (y/N):'):
sys.exit(-1)
# DPDK => Linux
drivers_table = {
'net_ixgbe': 'ixgbe',
'net_ixgbe_vf': 'ixgbevf',
'net_e1000_igb': 'igb',
'net_i40e': 'i40e',
'net_i40e_vf': 'i40evf',
'net_e1000_em': 'e1000',
'net_vmxnet3': 'vmxnet3',
'net_virtio': 'virtio-pci',
'net_enic': 'enic',
}
nics_info = dpdk_nic_bind.get_info_from_trex(dpdk_interfaces)
if not nics_info:
raise DpdkSetup('Could not determine interfaces information. Try to run manually: sudo ./t-rex-64 --dump-interfaces')
for pci, info in nics_info.items():
if pci not in self.m_devices:
raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
dev = self.m_devices[pci]
if info['TRex_Driver'] not in drivers_table:
raise DpdkSetup("Got unknown driver '%s', description: %s" % (info['TRex_Driver'], dev['Device_str']))
linux_driver = drivers_table[info['TRex_Driver']]
if linux_driver not in dpdk_nic_bind.get_loaded_modules():
print("No Linux driver installed, or wrong module name: %s" % linux_driver)
else:
print('Returning to Linux %s' % pci)
dpdk_nic_bind.bind_one(pci, linux_driver, False)
def split_pci_key(self, pci_id):
return pci_id.split('/')[0]
def _get_cpu_topology(self):
cpu_topology_file = '/proc/cpuinfo'
# physical processor -> physical core -> logical processing units (threads)
cpu_topology = OrderedDict()
if not os.path.exists(cpu_topology_file):
raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file)
with open(cpu_topology_file) as f:
for lcore in f.read().split('\n\n'):
if not lcore:
continue
lcore_dict = OrderedDict()
for line in lcore.split('\n'):
key, val = line.split(':', 1)
lcore_dict[key.strip()] = val.strip()
if 'processor' not in lcore_dict:
continue
numa = int(lcore_dict.get('physical id', -1))
if numa not in cpu_topology:
cpu_topology[numa] = OrderedDict()
core = int(lcore_dict.get('core id', lcore_dict['processor']))
if core not in cpu_topology[numa]:
cpu_topology[numa][core] = []
cpu_topology[numa][core].append(int(lcore_dict['processor']))
if not cpu_topology:
raise DpdkSetup('Could not determine CPU topology from %s' % cpu_topology_file)
return cpu_topology
# input: list of different descriptions of interfaces: index, pci, name etc.
# Binds to dpdk wanted interfaces, not bound to any driver.
# output: list of maps of devices in dpdk_* format (self.m_devices.values())
def _get_wanted_interfaces(self, input_interfaces, get_macs = True):
if type(input_interfaces) is not list:
raise DpdkSetup('type of input interfaces should be list')
if not len(input_interfaces):
raise DpdkSetup('Please specify interfaces to use in the config')
if len(input_interfaces) % 2:
raise DpdkSetup('Please specify even number of interfaces')
wanted_interfaces = []
sorted_pci = sorted(self.m_devices.keys())
for interface in input_interfaces:
dev = None
try:
interface = int(interface)
if interface < 0 or interface >= len(sorted_pci):
raise DpdkSetup('Index of an interfaces should be in range 0:%s' % (len(sorted_pci) - 1))
dev = self.m_devices[sorted_pci[interface]]
except ValueError:
for d in self.m_devices.values():
if interface in (d['Interface'], d['Slot'], d['Slot_str']):
dev = d
break
if not dev:
raise DpdkSetup('Could not find information about this interface: %s' % interface)
if dev in wanted_interfaces:
raise DpdkSetup('Interface %s is specified twice' % interface)
dev['Interface_argv'] = interface
wanted_interfaces.append(dev)
if get_macs:
unbound = []
dpdk_bound = []
for interface in wanted_interfaces:
if 'Driver_str' not in interface:
unbound.append(interface['Slot'])
elif interface.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
dpdk_bound.append(interface['Slot'])
if unbound or dpdk_bound:
for pci, info in dpdk_nic_bind.get_info_from_trex(unbound + dpdk_bound).items():
if pci not in self.m_devices:
raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
self.m_devices[pci].update(info)
return wanted_interfaces
def do_create(self):
ips = map_driver.args.ips
def_gws = map_driver.args.def_gws
dest_macs = map_driver.args.dest_macs
if map_driver.args.force_macs:
ip_config = False
if ips:
raise DpdkSetup("If using --force-macs, should not specify ips")
if def_gws:
raise DpdkSetup("If using --force-macs, should not specify default gateways")
elif ips:
ip_config = True
if not def_gws:
raise DpdkSetup("If specifying ips, must specify also def-gws")