dpdk_setup_ports.py revision de2af4eb
1#! /usr/bin/python
2# hhaim
3import sys
4import os
5python_ver = 'python%s' % sys.version_info[0]
6sys.path.append(os.path.join('external_libs', 'pyyaml-3.11', python_ver))
7import yaml
8import dpdk_nic_bind
9import re
10import argparse
11import copy
12import shlex
13import traceback
14from collections import defaultdict, OrderedDict
15from distutils.util import strtobool
16import getpass
17import subprocess
18import platform
19
20class ConfigCreator(object):
21    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
22    _2hex_re = '[\da-fA-F]{2}'
23    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
24
25    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
26    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
27    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
28        self.cpu_topology = copy.deepcopy(cpu_topology)
29        self.interfaces   = copy.deepcopy(interfaces)
30        del cpu_topology
31        del interfaces
32        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
33        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
34        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
35        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
36        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
37        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
38        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
39        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
40        if only_first_thread:
41            for cores in self.cpu_topology.values():
42                for core in cores.keys():
43                    cores[core] = cores[core][:1]
44        include_lcores = [int(x) for x in include_lcores]
45        exclude_lcores = [int(x) for x in exclude_lcores]
46
47        self.has_zero_lcore = False
48        self.lcores_per_numa = {}
49        total_lcores = 0
50        for numa, cores in self.cpu_topology.items():
51            self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
52            for core, lcores in cores.items():
53                total_lcores += len(lcores)
54                for lcore in list(lcores):
55                    if include_lcores and lcore not in include_lcores:
56                        cores[core].remove(lcore)
57                    if exclude_lcores and lcore in exclude_lcores:
58                        cores[core].remove(lcore)
59                if 0 in lcores:
60                    self.has_zero_lcore = True
61                    lcores.remove(0)
62                    self.lcores_per_numa[numa]['siblings'].extend(lcores)
63                else:
64                    self.lcores_per_numa[numa]['main'].extend(lcores[:1])
65                    self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
66                self.lcores_per_numa[numa]['all'].extend(lcores)
67
68        for interface in self.interfaces:
69            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
70                if mandatory_interface_field not in interface:
71                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
72
73        Device_str = self._verify_devices_same_type(self.interfaces)
74        if '40Gb' in Device_str:
75            self.speed = 40
76        else:
77            self.speed = 10
78
79        minimum_required_lcores = len(self.interfaces) // 2 + 2
80        if total_lcores < minimum_required_lcores:
81            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
82                    (minimum_required_lcores, len(self.interfaces), total_lcores))
83        interfaces_per_numa = defaultdict(int)
84
85        for i in range(0, len(self.interfaces), 2):
86            numa = self.interfaces[i]['NUMA']
87            if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
88                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
89                        (numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
90            interfaces_per_numa[numa] += 2
91
92        self.interfaces_per_numa = interfaces_per_numa
93        self.prefix              = prefix
94        self.zmq_pub_port        = zmq_pub_port
95        self.zmq_rpc_port        = zmq_rpc_port
96        self.ignore_numa         = ignore_numa
97
98    @staticmethod
99    def verify_mac(mac_string):
100        if not ConfigCreator.mac_re.match(mac_string):
101            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
102        return mac_string.lower()
103
104    @staticmethod
105    def _exit_if_bad_ip(ip):
106        if not ConfigCreator._verify_ip(ip):
107            raise DpdkSetup("Got bad IP %s" % ip)
108
109    @staticmethod
110    def _verify_ip(ip):
111        a = ip.split('.')
112        if len(a) != 4:
113            return False
114        for x in a:
115            if not x.isdigit():
116                return False
117            i = int(x)
118            if i < 0 or i > 255:
119                return False
120        return True
121
122    @staticmethod
123    def _verify_devices_same_type(interfaces_list):
124        Device_str = interfaces_list[0]['Device_str']
125        for interface in interfaces_list:
126            if Device_str != interface['Device_str']:
127                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
128        return Device_str
129
130    def create_config(self, filename = None, print_config = False):
131        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
132        config_str += '- port_limit: %s\n' % len(self.interfaces)
133        config_str += '  version: 2\n'
134        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
135        if self.speed > 10:
136            config_str += '  port_bandwidth_gb: %s\n' % self.speed
137        if self.prefix:
138            config_str += '  prefix: %s\n' % self.prefix
139        if self.zmq_pub_port:
140            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
141        if self.zmq_rpc_port:
142            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
143        config_str += '  port_info:\n'
144        for index, interface in enumerate(self.interfaces):
145            if 'ip' in interface:
146                self._exit_if_bad_ip(interface['ip'])
147                self._exit_if_bad_ip(interface['def_gw'])
148                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
149                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
150            else:
151                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
152                if interface.get('loopback_dest'):
153                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
154                else:
155                    config_str += '\n'
156                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
157            if index % 2:
158                config_str += '\n' # dual if barrier
159
160        if not self.ignore_numa:
161            config_str += '  platform:\n'
162            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
163                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
164                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
165                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
166                lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
167                config_str += ' '*6 + 'dual_if:\n'
168                for i in range(0, len(self.interfaces), 2):
169                    lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
170                    lcores_pool = lcores_pool[lcores_per_dual_if:]
171                    if not lcores_for_this_dual_if:
172                        raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
173                    config_str += ' '*8 + '- socket: 0\n'
174                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
175            else:
176                # we will take common minimum among all NUMAs, to satisfy all
177                lcores_per_dual_if = 99
178                extra_lcores = 1 if self.has_zero_lcore else 2
179                # worst case 3 iterations, to ensure master and "rx" have cores left
180                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
181                    lcores_per_dual_if -= 1
182                    for numa, lcores_dict in self.lcores_per_numa.items():
183                        if not self.interfaces_per_numa[numa]:
184                            continue
185                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
186                lcores_pool = copy.deepcopy(self.lcores_per_numa)
187                # first, allocate lcores for dual_if section
188                dual_if_section = ' '*6 + 'dual_if:\n'
189                for i in range(0, len(self.interfaces), 2):
190                    numa = self.interfaces[i]['NUMA']
191                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
192                    lcores_for_this_dual_if  = lcores_pool[numa]['all'][:lcores_per_dual_if]
193                    lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
194                    for lcore in lcores_for_this_dual_if:
195                        if lcore in lcores_pool[numa]['main']:
196                            lcores_pool[numa]['main'].remove(lcore)
197                        elif lcore in lcores_pool[numa]['siblings']:
198                            lcores_pool[numa]['siblings'].remove(lcore)
199                        else:
200                            raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
201                    if not lcores_for_this_dual_if:
202                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
203                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))
204
205                # take the cores left to master and rx
206                mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
207                siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
208                if mains_left:
209                    rx_core = mains_left.pop(0)
210                else:
211                    rx_core = siblings_left.pop(0)
212                if self.has_zero_lcore:
213                    master_core = 0
214                elif mains_left:
215                    master_core = mains_left.pop(0)
216                else:
217                    master_core = siblings_left.pop(0)
218                config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
219                config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
220                # add the dual_if section
221                config_str += dual_if_section
222
223        # verify config is correct YAML format
224        try:
225            yaml.safe_load(config_str)
226        except Exception as e:
227            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
228
229        if print_config:
230            print(config_str)
231        if filename:
232            if os.path.exists(filename):
233                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
234                    print('Skipping.')
235                    return config_str
236            with open(filename, 'w') as f:
237                f.write(config_str)
238            print('Saved.')
239        return config_str
240
241
242class map_driver(object):
243    args=None;
244    cfg_file='/etc/trex_cfg.yaml'
245    parent_cfg = None
246    dump_interfaces = None
247    no_ofed_check = None
248
249class DpdkSetup(Exception):
250    pass
251
252class CIfMap:
253
254    def __init__(self, cfg_file):
255        self.m_cfg_file =cfg_file;
256        self.m_cfg_dict={};
257        self.m_devices={};
258        self.m_is_mellanox_mode=False;
259
260    def dump_error (self,err):
261        s="""%s
262From this TRex version a configuration file must exist in /etc/ folder "
263The name of the configuration file should be /etc/trex_cfg.yaml "
264The minimum configuration file should include something like this
265- version       : 2 # version 2 of the configuration file
266  interfaces    : ["03:00.0","03:00.1","13:00.1","13:00.0"]  # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
267  port_limit      : 2 # number of ports to use valid is 2,4,6,8,10,12
268
269example of already bind devices
270
271$ ./dpdk_nic_bind.py --status
272
273Network devices using DPDK-compatible driver
274============================================
2750000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2760000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2770000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2780000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
279
280Network devices using kernel driver
281===================================
2820000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
283
284Other network devices
285=====================
286
287
288          """ % (err);
289        return s;
290
291
292    def raise_error  (self,err):
293        s= self.dump_error (err)
294        raise DpdkSetup(s)
295
296    def set_only_mellanox_nics(self):
297        self.m_is_mellanox_mode=True;
298
299    def get_only_mellanox_nics(self):
300        return self.m_is_mellanox_mode
301
302
303    def read_pci (self,pci_id,reg_id):
304        out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])
305        out=out.decode(errors='replace');
306        return (out.strip());
307
308    def write_pci (self,pci_id,reg_id,val):
309        out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])
310        out=out.decode(errors='replace');
311        return (out.strip());
312
313    def tune_mlx5_device (self,pci_id):
314        # set PCIe Read to 1024 and not 512 ... need to add it to startup s
315        val=self.read_pci (pci_id,68)
316        if val[0]!='3':
317            val='3'+val[1:]
318            self.write_pci (pci_id,68,val)
319            assert(self.read_pci (pci_id,68)==val);
320
321    def get_mtu_mlx5 (self,dev_id):
322        if len(dev_id)>0:
323            out=subprocess.check_output(['ifconfig', dev_id])
324            out=out.decode(errors='replace');
325            obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);
326            if obj:
327                return int(obj.group(1));
328            else:
329                obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);
330                if obj:
331                    return int(obj.group(1));
332                else:
333                    return -1
334
335    def set_mtu_mlx5 (self,dev_id,new_mtu):
336        if len(dev_id)>0:
337            out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])
338            out=out.decode(errors='replace');
339
340
341    def set_max_mtu_mlx5_device(self,dev_id):
342        mtu=9*1024+22
343        dev_mtu=self.get_mtu_mlx5 (dev_id);
344        if (dev_mtu>0) and (dev_mtu!=mtu):
345            self.set_mtu_mlx5(dev_id,mtu);
346            if self.get_mtu_mlx5(dev_id) != mtu:
347                print("Could not set MTU to %d" % mtu)
348                exit(-1);
349
350
351    def disable_flow_control_mlx5_device (self,dev_id):
352
353           if len(dev_id)>0:
354               my_stderr = open("/dev/null","wb")
355               cmd ='ethtool -A '+dev_id + ' rx off tx off '
356               subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)
357               my_stderr.close();
358
359    def check_ofed_version (self):
360        ofed_info='/usr/bin/ofed_info'
361        ofed_ver= '-3.4-'
362        ofed_ver_show= '3.4-1'
363
364
365        if not os.path.isfile(ofed_info):
366            print("OFED %s is not installed on this setup" % ofed_info)
367            exit(-1);
368
369        try:
370          out = subprocess.check_output([ofed_info])
371        except Exception as e:
372            print("OFED %s can't run " % (ofed_info))
373            exit(-1);
374
375        lines=out.splitlines();
376
377        if len(lines)>1:
378            if not (ofed_ver in str(lines[0])):
379                print("installed OFED version is '%s' should be at least '%s' and up" % (lines[0],ofed_ver_show))
380                exit(-1);
381
382    def verify_ofed_os(self):
383        err_msg = 'Warning: Mellanox NICs where tested only with RedHat/CentOS 7.2\n'
384        err_msg += 'Correct usage with other Linux distributions is not guaranteed.'
385        try:
386            dist = platform.dist()
387            if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.2'):
388                print(err_msg)
389        except Exception as e:
390            print('Error while determining OS type: %s' % e)
391
392    def load_config_file (self):
393
394        fcfg=self.m_cfg_file
395
396        if not os.path.isfile(fcfg) :
397            self.raise_error ("There is no valid configuration file %s " % fcfg)
398
399        try:
400          stream = open(fcfg, 'r')
401          self.m_cfg_dict= yaml.safe_load(stream)
402        except Exception as e:
403          print(e);
404          raise e
405
406        stream.close();
407        cfg_dict = self.m_cfg_dict[0]
408        if 'version' not in cfg_dict:
409            self.raise_error ("Configuration file %s is old, should include version field\n" % fcfg )
410
411        if int(cfg_dict['version'])<2 :
412            self.raise_error ("Configuration file %s is old, should include version field with value greater than 2\n" % fcfg)
413
414        if 'interfaces' not in self.m_cfg_dict[0]:
415            self.raise_error ("Configuration file %s is old, should include interfaces field even number of elemets" % fcfg)
416
417        if_list=self.m_cfg_dict[0]['interfaces']
418        l=len(if_list);
419        if (l>20):
420            self.raise_error ("Configuration file %s should include interfaces field with maximum of number of elemets" % (fcfg,l))
421        if ((l % 2)==1):
422            self.raise_error ("Configuration file %s should include even number of interfaces " % (fcfg,l))
423        if 'port_limit' in cfg_dict and cfg_dict['port_limit'] > len(if_list):
424            self.raise_error ('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
425
426
427    def do_bind_one (self,key,mellanox):
428        if mellanox:
429            drv="mlx5_core"
430        else:
431            drv="igb_uio"
432
433        cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
434        print(cmd)
435        res=os.system(cmd);
436        if res!=0:
437            raise DpdkSetup('')
438
439
440    def pci_name_to_full_name (self,pci_name):
441          c='[0-9A-Fa-f]';
442          sp='[:]'
443          s_short=c+c+sp+c+c+'[.]'+c;
444          s_full=c+c+c+c+sp+s_short
445          re_full = re.compile(s_full)
446          re_short = re.compile(s_short)
447
448          if re_short.match(pci_name):
449              return '0000:'+pci_name
450
451          if re_full.match(pci_name):
452              return pci_name
453
454          err=" %s is not a valid pci address \n" %pci_name;
455          raise DpdkSetup(err)
456
457
458    def run_dpdk_lspci (self):
459        dpdk_nic_bind.get_nic_details()
460        self.m_devices= dpdk_nic_bind.devices
461
462    def do_run (self,only_check_all_mlx=False):
463        self.run_dpdk_lspci ()
464        if map_driver.dump_interfaces is None or (map_driver.dump_interfaces == [] and map_driver.parent_cfg):
465            self.load_config_file()
466            if_list=self.m_cfg_dict[0]['interfaces']
467        else:
468            if_list = map_driver.dump_interfaces
469            if not if_list:
470                for dev in self.m_devices.values():
471                    if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
472                        if_list.append(dev['Slot'])
473
474        if_list = list(map(self.pci_name_to_full_name, if_list))
475
476
477        # check how many mellanox cards we have
478        Mellanox_cnt=0;
479        for key in if_list:
480            if key not in self.m_devices:
481                err=" %s does not exist " %key;
482                raise DpdkSetup(err)
483
484            if 'Vendor_str' not in self.m_devices[key]:
485                err=" %s does not have Vendor_str " %key;
486                raise DpdkSetup(err)
487
488            if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
489                Mellanox_cnt=Mellanox_cnt+1
490
491
492        if not map_driver.dump_interfaces :
493            if  ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
494               err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
495               raise DpdkSetup(err)
496
497
498        if not map_driver.dump_interfaces :
499            if  Mellanox_cnt>0 :
500                self.set_only_mellanox_nics()
501
502        if self.get_only_mellanox_nics():
503            if not map_driver.no_ofed_check:
504                self.verify_ofed_os()
505                self.check_ofed_version()
506
507            for key in if_list:
508                pci_id=self.m_devices[key]['Slot_str']
509                self.tune_mlx5_device (pci_id)
510                if 'Interface' in self.m_devices[key]:
511                    dev_id=self.m_devices[key]['Interface']
512                    self.disable_flow_control_mlx5_device (dev_id)
513                    self.set_max_mtu_mlx5_device(dev_id)
514
515
516        if only_check_all_mlx:
517            if Mellanox_cnt >0:
518                exit(1);
519            else:
520                exit(0);
521
522        for key in if_list:
523            if key not in self.m_devices:
524                err=" %s does not exist " %key;
525                raise DpdkSetup(err)
526
527            if 'Driver_str' in self.m_devices[key]:
528                if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) :
529                    self.do_bind_one (key,(Mellanox_cnt>0))
530                    pass;
531            else:
532                self.do_bind_one (key,(Mellanox_cnt>0))
533                pass;
534
535        if (Mellanox_cnt==0):
536            # We are not in Mellanox case, we can do this check only in case of Intel (another process is running)
537            if if_list and map_driver.args.parent and (dpdk_nic_bind.get_igb_uio_usage()):
538                pid = dpdk_nic_bind.get_pid_using_pci(if_list)
539                if pid:
540                    cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
541                    print('Some or all of given interfaces are in use by following process:\npid: %s, cmd: %s' % (pid, cmdline))
542                    if not dpdk_nic_bind.confirm('Ignore and proceed (y/N):'):
543                        sys.exit(1)
544                else:
545                       print('WARNING: Some other program is using DPDK driver.\nIf it is TRex and you did not configure it for dual run, current command will fail.')
546
547    def do_return_to_linux(self):
548        if not self.m_devices:
549            self.run_dpdk_lspci()
550        dpdk_interfaces = []
551        for device in self.m_devices.values():
552            if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
553                dpdk_interfaces.append(device['Slot'])
554        if not dpdk_interfaces:
555            print('No DPDK bound interfaces.')
556            return
557        if dpdk_nic_bind.get_igb_uio_usage():
558            pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
559            if pid:
560                cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
561                print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
562                if not dpdk_nic_bind.confirm('Confirm (y/N):'):
563                    return
564        drivers_table = {
565            'rte_ixgbe_pmd': 'ixgbe',
566            'rte_igb_pmd': 'igb',
567            'rte_i40e_pmd': 'i40e',
568            'rte_em_pmd': 'e1000',
569            'rte_vmxnet3_pmd': 'vmxnet3',
570            'rte_virtio_pmd': 'virtio-pci',
571            'rte_enic_pmd': 'enic',
572        }
573        for pci, info in dpdk_nic_bind.get_info_from_trex(dpdk_interfaces).items():
574            if pci not in self.m_devices:
575                raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
576            dev = self.m_devices[pci]
577            if info['TRex_Driver'] not in drivers_table:
578                print('Got unknown driver %s, description: %s' % (info['TRex_Driver'], dev['Device_str']))
579            else:
580                print('Returning to Linux %s' % pci)
581                dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
582
583    def _get_cpu_topology(self):
584        cpu_topology_file = '/proc/cpuinfo'
585        # physical processor -> physical core -> logical processing units (threads)
586        cpu_topology = OrderedDict()
587        if not os.path.exists(cpu_topology_file):
588            raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file)
589        with open(cpu_topology_file) as f:
590            for lcore in f.read().split('\n\n'):
591                if not lcore:
592                    continue
593                lcore_dict = OrderedDict()
594                for line in lcore.split('\n'):
595                    key, val = line.split(':', 1)
596                    lcore_dict[key.strip()] = val.strip()
597                if 'processor' not in lcore_dict:
598                    continue
599                numa = int(lcore_dict.get('physical id', -1))
600                if numa not in cpu_topology:
601                    cpu_topology[numa] = OrderedDict()
602                core = int(lcore_dict.get('core id', lcore_dict['processor']))
603                if core not in cpu_topology[numa]:
604                    cpu_topology[numa][core] = []
605                cpu_topology[numa][core].append(int(lcore_dict['processor']))
606        if not cpu_topology:
607            raise DpdkSetup('Could not determine CPU topology from %s' % cpu_topology_file)
608        return cpu_topology
609
610    # input: list of different descriptions of interfaces: index, pci, name etc.
611    # Binds to dpdk wanted interfaces, not bound to any driver.
612    # output: list of maps of devices in dpdk_* format (self.m_devices.values())
613    def _get_wanted_interfaces(self, input_interfaces, get_macs = True):
614        if type(input_interfaces) is not list:
615            raise DpdkSetup('type of input interfaces should be list')
616        if not len(input_interfaces):
617            raise DpdkSetup('Please specify interfaces to use in the config')
618        if len(input_interfaces) % 2:
619            raise DpdkSetup('Please specify even number of interfaces')
620        wanted_interfaces = []
621        sorted_pci = sorted(self.m_devices.keys())
622        for interface in input_interfaces:
623            dev = None
624            try:
625                interface = int(interface)
626                if interface < 0 or interface >= len(sorted_pci):
627                    raise DpdkSetup('Index of an interfaces should be in range 0:%s' % (len(sorted_pci) - 1))
628                dev = self.m_devices[sorted_pci[interface]]
629            except ValueError:
630                for d in self.m_devices.values():
631                    if interface in (d['Interface'], d['Slot'], d['Slot_str']):
632                        dev = d
633                        break
634            if not dev:
635                raise DpdkSetup('Could not find information about this interface: %s' % interface)
636            if dev in wanted_interfaces:
637                raise DpdkSetup('Interface %s is specified twice' % interface)
638            dev['Interface_argv'] = interface
639            wanted_interfaces.append(dev)
640
641        if get_macs:
642            unbound = []
643            dpdk_bound = []
644            for interface in wanted_interfaces:
645                if 'Driver_str' not in interface:
646                    unbound.append(interface['Slot'])
647                elif interface.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
648                    dpdk_bound.append(interface['Slot'])
649            if unbound or dpdk_bound:
650                for pci, info in dpdk_nic_bind.get_info_from_trex(unbound + dpdk_bound).items():
651                    if pci not in self.m_devices:
652                        raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
653                    self.m_devices[pci].update(info)
654
655        return wanted_interfaces
656
657    def do_create(self):
658
659        ips = map_driver.args.ips
660        def_gws = map_driver.args.def_gws
661        dest_macs = map_driver.args.dest_macs
662        if map_driver.args.force_macs:
663            ip_config = False
664            if ips:
665                raise DpdkSetup("If using --force-macs, should not specify ips")
666            if def_gws:
667                raise DpdkSetup("If using --force-macs, should not specify default gateways")
668        elif ips:
669            ip_config = True
670            if not def_gws:
671                raise DpdkSetup("If specifying ips, must specify also def-gws")
672            if dest_macs:
673                raise DpdkSetup("If specifying ips, should not specify dest--macs")
674            if len(ips) != len(def_gws) or len(ips) != len(map_driver.args.create_interfaces):
675                raise DpdkSetup("Number of given IPs should equal number of given def-gws and number of interfaces")
676        else:
677            if dest_macs:
678                ip_config = False
679            else:
680                ip_config = True
681
682        # gather info about NICS from dpdk_nic_bind.py
683        if not self.m_devices:
684            self.run_dpdk_lspci()
685        wanted_interfaces = self._get_wanted_interfaces(map_driver.args.create_interfaces, get_macs = not ip_config)
686
687        for i, interface in enumerate(wanted_interfaces):
688            dual_index = i + 1 - (i % 2) * 2
689            if ip_config:
690                if isinstance(ips, list) and len(ips) > i:
691                    interface['ip'] = ips[i]
692                else:
693                    interface['ip'] = ".".join(list(str(i+1))*4)
694                if isinstance(def_gws, list) and len(def_gws) > i:
695                    interface['def_gw'] = def_gws[i]
696                else:
697                    interface['def_gw'] = ".".join(list(str(dual_index+1))*4)
698            else:
699                dual_if = wanted_interfaces[dual_index]
700                if 'MAC' not in interface:
701                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
702                if 'MAC' not in dual_if:
703                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % dual_if['Interface_argv'])
704                interface['src_mac'] = interface['MAC']
705                if isinstance(dest_macs, list) and len(dest_macs) > i:
706                    interface['dest_mac'] = dest_macs[i]
707                else:
708                    interface['dest_mac'] = dual_if['MAC']
709                    interface['loopback_dest'] = True
710
711        config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
712                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa,
713                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
714        config.create_config(filename = map_driver.args.o, print_config = map_driver.args.dump)
715
716    def do_interactive_create(self):
717        ignore_numa = False
718        cpu_topology = self._get_cpu_topology()
719        total_lcores = sum([len(lcores) for cores in cpu_topology.values() for lcores in cores.values()])
720        if total_lcores < 1:
721            raise DpdkSetup('Script could not determine number of cores of the system, exiting.')
722        elif total_lcores < 2:
723            if dpdk_nic_bind.confirm("You only have 1 core and can't run TRex at all. Ignore and continue? (y/N): "):
724                ignore_numa = True
725            else:
726                sys.exit(1)
727        elif total_lcores < 3:
728            if dpdk_nic_bind.confirm("You only have 2 cores and will be able to run only stateful without latency checks.\nIgnore and continue? (y/N): "):
729                ignore_numa = True
730            else:
731                sys.exit(1)
732
733        if map_driver.args.force_macs:
734            ip_based = False
735        elif dpdk_nic_bind.confirm("By default, IP based configuration file will be created. Do you want to use MAC based config? (y/N)"):
736            ip_based = False
737        else:
738            ip_based = True
739            ip_addr_digit = 1
740
741        if not self.m_devices:
742            self.run_dpdk_lspci()
743        dpdk_nic_bind.show_table(get_macs = not ip_based)
744        print('Please choose even number of interfaces from the list above, either by ID , PCI or Linux IF')
745        print('Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.')
746        print('Stateless can be in any order.')
747        numa = None
748        for dev in self.m_devices.values():
749            if numa is None:
750                numa = dev['NUMA']
751            elif numa != dev['NUMA']:
752                print('For performance, try to choose each pair of interfaces to be on the same NUMA.')
753                break
754        while True:
755            try:
756                input = dpdk_nic_bind.read_line('Enter list of interfaces separated by space (for example: 1 3) : ')
757                create_interfaces = input.replace(',', ' ').replace(';', ' ').split()
758                wanted_interfaces = self._get_wanted_interfaces(create_interfaces)
759                ConfigCreator._verify_devices_same_type(wanted_interfaces)
760            except Exception as e:
761                print(e)
762                continue
763            break
764        print('')
765
766        for interface in wanted_interfaces:
767            if interface['Active']:
768                print('Interface %s is active. Using it by TRex might close ssh connections etc.' % interface['Interface_argv'])
769                if not dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
770                    sys.exit(1)
771
772        for i, interface in enumerate(wanted_interfaces):
773            if not ip_based:
774                if 'MAC' not in interface:
775                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
776                interface['src_mac'] = interface['MAC']
777            dual_index = i + 1 - (i % 2) * 2
778            dual_int = wanted_interfaces[dual_index]
779            if not ignore_numa and interface['NUMA'] != dual_int['NUMA']:
780                print('NUMA is different at pair of interfaces: %s and %s. It will reduce performance.' % (interface['Interface_argv'], dual_int['Interface_argv']))
781                if dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
782                    ignore_numa = True
783                    print('')
784                else:
785                    return
786
787            if ip_based:
788                if ip_addr_digit % 2 == 0:
789                    dual_ip_digit = ip_addr_digit - 1
790                else:
791                    dual_ip_digit = ip_addr_digit + 1
792                ip = ".".join(list(str(ip_addr_digit))*4)
793                def_gw= ".".join(list(str(dual_ip_digit))*4)
794                ip_addr_digit += 1
795
796                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
797                if dpdk_nic_bind.confirm("Putting IP %s, default gw %s Change it?(y/N)." % (ip, def_gw)):
798                    while True:
799                        ip = dpdk_nic_bind.read_line('Please enter IP address for interface %s: ' % interface['Interface_argv'])
800                        if not ConfigCreator._verify_ip(ip):
801                            print ("Bad IP address format")
802                        else:
803                            break
804                    while True:
805                        def_gw = dpdk_nic_bind.read_line('Please enter default gateway for interface %s: ' % interface['Interface_argv'])
806                        if not ConfigCreator._verify_ip(def_gw):
807                            print ("Bad IP address format")
808                        else:
809                            break
810                wanted_interfaces[i]['ip'] = ip
811                wanted_interfaces[i]['def_gw'] = def_gw
812            else:
813                dest_mac = dual_int['MAC']
814                loopback_dest = True
815                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
816                if dpdk_nic_bind.confirm("Destination MAC is %s. Change it to MAC of DUT? (y/N)." % dest_mac):
817                    while True:
818                        input_mac = dpdk_nic_bind.read_line('Please enter new destination MAC of interface %s: ' % interface['Interface_argv'])
819                        try:
820                            if input_mac:
821                                ConfigCreator._convert_mac(input_mac) # verify format
822                                dest_mac = input_mac
823                                loopback_dest = False
824                            else:
825                                print('Leaving the loopback MAC.')
826                        except Exception as e:
827                            print(e)
828                            continue
829                        break
830                wanted_interfaces[i]['dest_mac'] = dest_mac
831                wanted_interfaces[i]['loopback_dest'] = loopback_dest
832
833        config = ConfigCreator(cpu_topology, wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
834                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa or ignore_numa,
835                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
836        if dpdk_nic_bind.confirm('Print preview of generated config? (Y/n)', default = True):
837            config.create_config(print_config = True)
838        if dpdk_nic_bind.confirm('Save the config to file? (Y/n)', default = True):
839            print('Default filename is /etc/trex_cfg.yaml')
840            filename = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new file: ')
841            if not filename:
842                filename = '/etc/trex_cfg.yaml'
843            config.create_config(filename = filename)
844
845
846def parse_parent_cfg (parent_cfg):
847    parent_parser = argparse.ArgumentParser(add_help = False)
848    parent_parser.add_argument('-?', '-h', '--help', dest = 'help', action = 'store_true')
849    parent_parser.add_argument('--cfg', default='')
850    parent_parser.add_argument('--dump-interfaces', nargs='*', default=None)
851    parent_parser.add_argument('--no-ofed-check', action = 'store_true')
852    parent_parser.add_argument('--no-watchdog', action = 'store_true')
853    args, _ = parent_parser.parse_known_args(shlex.split(parent_cfg))
854    if args.help:
855        sys.exit(0)
856    return (args.cfg, args.dump_interfaces, args.no_ofed_check)
857
858def process_options ():
859    parser = argparse.ArgumentParser(usage="""
860
861Examples:
862---------
863
864To return to Linux the DPDK bound interfaces (for ifconfig etc.)
865  sudo ./dpdk_set_ports.py -l
866
867To create TRex config file using interactive mode
868  sudo ./dpdk_set_ports.py -i
869
870To create a default config file (example1)
871  sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
872
873To create a default config file (example2)
874  sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
875
876To show interfaces status
877  sudo ./dpdk_set_ports.py -s
878
879To see more detailed info on interfaces (table):
880  sudo ./dpdk_set_ports.py -t
881
882    """,
883    description=" unbind dpdk interfaces ",
884    epilog=" written by hhaim");
885
886    parser.add_argument("-l", "--linux", action='store_true',
887                      help=""" Return all DPDK interfaces to Linux driver """,
888     )
889
890    parser.add_argument("--cfg",
891                      help=""" configuration file name  """,
892     )
893
894    parser.add_argument("--parent",
895                      help=argparse.SUPPRESS
896     )
897
898    parser.add_argument('--dump-pci-description', help=argparse.SUPPRESS, dest='dump_pci_desc', action='store_true')
899
900    parser.add_argument("-i", "--interactive", action='store_true',
901                      help=""" Create TRex config in interactive mode """,
902     )
903
904    parser.add_argument("-c", "--create", nargs='*', default=None, dest='create_interfaces', metavar='<interface>',
905                      help="""Try to create a configuration file by specifying needed interfaces by PCI address or Linux names: eth1 etc.""",
906     )
907
908    parser.add_argument("--ci", "--cores-include", nargs='*', default=[], dest='create_include', metavar='<cores>',
909                      help="""White list of cores to use. Make sure there is enough for each NUMA.""",
910     )
911
912    parser.add_argument("--ce", "--cores-exclude", nargs='*', default=[], dest='create_exclude', metavar='<cores>',
913                      help="""Black list of cores to exclude. Make sure there will be enough for each NUMA.""",
914     )
915
916    parser.add_argument("--dump", default=False, action='store_true',
917                      help="""Dump created config to screen.""",
918     )
919
920    parser.add_argument("--no-ht", default=False, dest='no_ht', action='store_true',
921                      help="""Use only one thread of each Core in created config yaml (No Hyper-Threading).""",
922     )
923
924    parser.add_argument("--dest-macs", nargs='*', default=[], action='store',
925                      help="""Destination MACs to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
926     )
927
928    parser.add_argument("--force-macs", default=False, action='store_true',
929                      help="""Use MACs in created config file.""",
930     )
931
932    parser.add_argument("--ips", nargs='*', default=[], action='store',
933                      help="""IP addresses to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
934     )
935
936    parser.add_argument("--def-gws", nargs='*', default=[], action='store',
937                      help="""Default gateways to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
938     )
939
940    parser.add_argument("-o", default=None, action='store', metavar='PATH',
941                      help="""Output the config to this file.""",
942     )
943
944    parser.add_argument("--prefix", default=None, action='store',
945                      help="""Advanced option: prefix to be used in TRex config in case of parallel instances.""",
946     )
947
948    parser.add_argument("--zmq-pub-port", default=None, action='store',
949                      help="""Advanced option: ZMQ Publisher port to be used in TRex config in case of parallel instances.""",
950     )
951
952    parser.add_argument("--zmq-rpc-port", default=None, action='store',
953                      help="""Advanced option: ZMQ RPC port to be used in TRex config in case of parallel instances.""",
954     )
955
956    parser.add_argument("--ignore-numa", default=False, action='store_true',
957                      help="""Advanced option: Ignore NUMAs for config creation. Use this option only if you have to, as it will reduce performance.""",
958     )
959
960    parser.add_argument("-s", "--show", action='store_true',
961                      help=""" show the status """,
962     )
963
964    parser.add_argument("-t", "--table", action='store_true',
965                      help=""" show table with NICs info """,
966     )
967
968    parser.add_argument('--version', action='version',
969                        version="0.2" )
970
971    map_driver.args = parser.parse_args();
972    if map_driver.args.parent :
973        map_driver.parent_cfg, map_driver.dump_interfaces, map_driver.no_ofed_check = parse_parent_cfg (map_driver.args.parent)
974        if map_driver.parent_cfg != '':
975            map_driver.cfg_file = map_driver.parent_cfg;
976    if  map_driver.args.cfg :
977        map_driver.cfg_file = map_driver.args.cfg;
978
979def main ():
980    try:
981        if getpass.getuser() != 'root':
982            raise DpdkSetup('Please run this program as root/with sudo')
983
984        process_options ()
985
986        if map_driver.args.show:
987            dpdk_nic_bind.show_status()
988            return
989
990        if map_driver.args.table:
991            dpdk_nic_bind.show_table()
992            return
993
994        if map_driver.args.dump_pci_desc:
995            dpdk_nic_bind.dump_pci_description()
996            return
997
998        obj =CIfMap(map_driver.cfg_file);
999
1000        if map_driver.args.create_interfaces is not None:
1001            obj.do_create();
1002        elif map_driver.args.interactive:
1003            obj.do_interactive_create();
1004        elif map_driver.args.linux:
1005            obj.do_return_to_linux();
1006        else:
1007            obj.do_run();
1008        print('')
1009    except DpdkSetup as e:
1010        print(e)
1011        exit(-1)
1012
1013
1014
1015if __name__ == '__main__':
1016    main()
1017