dpdk_setup_ports.py revision 0d83d222
1#! /usr/bin/python
2# hhaim
3import sys
4import os
5python_ver = 'python%s' % sys.version_info[0]
6yaml_path = os.path.join('external_libs', 'pyyaml-3.11', python_ver)
7if yaml_path not in sys.path:
8    sys.path.append(yaml_path)
9import yaml
10import dpdk_nic_bind
11import re
12import argparse
13import copy
14import shlex
15import traceback
16from collections import defaultdict, OrderedDict
17from distutils.util import strtobool
18import subprocess
19import platform
20
21class ConfigCreator(object):
22    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
23    _2hex_re = '[\da-fA-F]{2}'
24    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
25
26    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
27    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
28    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
29        self.cpu_topology = copy.deepcopy(cpu_topology)
30        self.interfaces   = copy.deepcopy(interfaces)
31        del cpu_topology
32        del interfaces
33        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
34        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
35        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
36        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
37        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
38        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
39        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
40        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
41        if only_first_thread:
42            for cores in self.cpu_topology.values():
43                for core in cores.keys():
44                    cores[core] = cores[core][:1]
45        include_lcores = [int(x) for x in include_lcores]
46        exclude_lcores = [int(x) for x in exclude_lcores]
47
48        self.has_zero_lcore = False
49        self.lcores_per_numa = {}
50        total_lcores = 0
51        for numa, cores in self.cpu_topology.items():
52            self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
53            for core, lcores in cores.items():
54                total_lcores += len(lcores)
55                for lcore in list(lcores):
56                    if include_lcores and lcore not in include_lcores:
57                        cores[core].remove(lcore)
58                    if exclude_lcores and lcore in exclude_lcores:
59                        cores[core].remove(lcore)
60                if 0 in lcores:
61                    self.has_zero_lcore = True
62                    lcores.remove(0)
63                    self.lcores_per_numa[numa]['siblings'].extend(lcores)
64                else:
65                    self.lcores_per_numa[numa]['main'].extend(lcores[:1])
66                    self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
67                self.lcores_per_numa[numa]['all'].extend(lcores)
68
69        for interface in self.interfaces:
70            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
71                if mandatory_interface_field not in interface:
72                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
73
74        Device_str = self._verify_devices_same_type(self.interfaces)
75        if '40Gb' in Device_str:
76            self.speed = 40
77        else:
78            self.speed = 10
79
80        minimum_required_lcores = len(self.interfaces) // 2 + 2
81        if total_lcores < minimum_required_lcores:
82            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
83                    (minimum_required_lcores, len(self.interfaces), total_lcores))
84        interfaces_per_numa = defaultdict(int)
85
86        for i in range(0, len(self.interfaces), 2):
87            numa = self.interfaces[i]['NUMA']
88            if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
89                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
90                        (numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
91            interfaces_per_numa[numa] += 2
92
93        self.interfaces_per_numa = interfaces_per_numa
94        self.prefix              = prefix
95        self.zmq_pub_port        = zmq_pub_port
96        self.zmq_rpc_port        = zmq_rpc_port
97        self.ignore_numa         = ignore_numa
98
99    @staticmethod
100    def verify_mac(mac_string):
101        if not ConfigCreator.mac_re.match(mac_string):
102            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
103        return mac_string.lower()
104
105    @staticmethod
106    def _exit_if_bad_ip(ip):
107        if not ConfigCreator._verify_ip(ip):
108            raise DpdkSetup("Got bad IP %s" % ip)
109
110    @staticmethod
111    def _verify_ip(ip):
112        a = ip.split('.')
113        if len(a) != 4:
114            return False
115        for x in a:
116            if not x.isdigit():
117                return False
118            i = int(x)
119            if i < 0 or i > 255:
120                return False
121        return True
122
123    @staticmethod
124    def _verify_devices_same_type(interfaces_list):
125        Device_str = interfaces_list[0]['Device_str']
126        for interface in interfaces_list:
127            if Device_str != interface['Device_str']:
128                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
129        return Device_str
130
131    def create_config(self, filename = None, print_config = False):
132        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
133        config_str += '- port_limit: %s\n' % len(self.interfaces)
134        config_str += '  version: 2\n'
135        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
136        if self.speed > 10:
137            config_str += '  port_bandwidth_gb: %s\n' % self.speed
138        if self.prefix:
139            config_str += '  prefix: %s\n' % self.prefix
140        if self.zmq_pub_port:
141            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
142        if self.zmq_rpc_port:
143            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
144        config_str += '  port_info:\n'
145        for index, interface in enumerate(self.interfaces):
146            if 'ip' in interface:
147                self._exit_if_bad_ip(interface['ip'])
148                self._exit_if_bad_ip(interface['def_gw'])
149                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
150                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
151            else:
152                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
153                if interface.get('loopback_dest'):
154                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
155                else:
156                    config_str += '\n'
157                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
158            if index % 2:
159                config_str += '\n' # dual if barrier
160
161        if not self.ignore_numa:
162            config_str += '  platform:\n'
163            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
164                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
165                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
166                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
167                lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
168                config_str += ' '*6 + 'dual_if:\n'
169                for i in range(0, len(self.interfaces), 2):
170                    lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
171                    lcores_pool = lcores_pool[lcores_per_dual_if:]
172                    if not lcores_for_this_dual_if:
173                        raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
174                    config_str += ' '*8 + '- socket: 0\n'
175                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
176            else:
177                # we will take common minimum among all NUMAs, to satisfy all
178                lcores_per_dual_if = 99
179                extra_lcores = 1 if self.has_zero_lcore else 2
180                # worst case 3 iterations, to ensure master and "rx" have cores left
181                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
182                    lcores_per_dual_if -= 1
183                    for numa, lcores_dict in self.lcores_per_numa.items():
184                        if not self.interfaces_per_numa[numa]:
185                            continue
186                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
187                lcores_pool = copy.deepcopy(self.lcores_per_numa)
188                # first, allocate lcores for dual_if section
189                dual_if_section = ' '*6 + 'dual_if:\n'
190                for i in range(0, len(self.interfaces), 2):
191                    numa = self.interfaces[i]['NUMA']
192                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
193                    lcores_for_this_dual_if  = lcores_pool[numa]['all'][:lcores_per_dual_if]
194                    lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
195                    for lcore in lcores_for_this_dual_if:
196                        if lcore in lcores_pool[numa]['main']:
197                            lcores_pool[numa]['main'].remove(lcore)
198                        elif lcore in lcores_pool[numa]['siblings']:
199                            lcores_pool[numa]['siblings'].remove(lcore)
200                        else:
201                            raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
202                    if not lcores_for_this_dual_if:
203                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
204                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))
205
206                # take the cores left to master and rx
207                mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
208                siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
209                if mains_left:
210                    rx_core = mains_left.pop(0)
211                else:
212                    rx_core = siblings_left.pop(0)
213                if self.has_zero_lcore:
214                    master_core = 0
215                elif mains_left:
216                    master_core = mains_left.pop(0)
217                else:
218                    master_core = siblings_left.pop(0)
219                config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
220                config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
221                # add the dual_if section
222                config_str += dual_if_section
223
224        # verify config is correct YAML format
225        try:
226            yaml.safe_load(config_str)
227        except Exception as e:
228            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
229
230        if print_config:
231            print(config_str)
232        if filename:
233            if os.path.exists(filename):
234                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
235                    print('Skipping.')
236                    return config_str
237            with open(filename, 'w') as f:
238                f.write(config_str)
239            print('Saved to %s.' % filename)
240        return config_str
241
242
243class map_driver(object):
244    args=None;
245    cfg_file='/etc/trex_cfg.yaml'
246    parent_args = None
247
248class DpdkSetup(Exception):
249    pass
250
251class CIfMap:
252
253    def __init__(self, cfg_file):
254        self.m_cfg_file =cfg_file;
255        self.m_cfg_dict={};
256        self.m_devices={};
257        self.m_is_mellanox_mode=False;
258
259    def dump_error (self,err):
260        s="""%s
261From this TRex version a configuration file must exist in /etc/ folder "
262The name of the configuration file should be /etc/trex_cfg.yaml "
263The minimum configuration file should include something like this
264- version       : 2 # version 2 of the configuration file
265  interfaces    : ["03:00.0","03:00.1","13:00.1","13:00.0"]  # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
266  port_limit      : 2 # number of ports to use valid is 2,4,6,8,10,12
267
268example of already bind devices
269
270$ ./dpdk_nic_bind.py --status
271
272Network devices using DPDK-compatible driver
273============================================
2740000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2750000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2760000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2770000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
278
279Network devices using kernel driver
280===================================
2810000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
282
283Other network devices
284=====================
285
286
287          """ % (err);
288        return s;
289
290
291    def raise_error  (self,err):
292        s= self.dump_error (err)
293        raise DpdkSetup(s)
294
295    def set_only_mellanox_nics(self):
296        self.m_is_mellanox_mode=True;
297
298    def get_only_mellanox_nics(self):
299        return self.m_is_mellanox_mode
300
301
302    def read_pci (self,pci_id,reg_id):
303        out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])
304        out=out.decode(errors='replace');
305        return (out.strip());
306
307    def write_pci (self,pci_id,reg_id,val):
308        out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])
309        out=out.decode(errors='replace');
310        return (out.strip());
311
312    def tune_mlx5_device (self,pci_id):
313        # set PCIe Read to 1024 and not 512 ... need to add it to startup s
314        val=self.read_pci (pci_id,68)
315        if val[0]!='3':
316            val='3'+val[1:]
317            self.write_pci (pci_id,68,val)
318            assert(self.read_pci (pci_id,68)==val);
319
320    def get_mtu_mlx5 (self,dev_id):
321        if len(dev_id)>0:
322            out=subprocess.check_output(['ifconfig', dev_id])
323            out=out.decode(errors='replace');
324            obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);
325            if obj:
326                return int(obj.group(1));
327            else:
328                obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);
329                if obj:
330                    return int(obj.group(1));
331                else:
332                    return -1
333
334    def set_mtu_mlx5 (self,dev_id,new_mtu):
335        if len(dev_id)>0:
336            out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])
337            out=out.decode(errors='replace');
338
339
340    def set_max_mtu_mlx5_device(self,dev_id):
341        mtu=9*1024+22
342        dev_mtu=self.get_mtu_mlx5 (dev_id);
343        if (dev_mtu>0) and (dev_mtu!=mtu):
344            self.set_mtu_mlx5(dev_id,mtu);
345            if self.get_mtu_mlx5(dev_id) != mtu:
346                print("Could not set MTU to %d" % mtu)
347                exit(-1);
348
349
350    def disable_flow_control_mlx5_device (self,dev_id):
351
352           if len(dev_id)>0:
353               my_stderr = open("/dev/null","wb")
354               cmd ='ethtool -A '+dev_id + ' rx off tx off '
355               subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)
356               my_stderr.close();
357
358    def check_ofed_version (self):
359        ofed_info='/usr/bin/ofed_info'
360
361        ofed_ver_re = re.compile('.*[-](\d)[.](\d)[-].*')
362
363        ofed_ver= 34
364        ofed_ver_show= '3.4-1'
365
366
367        if not os.path.isfile(ofed_info):
368            print("OFED %s is not installed on this setup" % ofed_info)
369            exit(-1);
370
371        try:
372          out = subprocess.check_output([ofed_info])
373        except Exception as e:
374            print("OFED %s can't run " % (ofed_info))
375            exit(-1);
376
377        lines=out.splitlines();
378
379        if len(lines)>1:
380            m= ofed_ver_re.match(str(lines[0]))
381            if m:
382                ver=int(m.group(1))*10+int(m.group(2))
383                if ver < ofed_ver:
384                  print("installed OFED version is '%s' should be at least '%s' and up" % (lines[0],ofed_ver_show))
385                  exit(-1);
386            else:
387                print("not found valid  OFED version '%s' " % (lines[0]))
388                exit(-1);
389
390
391    def verify_ofed_os(self):
392        err_msg = 'Warning: Mellanox NICs where tested only with RedHat/CentOS 7.2\n'
393        err_msg += 'Correct usage with other Linux distributions is not guaranteed.'
394        try:
395            dist = platform.dist()
396            if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.2'):
397                print(err_msg)
398        except Exception as e:
399            print('Error while determining OS type: %s' % e)
400
401    def load_config_file (self):
402
403        fcfg=self.m_cfg_file
404
405        if not os.path.isfile(fcfg) :
406            self.raise_error ("There is no valid configuration file %s\n" % fcfg)
407
408        try:
409          stream = open(fcfg, 'r')
410          self.m_cfg_dict= yaml.safe_load(stream)
411        except Exception as e:
412          print(e);
413          raise e
414
415        stream.close();
416        cfg_dict = self.m_cfg_dict[0]
417        if 'version' not in cfg_dict:
418            raise DpdkSetup("Configuration file %s is old, it should include version field\n" % fcfg )
419
420        if int(cfg_dict['version'])<2 :
421            raise DpdkSetup("Configuration file %s is old, expected version 2, got: %s\n" % (fcfg, cfg_dict['version']))
422
423        if 'interfaces' not in self.m_cfg_dict[0]:
424            raise DpdkSetup("Configuration file %s is old, it should include interfaces field with even number of elements" % fcfg)
425
426        if_list=self.m_cfg_dict[0]['interfaces']
427        l=len(if_list);
428        if l > 16:
429            raise DpdkSetup("Configuration file %s should include interfaces field with maximum 16 elements, got: %s." % (fcfg,l))
430        if l % 2:
431            raise DpdkSetup("Configuration file %s should include even number of interfaces " % (fcfg,l))
432        if 'port_limit' in cfg_dict and cfg_dict['port_limit'] > len(if_list):
433            raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
434
435
436    def do_bind_one (self,key,mellanox):
437        if mellanox:
438            drv="mlx5_core"
439        else:
440            drv="igb_uio"
441
442        cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
443        print(cmd)
444        res=os.system(cmd);
445        if res!=0:
446            raise DpdkSetup('')
447
448
449    def pci_name_to_full_name (self,pci_name):
450          c='[0-9A-Fa-f]';
451          sp='[:]'
452          s_short=c+c+sp+c+c+'[.]'+c;
453          s_full=c+c+c+c+sp+s_short
454          re_full = re.compile(s_full)
455          re_short = re.compile(s_short)
456
457          if re_short.match(pci_name):
458              return '0000:'+pci_name
459
460          if re_full.match(pci_name):
461              return pci_name
462
463          err=" %s is not a valid pci address \n" %pci_name;
464          raise DpdkSetup(err)
465
466
467    def run_dpdk_lspci (self):
468        dpdk_nic_bind.get_nic_details()
469        self.m_devices= dpdk_nic_bind.devices
470
471    def do_run (self,only_check_all_mlx=False):
472        """ return the number of mellanox drivers"""
473
474        self.run_dpdk_lspci ()
475        if (map_driver.parent_args.dump_interfaces is None or
476                    (map_driver.parent_args.dump_interfaces == [] and
477                            map_driver.parent_args.cfg)):
478            self.load_config_file()
479            if_list=self.m_cfg_dict[0]['interfaces']
480        else:
481            if_list = map_driver.parent_args.dump_interfaces
482            if not if_list:
483                for dev in self.m_devices.values():
484                    if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
485                        if_list.append(dev['Slot'])
486
487        if_list = list(map(self.pci_name_to_full_name, if_list))
488
489
490        # check how many mellanox cards we have
491        Mellanox_cnt=0;
492        for key in if_list:
493            if key not in self.m_devices:
494                err=" %s does not exist " %key;
495                raise DpdkSetup(err)
496
497            if 'Vendor_str' not in self.m_devices[key]:
498                err=" %s does not have Vendor_str " %key;
499                raise DpdkSetup(err)
500
501            if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
502                Mellanox_cnt=Mellanox_cnt+1
503
504
505        if not map_driver.parent_args.dump_interfaces:
506            if  ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
507               err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
508               raise DpdkSetup(err)
509
510
511        if not map_driver.parent_args.dump_interfaces:
512            if  Mellanox_cnt>0 :
513                self.set_only_mellanox_nics()
514
515        if self.get_only_mellanox_nics():
516            if not map_driver.parent_args.no_ofed_check:
517                self.verify_ofed_os()
518                self.check_ofed_version()
519
520            for key in if_list:
521                pci_id=self.m_devices[key]['Slot_str']
522                self.tune_mlx5_device (pci_id)
523                if 'Interface' in self.m_devices[key]:
524                    dev_id=self.m_devices[key]['Interface']
525                    self.disable_flow_control_mlx5_device (dev_id)
526                    self.set_max_mtu_mlx5_device(dev_id)
527
528
529        if only_check_all_mlx:
530            if Mellanox_cnt >0:
531                exit(1);
532            else:
533                exit(0);
534
535        for key in if_list:
536            if key not in self.m_devices:
537                err=" %s does not exist " %key;
538                raise DpdkSetup(err)
539
540            if 'Driver_str' in self.m_devices[key]:
541                if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) :
542                    self.do_bind_one (key,(Mellanox_cnt>0))
543                    pass;
544            else:
545                self.do_bind_one (key,(Mellanox_cnt>0))
546                pass;
547
548        if (Mellanox_cnt==0):
549            # We are not in Mellanox case, we can do this check only in case of Intel (another process is running)
550            if if_list and map_driver.args.parent and (dpdk_nic_bind.get_igb_uio_usage()):
551                pid = dpdk_nic_bind.get_pid_using_pci(if_list)
552                if pid:
553                    cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
554                    print('Some or all of given interfaces are in use by following process:\npid: %s, cmd: %s' % (pid, cmdline))
555                    if not dpdk_nic_bind.confirm('Ignore and proceed (y/N):'):
556                        sys.exit(1)
557                else:
558                       print('WARNING: Some other program is using DPDK driver.\nIf it is TRex and you did not configure it for dual run, current command will fail.')
559        if map_driver.parent_args.stl and not map_driver.parent_args.no_scapy_server:
560            try:
561                master_core = self.m_cfg_dict[0]['platform']['master_thread_id']
562            except:
563                master_core = 0
564            ret = os.system('%s scapy_daemon_server restart -c %s' % (sys.executable, master_core))
565            if ret:
566                print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
567                sys.exit(1)
568
569        if Mellanox_cnt:
570            return 1
571        else:
572            return 0
573
574
575
576    def do_return_to_linux(self):
577        if not self.m_devices:
578            self.run_dpdk_lspci()
579        dpdk_interfaces = []
580        for device in self.m_devices.values():
581            if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
582                dpdk_interfaces.append(device['Slot'])
583        if not dpdk_interfaces:
584            print('No DPDK bound interfaces.')
585            return
586        if dpdk_nic_bind.get_igb_uio_usage():
587            pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
588            if pid:
589                cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
590                print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
591                if not dpdk_nic_bind.confirm('Confirm (y/N):'):
592                    return
593        drivers_table = {
594            'net_ixgbe': 'ixgbe',
595            'net_ixgbe_vf': 'ixgbe_vf',
596            'net_e1000_igb': 'e1000_igb',
597            'net_i40e': 'i40e',
598            'net_i40e_vf': 'i40e_vf',
599            'net_e1000_em': 'e1000_em',
600            'net_vmxnet3': 'vmxnet3',
601            'net_virtio': 'virtio-pci',
602            'net_enic': 'enic',
603        }
604        for pci, info in dpdk_nic_bind.get_info_from_trex(dpdk_interfaces).items():
605            if pci not in self.m_devices:
606                raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
607            dev = self.m_devices[pci]
608            if info['TRex_Driver'] not in drivers_table:
609                print('Got unknown driver %s, description: %s' % (info['TRex_Driver'], dev['Device_str']))
610            else:
611                print('Returning to Linux %s' % pci)
612                dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
613
614    def _get_cpu_topology(self):
615        cpu_topology_file = '/proc/cpuinfo'
616        # physical processor -> physical core -> logical processing units (threads)
617        cpu_topology = OrderedDict()
618        if not os.path.exists(cpu_topology_file):
619            raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file)
620        with open(cpu_topology_file) as f:
621            for lcore in f.read().split('\n\n'):
622                if not lcore:
623                    continue
624                lcore_dict = OrderedDict()
625                for line in lcore.split('\n'):
626                    key, val = line.split(':', 1)
627                    lcore_dict[key.strip()] = val.strip()
628                if 'processor' not in lcore_dict:
629                    continue
630                numa = int(lcore_dict.get('physical id', -1))
631                if numa not in cpu_topology:
632                    cpu_topology[numa] = OrderedDict()
633                core = int(lcore_dict.get('core id', lcore_dict['processor']))
634                if core not in cpu_topology[numa]:
635                    cpu_topology[numa][core] = []
636                cpu_topology[numa][core].append(int(lcore_dict['processor']))
637        if not cpu_topology:
638            raise DpdkSetup('Could not determine CPU topology from %s' % cpu_topology_file)
639        return cpu_topology
640
641    # input: list of different descriptions of interfaces: index, pci, name etc.
642    # Binds to dpdk wanted interfaces, not bound to any driver.
643    # output: list of maps of devices in dpdk_* format (self.m_devices.values())
644    def _get_wanted_interfaces(self, input_interfaces, get_macs = True):
645        if type(input_interfaces) is not list:
646            raise DpdkSetup('type of input interfaces should be list')
647        if not len(input_interfaces):
648            raise DpdkSetup('Please specify interfaces to use in the config')
649        if len(input_interfaces) % 2:
650            raise DpdkSetup('Please specify even number of interfaces')
651        wanted_interfaces = []
652        sorted_pci = sorted(self.m_devices.keys())
653        for interface in input_interfaces:
654            dev = None
655            try:
656                interface = int(interface)
657                if interface < 0 or interface >= len(sorted_pci):
658                    raise DpdkSetup('Index of an interfaces should be in range 0:%s' % (len(sorted_pci) - 1))
659                dev = self.m_devices[sorted_pci[interface]]
660            except ValueError:
661                for d in self.m_devices.values():
662                    if interface in (d['Interface'], d['Slot'], d['Slot_str']):
663                        dev = d
664                        break
665            if not dev:
666                raise DpdkSetup('Could not find information about this interface: %s' % interface)
667            if dev in wanted_interfaces:
668                raise DpdkSetup('Interface %s is specified twice' % interface)
669            dev['Interface_argv'] = interface
670            wanted_interfaces.append(dev)
671
672        if get_macs:
673            unbound = []
674            dpdk_bound = []
675            for interface in wanted_interfaces:
676                if 'Driver_str' not in interface:
677                    unbound.append(interface['Slot'])
678                elif interface.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
679                    dpdk_bound.append(interface['Slot'])
680            if unbound or dpdk_bound:
681                for pci, info in dpdk_nic_bind.get_info_from_trex(unbound + dpdk_bound).items():
682                    if pci not in self.m_devices:
683                        raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
684                    self.m_devices[pci].update(info)
685
686        return wanted_interfaces
687
688    def do_create(self):
689
690        ips = map_driver.args.ips
691        def_gws = map_driver.args.def_gws
692        dest_macs = map_driver.args.dest_macs
693        if map_driver.args.force_macs:
694            ip_config = False
695            if ips:
696                raise DpdkSetup("If using --force-macs, should not specify ips")
697            if def_gws:
698                raise DpdkSetup("If using --force-macs, should not specify default gateways")
699        elif ips:
700            ip_config = True
701            if not def_gws:
702                raise DpdkSetup("If specifying ips, must specify also def-gws")
703            if dest_macs:
704                raise DpdkSetup("If specifying ips, should not specify dest--macs")
705            if len(ips) != len(def_gws) or len(ips) != len(map_driver.args.create_interfaces):
706                raise DpdkSetup("Number of given IPs should equal number of given def-gws and number of interfaces")
707        else:
708            if dest_macs:
709                ip_config = False
710            else:
711                ip_config = True
712
713        # gather info about NICS from dpdk_nic_bind.py
714        if not self.m_devices:
715            self.run_dpdk_lspci()
716        wanted_interfaces = self._get_wanted_interfaces(map_driver.args.create_interfaces, get_macs = not ip_config)
717
718        for i, interface in enumerate(wanted_interfaces):
719            dual_index = i + 1 - (i % 2) * 2
720            if ip_config:
721                if isinstance(ips, list) and len(ips) > i:
722                    interface['ip'] = ips[i]
723                else:
724                    interface['ip'] = '.'.join([str(i+1) for _ in range(4)])
725                if isinstance(def_gws, list) and len(def_gws) > i:
726                    interface['def_gw'] = def_gws[i]
727                else:
728                    interface['def_gw'] = '.'.join([str(dual_index+1) for _ in range(4)])
729            else:
730                dual_if = wanted_interfaces[dual_index]
731                if 'MAC' not in interface:
732                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
733                if 'MAC' not in dual_if:
734                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % dual_if['Interface_argv'])
735                interface['src_mac'] = interface['MAC']
736                if isinstance(dest_macs, list) and len(dest_macs) > i:
737                    interface['dest_mac'] = dest_macs[i]
738                else:
739                    interface['dest_mac'] = dual_if['MAC']
740                    interface['loopback_dest'] = True
741
742        config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
743                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa,
744                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
745        if map_driver.args.output_config:
746            config.create_config(filename = map_driver.args.output_config)
747        else:
748            print('### Dumping config to screen, use -o flag to save to file')
749            config.create_config(print_config = True)
750
751    def do_interactive_create(self):
752        ignore_numa = False
753        cpu_topology = self._get_cpu_topology()
754        total_lcores = sum([len(lcores) for cores in cpu_topology.values() for lcores in cores.values()])
755        if total_lcores < 1:
756            raise DpdkSetup('Script could not determine number of cores of the system, exiting.')
757        elif total_lcores < 2:
758            if dpdk_nic_bind.confirm("You only have 1 core and can't run TRex at all. Ignore and continue? (y/N): "):
759                ignore_numa = True
760            else:
761                sys.exit(1)
762        elif total_lcores < 3:
763            if dpdk_nic_bind.confirm("You only have 2 cores and will be able to run only stateful without latency checks.\nIgnore and continue? (y/N): "):
764                ignore_numa = True
765            else:
766                sys.exit(1)
767
768        if map_driver.args.force_macs:
769            ip_based = False
770        elif dpdk_nic_bind.confirm("By default, IP based configuration file will be created. Do you want to use MAC based config? (y/N)"):
771            ip_based = False
772        else:
773            ip_based = True
774            ip_addr_digit = 1
775
776        if not self.m_devices:
777            self.run_dpdk_lspci()
778        dpdk_nic_bind.show_table(get_macs = not ip_based)
779        print('Please choose even number of interfaces from the list above, either by ID , PCI or Linux IF')
780        print('Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.')
781        print('Stateless can be in any order.')
782        numa = None
783        for dev in self.m_devices.values():
784            if numa is None:
785                numa = dev['NUMA']
786            elif numa != dev['NUMA']:
787                print('For performance, try to choose each pair of interfaces to be on the same NUMA.')
788                break
789        while True:
790            try:
791                input = dpdk_nic_bind.read_line('Enter list of interfaces separated by space (for example: 1 3) : ')
792                create_interfaces = input.replace(',', ' ').replace(';', ' ').split()
793                wanted_interfaces = self._get_wanted_interfaces(create_interfaces)
794                ConfigCreator._verify_devices_same_type(wanted_interfaces)
795            except Exception as e:
796                print(e)
797                continue
798            break
799        print('')
800
801        for interface in wanted_interfaces:
802            if interface['Active']:
803                print('Interface %s is active. Using it by TRex might close ssh connections etc.' % interface['Interface_argv'])
804                if not dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
805                    sys.exit(1)
806
807        for i, interface in enumerate(wanted_interfaces):
808            if not ip_based:
809                if 'MAC' not in interface:
810                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
811                interface['src_mac'] = interface['MAC']
812            dual_index = i + 1 - (i % 2) * 2
813            dual_int = wanted_interfaces[dual_index]
814            if not ignore_numa and interface['NUMA'] != dual_int['NUMA']:
815                print('NUMA is different at pair of interfaces: %s and %s. It will reduce performance.' % (interface['Interface_argv'], dual_int['Interface_argv']))
816                if dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
817                    ignore_numa = True
818                    print('')
819                else:
820                    return
821
822            if ip_based:
823                if ip_addr_digit % 2 == 0:
824                    dual_ip_digit = ip_addr_digit - 1
825                else:
826                    dual_ip_digit = ip_addr_digit + 1
827                ip = '.'.join([str(ip_addr_digit) for _ in range(4)])
828                def_gw= '.'.join([str(dual_ip_digit) for _ in range(4)])
829                ip_addr_digit += 1
830
831                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
832                if dpdk_nic_bind.confirm("Putting IP %s, default gw %s Change it?(y/N)." % (ip, def_gw)):
833                    while True:
834                        ip = dpdk_nic_bind.read_line('Please enter IP address for interface %s: ' % interface['Interface_argv'])
835                        if not ConfigCreator._verify_ip(ip):
836                            print ("Bad IP address format")
837                        else:
838                            break
839                    while True:
840                        def_gw = dpdk_nic_bind.read_line('Please enter default gateway for interface %s: ' % interface['Interface_argv'])
841                        if not ConfigCreator._verify_ip(def_gw):
842                            print ("Bad IP address format")
843                        else:
844                            break
845                wanted_interfaces[i]['ip'] = ip
846                wanted_interfaces[i]['def_gw'] = def_gw
847            else:
848                dest_mac = dual_int['MAC']
849                loopback_dest = True
850                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
851                if dpdk_nic_bind.confirm("Destination MAC is %s. Change it to MAC of DUT? (y/N)." % dest_mac):
852                    while True:
853                        input_mac = dpdk_nic_bind.read_line('Please enter new destination MAC of interface %s: ' % interface['Interface_argv'])
854                        try:
855                            if input_mac:
856                                ConfigCreator.verify_mac(input_mac) # verify format
857                                dest_mac = input_mac
858                                loopback_dest = False
859                            else:
860                                print('Leaving the loopback MAC.')
861                        except Exception as e:
862                            print(e)
863                            continue
864                        break
865                wanted_interfaces[i]['dest_mac'] = dest_mac
866                wanted_interfaces[i]['loopback_dest'] = loopback_dest
867
868        config = ConfigCreator(cpu_topology, wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
869                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa or ignore_numa,
870                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
871        if dpdk_nic_bind.confirm('Print preview of generated config? (Y/n)', default = True):
872            config.create_config(print_config = True)
873        if dpdk_nic_bind.confirm('Save the config to file? (Y/n)', default = True):
874            print('Default filename is /etc/trex_cfg.yaml')
875            filename = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new file: ')
876            if not filename:
877                filename = '/etc/trex_cfg.yaml'
878            config.create_config(filename = filename)
879
880
881def parse_parent_cfg (parent_cfg):
882    parent_parser = argparse.ArgumentParser(add_help = False)
883    parent_parser.add_argument('-?', '-h', '--help', dest = 'help', action = 'store_true')
884    parent_parser.add_argument('--cfg', default='')
885    parent_parser.add_argument('--dump-interfaces', nargs='*', default=None)
886    parent_parser.add_argument('--no-ofed-check', action = 'store_true')
887    parent_parser.add_argument('--no-scapy-server', action = 'store_true')
888    parent_parser.add_argument('--no-watchdog', action = 'store_true')
889    parent_parser.add_argument('-i', action = 'store_true', dest = 'stl', default = False)
890    map_driver.parent_args, _ = parent_parser.parse_known_args(shlex.split(parent_cfg))
891    if map_driver.parent_args.help:
892        sys.exit(0)
893
894
895def process_options ():
896    parser = argparse.ArgumentParser(usage="""
897
898Examples:
899---------
900
901To return to Linux the DPDK bound interfaces (for ifconfig etc.)
902  sudo ./dpdk_set_ports.py -l
903
904To create TRex config file using interactive mode
905  sudo ./dpdk_set_ports.py -i
906
907To create a default config file (example1)
908  sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
909
910To create a default config file (example2)
911  sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
912
913To show interfaces status
914  sudo ./dpdk_set_ports.py -s
915
916To see more detailed info on interfaces (table):
917  sudo ./dpdk_set_ports.py -t
918
919    """,
920    description=" unbind dpdk interfaces ",
921    epilog=" written by hhaim");
922
923    parser.add_argument("-l", "--linux", action='store_true',
924                      help=""" Return all DPDK interfaces to Linux driver """,
925     )
926
927    parser.add_argument("--cfg",
928                      help=""" configuration file name  """,
929     )
930
931    parser.add_argument("--parent",
932                      help=argparse.SUPPRESS
933     )
934
935    parser.add_argument('--dump-pci-description', help=argparse.SUPPRESS, dest='dump_pci_desc', action='store_true')
936
937    parser.add_argument("-i", "--interactive", action='store_true',
938                      help=""" Create TRex config in interactive mode """,
939     )
940
941    parser.add_argument("-c", "--create", nargs='*', default=None, dest='create_interfaces', metavar='<interface>',
942                      help="""Try to create a configuration file by specifying needed interfaces by PCI address or Linux names: eth1 etc.""",
943     )
944
945    parser.add_argument("--ci", "--cores-include", nargs='*', default=[], dest='create_include', metavar='<cores>',
946                      help="""White list of cores to use. Make sure there is enough for each NUMA.""",
947     )
948
949    parser.add_argument("--ce", "--cores-exclude", nargs='*', default=[], dest='create_exclude', metavar='<cores>',
950                      help="""Black list of cores to exclude. Make sure there will be enough for each NUMA.""",
951     )
952
953    parser.add_argument("--no-ht", default=False, dest='no_ht', action='store_true',
954                      help="""Use only one thread of each Core in created config yaml (No Hyper-Threading).""",
955     )
956
957    parser.add_argument("--dest-macs", nargs='*', default=[], action='store',
958                      help="""Destination MACs to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
959     )
960
961    parser.add_argument("--force-macs", default=False, action='store_true',
962                      help="""Use MACs in created config file.""",
963     )
964
965    parser.add_argument("--ips", nargs='*', default=[], action='store',
966                      help="""IP addresses to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
967     )
968
969    parser.add_argument("--def-gws", nargs='*', default=[], action='store',
970                      help="""Default gateways to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
971     )
972
973    parser.add_argument("-o", default=None, action='store', metavar='PATH', dest = 'output_config',
974                      help="""Output the config to this file.""",
975     )
976
977    parser.add_argument("--prefix", default=None, action='store',
978                      help="""Advanced option: prefix to be used in TRex config in case of parallel instances.""",
979     )
980
981    parser.add_argument("--zmq-pub-port", default=None, action='store',
982                      help="""Advanced option: ZMQ Publisher port to be used in TRex config in case of parallel instances.""",
983     )
984
985    parser.add_argument("--zmq-rpc-port", default=None, action='store',
986                      help="""Advanced option: ZMQ RPC port to be used in TRex config in case of parallel instances.""",
987     )
988
989    parser.add_argument("--ignore-numa", default=False, action='store_true',
990                      help="""Advanced option: Ignore NUMAs for config creation. Use this option only if you have to, as it will reduce performance.""",
991     )
992
993    parser.add_argument("-s", "--show", action='store_true',
994                      help=""" show the status """,
995     )
996
997    parser.add_argument("-t", "--table", action='store_true',
998                      help=""" show table with NICs info """,
999     )
1000
1001    parser.add_argument('--version', action='version',
1002                        version="0.2" )
1003
1004    map_driver.args = parser.parse_args();
1005    if map_driver.args.parent :
1006        parse_parent_cfg (map_driver.args.parent)
1007        if map_driver.parent_args.cfg:
1008            map_driver.cfg_file = map_driver.parent_args.cfg;
1009    if  map_driver.args.cfg :
1010        map_driver.cfg_file = map_driver.args.cfg;
1011
1012def main ():
1013    try:
1014        if os.getuid() != 0:
1015            raise DpdkSetup('Please run this program as root/with sudo')
1016
1017        process_options ()
1018
1019        if map_driver.args.show:
1020            dpdk_nic_bind.show_status()
1021            return
1022
1023        if map_driver.args.table:
1024            dpdk_nic_bind.show_table()
1025            return
1026
1027        if map_driver.args.dump_pci_desc:
1028            dpdk_nic_bind.dump_pci_description()
1029            return
1030
1031        obj =CIfMap(map_driver.cfg_file);
1032
1033        if map_driver.args.create_interfaces is not None:
1034            obj.do_create();
1035        elif map_driver.args.interactive:
1036            obj.do_interactive_create();
1037        elif map_driver.args.linux:
1038            obj.do_return_to_linux();
1039        else:
1040            exit(obj.do_run());
1041        print('')
1042    except DpdkSetup as e:
1043        print(e)
1044        exit(-1)
1045
1046
1047
1048if __name__ == '__main__':
1049    main()
1050