dpdk_setup_ports.py revision 9fb4cf6f
1#! /usr/bin/python
2# hhaim
3import sys
4import os
5python_ver = 'python%s' % sys.version_info[0]
6sys.path.append(os.path.join('external_libs', 'pyyaml-3.11', python_ver))
7import yaml
8import dpdk_nic_bind
9import re
10import argparse
11import copy
12import shlex
13import traceback
14from collections import defaultdict, OrderedDict
15from distutils.util import strtobool
16import subprocess
17import platform
18
19class ConfigCreator(object):
20    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
21    _2hex_re = '[\da-fA-F]{2}'
22    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
23
24    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
25    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
26    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
27        self.cpu_topology = copy.deepcopy(cpu_topology)
28        self.interfaces   = copy.deepcopy(interfaces)
29        del cpu_topology
30        del interfaces
31        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
32        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
33        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
34        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
35        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
36        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
37        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
38        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
39        if only_first_thread:
40            for cores in self.cpu_topology.values():
41                for core in cores.keys():
42                    cores[core] = cores[core][:1]
43        include_lcores = [int(x) for x in include_lcores]
44        exclude_lcores = [int(x) for x in exclude_lcores]
45
46        self.has_zero_lcore = False
47        self.lcores_per_numa = {}
48        total_lcores = 0
49        for numa, cores in self.cpu_topology.items():
50            self.lcores_per_numa[numa] = {'main': [], 'siblings': [], 'all': []}
51            for core, lcores in cores.items():
52                total_lcores += len(lcores)
53                for lcore in list(lcores):
54                    if include_lcores and lcore not in include_lcores:
55                        cores[core].remove(lcore)
56                    if exclude_lcores and lcore in exclude_lcores:
57                        cores[core].remove(lcore)
58                if 0 in lcores:
59                    self.has_zero_lcore = True
60                    lcores.remove(0)
61                    self.lcores_per_numa[numa]['siblings'].extend(lcores)
62                else:
63                    self.lcores_per_numa[numa]['main'].extend(lcores[:1])
64                    self.lcores_per_numa[numa]['siblings'].extend(lcores[1:])
65                self.lcores_per_numa[numa]['all'].extend(lcores)
66
67        for interface in self.interfaces:
68            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
69                if mandatory_interface_field not in interface:
70                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
71
72        Device_str = self._verify_devices_same_type(self.interfaces)
73        if '40Gb' in Device_str:
74            self.speed = 40
75        else:
76            self.speed = 10
77
78        minimum_required_lcores = len(self.interfaces) // 2 + 2
79        if total_lcores < minimum_required_lcores:
80            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
81                    (minimum_required_lcores, len(self.interfaces), total_lcores))
82        interfaces_per_numa = defaultdict(int)
83
84        for i in range(0, len(self.interfaces), 2):
85            numa = self.interfaces[i]['NUMA']
86            if numa != self.interfaces[i+1]['NUMA'] and not ignore_numa:
87                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
88                        (numa, self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
89            interfaces_per_numa[numa] += 2
90
91        self.interfaces_per_numa = interfaces_per_numa
92        self.prefix              = prefix
93        self.zmq_pub_port        = zmq_pub_port
94        self.zmq_rpc_port        = zmq_rpc_port
95        self.ignore_numa         = ignore_numa
96
97    @staticmethod
98    def verify_mac(mac_string):
99        if not ConfigCreator.mac_re.match(mac_string):
100            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
101        return mac_string.lower()
102
103    @staticmethod
104    def _exit_if_bad_ip(ip):
105        if not ConfigCreator._verify_ip(ip):
106            raise DpdkSetup("Got bad IP %s" % ip)
107
108    @staticmethod
109    def _verify_ip(ip):
110        a = ip.split('.')
111        if len(a) != 4:
112            return False
113        for x in a:
114            if not x.isdigit():
115                return False
116            i = int(x)
117            if i < 0 or i > 255:
118                return False
119        return True
120
121    @staticmethod
122    def _verify_devices_same_type(interfaces_list):
123        Device_str = interfaces_list[0]['Device_str']
124        for interface in interfaces_list:
125            if Device_str != interface['Device_str']:
126                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
127        return Device_str
128
129    def create_config(self, filename = None, print_config = False):
130        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
131        config_str += '- port_limit: %s\n' % len(self.interfaces)
132        config_str += '  version: 2\n'
133        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
134        if self.speed > 10:
135            config_str += '  port_bandwidth_gb: %s\n' % self.speed
136        if self.prefix:
137            config_str += '  prefix: %s\n' % self.prefix
138        if self.zmq_pub_port:
139            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
140        if self.zmq_rpc_port:
141            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
142        config_str += '  port_info:\n'
143        for index, interface in enumerate(self.interfaces):
144            if 'ip' in interface:
145                self._exit_if_bad_ip(interface['ip'])
146                self._exit_if_bad_ip(interface['def_gw'])
147                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
148                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
149            else:
150                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
151                if interface.get('loopback_dest'):
152                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
153                else:
154                    config_str += '\n'
155                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
156            if index % 2:
157                config_str += '\n' # dual if barrier
158
159        if not self.ignore_numa:
160            config_str += '  platform:\n'
161            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores
162                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores['all']])
163                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop(0))
164                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
165                lcores_per_dual_if = int(len(lcores_pool) * 2 / len(self.interfaces))
166                config_str += ' '*6 + 'dual_if:\n'
167                for i in range(0, len(self.interfaces), 2):
168                    lcores_for_this_dual_if = list(map(str, sorted(lcores_pool[:lcores_per_dual_if])))
169                    lcores_pool = lcores_pool[lcores_per_dual_if:]
170                    if not lcores_for_this_dual_if:
171                        raise DpdkSetup('lcores_for_this_dual_if is empty (internal bug, please report with details of setup)')
172                    config_str += ' '*8 + '- socket: 0\n'
173                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
174            else:
175                # we will take common minimum among all NUMAs, to satisfy all
176                lcores_per_dual_if = 99
177                extra_lcores = 1 if self.has_zero_lcore else 2
178                # worst case 3 iterations, to ensure master and "rx" have cores left
179                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores['all']) for lcores in self.lcores_per_numa.values()]):
180                    lcores_per_dual_if -= 1
181                    for numa, lcores_dict in self.lcores_per_numa.items():
182                        if not self.interfaces_per_numa[numa]:
183                            continue
184                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(lcores_dict['all']) / self.interfaces_per_numa[numa]))
185                lcores_pool = copy.deepcopy(self.lcores_per_numa)
186                # first, allocate lcores for dual_if section
187                dual_if_section = ' '*6 + 'dual_if:\n'
188                for i in range(0, len(self.interfaces), 2):
189                    numa = self.interfaces[i]['NUMA']
190                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
191                    lcores_for_this_dual_if  = lcores_pool[numa]['all'][:lcores_per_dual_if]
192                    lcores_pool[numa]['all'] = lcores_pool[numa]['all'][lcores_per_dual_if:]
193                    for lcore in lcores_for_this_dual_if:
194                        if lcore in lcores_pool[numa]['main']:
195                            lcores_pool[numa]['main'].remove(lcore)
196                        elif lcore in lcores_pool[numa]['siblings']:
197                            lcores_pool[numa]['siblings'].remove(lcore)
198                        else:
199                            raise DpdkSetup('lcore not in main nor in siblings list (internal bug, please report with details of setup)')
200                    if not lcores_for_this_dual_if:
201                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
202                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(list(map(str, sorted(lcores_for_this_dual_if))))
203
204                # take the cores left to master and rx
205                mains_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['main']]
206                siblings_left = [lcore for lcores in lcores_pool.values() for lcore in lcores['siblings']]
207                if mains_left:
208                    rx_core = mains_left.pop(0)
209                else:
210                    rx_core = siblings_left.pop(0)
211                if self.has_zero_lcore:
212                    master_core = 0
213                elif mains_left:
214                    master_core = mains_left.pop(0)
215                else:
216                    master_core = siblings_left.pop(0)
217                config_str += ' '*6 + 'master_thread_id: %s\n' % master_core
218                config_str += ' '*6 + 'latency_thread_id: %s\n' % rx_core
219                # add the dual_if section
220                config_str += dual_if_section
221
222        # verify config is correct YAML format
223        try:
224            yaml.safe_load(config_str)
225        except Exception as e:
226            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
227
228        if print_config:
229            print(config_str)
230        if filename:
231            if os.path.exists(filename):
232                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
233                    print('Skipping.')
234                    return config_str
235            with open(filename, 'w') as f:
236                f.write(config_str)
237            print('Saved to %s.' % filename)
238        return config_str
239
240
241class map_driver(object):
242    args=None;
243    cfg_file='/etc/trex_cfg.yaml'
244    parent_args = None
245
246class DpdkSetup(Exception):
247    pass
248
249class CIfMap:
250
251    def __init__(self, cfg_file):
252        self.m_cfg_file =cfg_file;
253        self.m_cfg_dict={};
254        self.m_devices={};
255        self.m_is_mellanox_mode=False;
256
257    def dump_error (self,err):
258        s="""%s
259From this TRex version a configuration file must exist in /etc/ folder "
260The name of the configuration file should be /etc/trex_cfg.yaml "
261The minimum configuration file should include something like this
262- version       : 2 # version 2 of the configuration file
263  interfaces    : ["03:00.0","03:00.1","13:00.1","13:00.0"]  # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
264  port_limit      : 2 # number of ports to use valid is 2,4,6,8,10,12
265
266example of already bind devices
267
268$ ./dpdk_nic_bind.py --status
269
270Network devices using DPDK-compatible driver
271============================================
2720000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2730000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2740000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2750000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
276
277Network devices using kernel driver
278===================================
2790000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
280
281Other network devices
282=====================
283
284
285          """ % (err);
286        return s;
287
288
289    def raise_error  (self,err):
290        s= self.dump_error (err)
291        raise DpdkSetup(s)
292
293    def set_only_mellanox_nics(self):
294        self.m_is_mellanox_mode=True;
295
296    def get_only_mellanox_nics(self):
297        return self.m_is_mellanox_mode
298
299
300    def read_pci (self,pci_id,reg_id):
301        out=subprocess.check_output(['setpci', '-s',pci_id, '%s.w' %(reg_id)])
302        out=out.decode(errors='replace');
303        return (out.strip());
304
305    def write_pci (self,pci_id,reg_id,val):
306        out=subprocess.check_output(['setpci','-s',pci_id, '%s.w=%s' %(reg_id,val)])
307        out=out.decode(errors='replace');
308        return (out.strip());
309
310    def tune_mlx5_device (self,pci_id):
311        # set PCIe Read to 1024 and not 512 ... need to add it to startup s
312        val=self.read_pci (pci_id,68)
313        if val[0]!='3':
314            val='3'+val[1:]
315            self.write_pci (pci_id,68,val)
316            assert(self.read_pci (pci_id,68)==val);
317
318    def get_mtu_mlx5 (self,dev_id):
319        if len(dev_id)>0:
320            out=subprocess.check_output(['ifconfig', dev_id])
321            out=out.decode(errors='replace');
322            obj=re.search(r'MTU:(\d+)',out,flags=re.MULTILINE|re.DOTALL);
323            if obj:
324                return int(obj.group(1));
325            else:
326                obj=re.search(r'mtu (\d+)',out,flags=re.MULTILINE|re.DOTALL);
327                if obj:
328                    return int(obj.group(1));
329                else:
330                    return -1
331
332    def set_mtu_mlx5 (self,dev_id,new_mtu):
333        if len(dev_id)>0:
334            out=subprocess.check_output(['ifconfig', dev_id,'mtu',str(new_mtu)])
335            out=out.decode(errors='replace');
336
337
338    def set_max_mtu_mlx5_device(self,dev_id):
339        mtu=9*1024+22
340        dev_mtu=self.get_mtu_mlx5 (dev_id);
341        if (dev_mtu>0) and (dev_mtu!=mtu):
342            self.set_mtu_mlx5(dev_id,mtu);
343            if self.get_mtu_mlx5(dev_id) != mtu:
344                print("Could not set MTU to %d" % mtu)
345                exit(-1);
346
347
348    def disable_flow_control_mlx5_device (self,dev_id):
349
350           if len(dev_id)>0:
351               my_stderr = open("/dev/null","wb")
352               cmd ='ethtool -A '+dev_id + ' rx off tx off '
353               subprocess.call(cmd, stdout=my_stderr,stderr=my_stderr, shell=True)
354               my_stderr.close();
355
356    def check_ofed_version (self):
357        ofed_info='/usr/bin/ofed_info'
358        ofed_ver= '-3.4-'
359        ofed_ver_show= '3.4-1'
360
361
362        if not os.path.isfile(ofed_info):
363            print("OFED %s is not installed on this setup" % ofed_info)
364            exit(-1);
365
366        try:
367          out = subprocess.check_output([ofed_info])
368        except Exception as e:
369            print("OFED %s can't run " % (ofed_info))
370            exit(-1);
371
372        lines=out.splitlines();
373
374        if len(lines)>1:
375            if not (ofed_ver in str(lines[0])):
376                print("installed OFED version is '%s' should be at least '%s' and up" % (lines[0],ofed_ver_show))
377                exit(-1);
378
379    def verify_ofed_os(self):
380        err_msg = 'Warning: Mellanox NICs where tested only with RedHat/CentOS 7.2\n'
381        err_msg += 'Correct usage with other Linux distributions is not guaranteed.'
382        try:
383            dist = platform.dist()
384            if dist[0] not in ('redhat', 'centos') or not dist[1].startswith('7.2'):
385                print(err_msg)
386        except Exception as e:
387            print('Error while determining OS type: %s' % e)
388
389    def load_config_file (self):
390
391        fcfg=self.m_cfg_file
392
393        if not os.path.isfile(fcfg) :
394            self.raise_error ("There is no valid configuration file %s\n" % fcfg)
395
396        try:
397          stream = open(fcfg, 'r')
398          self.m_cfg_dict= yaml.safe_load(stream)
399        except Exception as e:
400          print(e);
401          raise e
402
403        stream.close();
404        cfg_dict = self.m_cfg_dict[0]
405        if 'version' not in cfg_dict:
406            raise DpdkSetup("Configuration file %s is old, it should include version field\n" % fcfg )
407
408        if int(cfg_dict['version'])<2 :
409            raise DpdkSetup("Configuration file %s is old, expected version 2, got: %s\n" % (fcfg, cfg_dict['version']))
410
411        if 'interfaces' not in self.m_cfg_dict[0]:
412            raise DpdkSetup("Configuration file %s is old, it should include interfaces field with even number of elements" % fcfg)
413
414        if_list=self.m_cfg_dict[0]['interfaces']
415        l=len(if_list);
416        if l > 16:
417            raise DpdkSetup("Configuration file %s should include interfaces field with maximum 16 elements, got: %s." % (fcfg,l))
418        if l % 2:
419            raise DpdkSetup("Configuration file %s should include even number of interfaces " % (fcfg,l))
420        if 'port_limit' in cfg_dict and cfg_dict['port_limit'] > len(if_list):
421            raise DpdkSetup('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
422
423
424    def do_bind_one (self,key,mellanox):
425        if mellanox:
426            drv="mlx5_core"
427        else:
428            drv="igb_uio"
429
430        cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
431        print(cmd)
432        res=os.system(cmd);
433        if res!=0:
434            raise DpdkSetup('')
435
436
437    def pci_name_to_full_name (self,pci_name):
438          c='[0-9A-Fa-f]';
439          sp='[:]'
440          s_short=c+c+sp+c+c+'[.]'+c;
441          s_full=c+c+c+c+sp+s_short
442          re_full = re.compile(s_full)
443          re_short = re.compile(s_short)
444
445          if re_short.match(pci_name):
446              return '0000:'+pci_name
447
448          if re_full.match(pci_name):
449              return pci_name
450
451          err=" %s is not a valid pci address \n" %pci_name;
452          raise DpdkSetup(err)
453
454
455    def run_dpdk_lspci (self):
456        dpdk_nic_bind.get_nic_details()
457        self.m_devices= dpdk_nic_bind.devices
458
459    def do_run (self,only_check_all_mlx=False):
460        self.run_dpdk_lspci ()
461        if (map_driver.parent_args.dump_interfaces is None or
462                    (map_driver.parent_args.dump_interfaces == [] and
463                            map_driver.parent_args.cfg)):
464            self.load_config_file()
465            if_list=self.m_cfg_dict[0]['interfaces']
466        else:
467            if_list = map_driver.parent_args.dump_interfaces
468            if not if_list:
469                for dev in self.m_devices.values():
470                    if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
471                        if_list.append(dev['Slot'])
472
473        if_list = list(map(self.pci_name_to_full_name, if_list))
474
475
476        # check how many mellanox cards we have
477        Mellanox_cnt=0;
478        for key in if_list:
479            if key not in self.m_devices:
480                err=" %s does not exist " %key;
481                raise DpdkSetup(err)
482
483            if 'Vendor_str' not in self.m_devices[key]:
484                err=" %s does not have Vendor_str " %key;
485                raise DpdkSetup(err)
486
487            if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
488                Mellanox_cnt=Mellanox_cnt+1
489
490
491        if not map_driver.parent_args.dump_interfaces:
492            if  ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
493               err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
494               raise DpdkSetup(err)
495
496
497        if not map_driver.parent_args.dump_interfaces:
498            if  Mellanox_cnt>0 :
499                self.set_only_mellanox_nics()
500
501        if self.get_only_mellanox_nics():
502            if not map_driver.parent_args.no_ofed_check:
503                self.verify_ofed_os()
504                self.check_ofed_version()
505
506            for key in if_list:
507                pci_id=self.m_devices[key]['Slot_str']
508                self.tune_mlx5_device (pci_id)
509                if 'Interface' in self.m_devices[key]:
510                    dev_id=self.m_devices[key]['Interface']
511                    self.disable_flow_control_mlx5_device (dev_id)
512                    self.set_max_mtu_mlx5_device(dev_id)
513
514
515        if only_check_all_mlx:
516            if Mellanox_cnt >0:
517                exit(1);
518            else:
519                exit(0);
520
521        for key in if_list:
522            if key not in self.m_devices:
523                err=" %s does not exist " %key;
524                raise DpdkSetup(err)
525
526            if 'Driver_str' in self.m_devices[key]:
527                if self.m_devices[key]['Driver_str'] not in (dpdk_nic_bind.dpdk_drivers+dpdk_nic_bind.dpdk_and_kernel) :
528                    self.do_bind_one (key,(Mellanox_cnt>0))
529                    pass;
530            else:
531                self.do_bind_one (key,(Mellanox_cnt>0))
532                pass;
533
534        if (Mellanox_cnt==0):
535            # We are not in Mellanox case, we can do this check only in case of Intel (another process is running)
536            if if_list and map_driver.args.parent and (dpdk_nic_bind.get_igb_uio_usage()):
537                pid = dpdk_nic_bind.get_pid_using_pci(if_list)
538                if pid:
539                    cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
540                    print('Some or all of given interfaces are in use by following process:\npid: %s, cmd: %s' % (pid, cmdline))
541                    if not dpdk_nic_bind.confirm('Ignore and proceed (y/N):'):
542                        sys.exit(1)
543                else:
544                       print('WARNING: Some other program is using DPDK driver.\nIf it is TRex and you did not configure it for dual run, current command will fail.')
545        if map_driver.parent_args.stl and not map_driver.parent_args.no_scapy_server:
546            try:
547                master_core = self.m_cfg_dict[0]['platform']['master_thread_id']
548            except:
549                master_core = 0
550            ret = os.system('%s scapy_daemon_server restart -c %s' % (sys.executable, master_core))
551            if ret:
552                print("Could not start scapy_daemon_server, which is needed by GUI to create packets.\nIf you don't need it, use --no-scapy-server flag.")
553                sys.exit(1)
554
555
556    def do_return_to_linux(self):
557        if not self.m_devices:
558            self.run_dpdk_lspci()
559        dpdk_interfaces = []
560        for device in self.m_devices.values():
561            if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
562                dpdk_interfaces.append(device['Slot'])
563        if not dpdk_interfaces:
564            print('No DPDK bound interfaces.')
565            return
566        if dpdk_nic_bind.get_igb_uio_usage():
567            pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
568            if pid:
569                cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
570                print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
571                if not dpdk_nic_bind.confirm('Confirm (y/N):'):
572                    return
573        drivers_table = {
574            'rte_ixgbe_pmd': 'ixgbe',
575            'rte_igb_pmd': 'igb',
576            'rte_i40e_pmd': 'i40e',
577            'rte_em_pmd': 'e1000',
578            'rte_vmxnet3_pmd': 'vmxnet3',
579            'rte_virtio_pmd': 'virtio-pci',
580            'rte_enic_pmd': 'enic',
581        }
582        for pci, info in dpdk_nic_bind.get_info_from_trex(dpdk_interfaces).items():
583            if pci not in self.m_devices:
584                raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
585            dev = self.m_devices[pci]
586            if info['TRex_Driver'] not in drivers_table:
587                print('Got unknown driver %s, description: %s' % (info['TRex_Driver'], dev['Device_str']))
588            else:
589                print('Returning to Linux %s' % pci)
590                dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
591
592    def _get_cpu_topology(self):
593        cpu_topology_file = '/proc/cpuinfo'
594        # physical processor -> physical core -> logical processing units (threads)
595        cpu_topology = OrderedDict()
596        if not os.path.exists(cpu_topology_file):
597            raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file)
598        with open(cpu_topology_file) as f:
599            for lcore in f.read().split('\n\n'):
600                if not lcore:
601                    continue
602                lcore_dict = OrderedDict()
603                for line in lcore.split('\n'):
604                    key, val = line.split(':', 1)
605                    lcore_dict[key.strip()] = val.strip()
606                if 'processor' not in lcore_dict:
607                    continue
608                numa = int(lcore_dict.get('physical id', -1))
609                if numa not in cpu_topology:
610                    cpu_topology[numa] = OrderedDict()
611                core = int(lcore_dict.get('core id', lcore_dict['processor']))
612                if core not in cpu_topology[numa]:
613                    cpu_topology[numa][core] = []
614                cpu_topology[numa][core].append(int(lcore_dict['processor']))
615        if not cpu_topology:
616            raise DpdkSetup('Could not determine CPU topology from %s' % cpu_topology_file)
617        return cpu_topology
618
619    # input: list of different descriptions of interfaces: index, pci, name etc.
620    # Binds to dpdk wanted interfaces, not bound to any driver.
621    # output: list of maps of devices in dpdk_* format (self.m_devices.values())
622    def _get_wanted_interfaces(self, input_interfaces, get_macs = True):
623        if type(input_interfaces) is not list:
624            raise DpdkSetup('type of input interfaces should be list')
625        if not len(input_interfaces):
626            raise DpdkSetup('Please specify interfaces to use in the config')
627        if len(input_interfaces) % 2:
628            raise DpdkSetup('Please specify even number of interfaces')
629        wanted_interfaces = []
630        sorted_pci = sorted(self.m_devices.keys())
631        for interface in input_interfaces:
632            dev = None
633            try:
634                interface = int(interface)
635                if interface < 0 or interface >= len(sorted_pci):
636                    raise DpdkSetup('Index of an interfaces should be in range 0:%s' % (len(sorted_pci) - 1))
637                dev = self.m_devices[sorted_pci[interface]]
638            except ValueError:
639                for d in self.m_devices.values():
640                    if interface in (d['Interface'], d['Slot'], d['Slot_str']):
641                        dev = d
642                        break
643            if not dev:
644                raise DpdkSetup('Could not find information about this interface: %s' % interface)
645            if dev in wanted_interfaces:
646                raise DpdkSetup('Interface %s is specified twice' % interface)
647            dev['Interface_argv'] = interface
648            wanted_interfaces.append(dev)
649
650        if get_macs:
651            unbound = []
652            dpdk_bound = []
653            for interface in wanted_interfaces:
654                if 'Driver_str' not in interface:
655                    unbound.append(interface['Slot'])
656                elif interface.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
657                    dpdk_bound.append(interface['Slot'])
658            if unbound or dpdk_bound:
659                for pci, info in dpdk_nic_bind.get_info_from_trex(unbound + dpdk_bound).items():
660                    if pci not in self.m_devices:
661                        raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
662                    self.m_devices[pci].update(info)
663
664        return wanted_interfaces
665
666    def do_create(self):
667
668        ips = map_driver.args.ips
669        def_gws = map_driver.args.def_gws
670        dest_macs = map_driver.args.dest_macs
671        if map_driver.args.force_macs:
672            ip_config = False
673            if ips:
674                raise DpdkSetup("If using --force-macs, should not specify ips")
675            if def_gws:
676                raise DpdkSetup("If using --force-macs, should not specify default gateways")
677        elif ips:
678            ip_config = True
679            if not def_gws:
680                raise DpdkSetup("If specifying ips, must specify also def-gws")
681            if dest_macs:
682                raise DpdkSetup("If specifying ips, should not specify dest--macs")
683            if len(ips) != len(def_gws) or len(ips) != len(map_driver.args.create_interfaces):
684                raise DpdkSetup("Number of given IPs should equal number of given def-gws and number of interfaces")
685        else:
686            if dest_macs:
687                ip_config = False
688            else:
689                ip_config = True
690
691        # gather info about NICS from dpdk_nic_bind.py
692        if not self.m_devices:
693            self.run_dpdk_lspci()
694        wanted_interfaces = self._get_wanted_interfaces(map_driver.args.create_interfaces, get_macs = not ip_config)
695
696        for i, interface in enumerate(wanted_interfaces):
697            dual_index = i + 1 - (i % 2) * 2
698            if ip_config:
699                if isinstance(ips, list) and len(ips) > i:
700                    interface['ip'] = ips[i]
701                else:
702                    interface['ip'] = '.'.join([str(i+1) for _ in range(4)])
703                if isinstance(def_gws, list) and len(def_gws) > i:
704                    interface['def_gw'] = def_gws[i]
705                else:
706                    interface['def_gw'] = '.'.join([str(dual_index+1) for _ in range(4)])
707            else:
708                dual_if = wanted_interfaces[dual_index]
709                if 'MAC' not in interface:
710                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
711                if 'MAC' not in dual_if:
712                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % dual_if['Interface_argv'])
713                interface['src_mac'] = interface['MAC']
714                if isinstance(dest_macs, list) and len(dest_macs) > i:
715                    interface['dest_mac'] = dest_macs[i]
716                else:
717                    interface['dest_mac'] = dual_if['MAC']
718                    interface['loopback_dest'] = True
719
720        config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
721                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa,
722                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
723        if map_driver.args.output_config:
724            config.create_config(filename = map_driver.args.output_config)
725        else:
726            print('### Dumping config to screen, use -o flag to save to file')
727            config.create_config(print_config = True)
728
729    def do_interactive_create(self):
730        ignore_numa = False
731        cpu_topology = self._get_cpu_topology()
732        total_lcores = sum([len(lcores) for cores in cpu_topology.values() for lcores in cores.values()])
733        if total_lcores < 1:
734            raise DpdkSetup('Script could not determine number of cores of the system, exiting.')
735        elif total_lcores < 2:
736            if dpdk_nic_bind.confirm("You only have 1 core and can't run TRex at all. Ignore and continue? (y/N): "):
737                ignore_numa = True
738            else:
739                sys.exit(1)
740        elif total_lcores < 3:
741            if dpdk_nic_bind.confirm("You only have 2 cores and will be able to run only stateful without latency checks.\nIgnore and continue? (y/N): "):
742                ignore_numa = True
743            else:
744                sys.exit(1)
745
746        if map_driver.args.force_macs:
747            ip_based = False
748        elif dpdk_nic_bind.confirm("By default, IP based configuration file will be created. Do you want to use MAC based config? (y/N)"):
749            ip_based = False
750        else:
751            ip_based = True
752            ip_addr_digit = 1
753
754        if not self.m_devices:
755            self.run_dpdk_lspci()
756        dpdk_nic_bind.show_table(get_macs = not ip_based)
757        print('Please choose even number of interfaces from the list above, either by ID , PCI or Linux IF')
758        print('Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.')
759        print('Stateless can be in any order.')
760        numa = None
761        for dev in self.m_devices.values():
762            if numa is None:
763                numa = dev['NUMA']
764            elif numa != dev['NUMA']:
765                print('For performance, try to choose each pair of interfaces to be on the same NUMA.')
766                break
767        while True:
768            try:
769                input = dpdk_nic_bind.read_line('Enter list of interfaces separated by space (for example: 1 3) : ')
770                create_interfaces = input.replace(',', ' ').replace(';', ' ').split()
771                wanted_interfaces = self._get_wanted_interfaces(create_interfaces)
772                ConfigCreator._verify_devices_same_type(wanted_interfaces)
773            except Exception as e:
774                print(e)
775                continue
776            break
777        print('')
778
779        for interface in wanted_interfaces:
780            if interface['Active']:
781                print('Interface %s is active. Using it by TRex might close ssh connections etc.' % interface['Interface_argv'])
782                if not dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
783                    sys.exit(1)
784
785        for i, interface in enumerate(wanted_interfaces):
786            if not ip_based:
787                if 'MAC' not in interface:
788                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
789                interface['src_mac'] = interface['MAC']
790            dual_index = i + 1 - (i % 2) * 2
791            dual_int = wanted_interfaces[dual_index]
792            if not ignore_numa and interface['NUMA'] != dual_int['NUMA']:
793                print('NUMA is different at pair of interfaces: %s and %s. It will reduce performance.' % (interface['Interface_argv'], dual_int['Interface_argv']))
794                if dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
795                    ignore_numa = True
796                    print('')
797                else:
798                    return
799
800            if ip_based:
801                if ip_addr_digit % 2 == 0:
802                    dual_ip_digit = ip_addr_digit - 1
803                else:
804                    dual_ip_digit = ip_addr_digit + 1
805                ip = '.'.join([str(ip_addr_digit) for _ in range(4)])
806                def_gw= '.'.join([str(dual_ip_digit) for _ in range(4)])
807                ip_addr_digit += 1
808
809                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
810                if dpdk_nic_bind.confirm("Putting IP %s, default gw %s Change it?(y/N)." % (ip, def_gw)):
811                    while True:
812                        ip = dpdk_nic_bind.read_line('Please enter IP address for interface %s: ' % interface['Interface_argv'])
813                        if not ConfigCreator._verify_ip(ip):
814                            print ("Bad IP address format")
815                        else:
816                            break
817                    while True:
818                        def_gw = dpdk_nic_bind.read_line('Please enter default gateway for interface %s: ' % interface['Interface_argv'])
819                        if not ConfigCreator._verify_ip(def_gw):
820                            print ("Bad IP address format")
821                        else:
822                            break
823                wanted_interfaces[i]['ip'] = ip
824                wanted_interfaces[i]['def_gw'] = def_gw
825            else:
826                dest_mac = dual_int['MAC']
827                loopback_dest = True
828                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
829                if dpdk_nic_bind.confirm("Destination MAC is %s. Change it to MAC of DUT? (y/N)." % dest_mac):
830                    while True:
831                        input_mac = dpdk_nic_bind.read_line('Please enter new destination MAC of interface %s: ' % interface['Interface_argv'])
832                        try:
833                            if input_mac:
834                                ConfigCreator._convert_mac(input_mac) # verify format
835                                dest_mac = input_mac
836                                loopback_dest = False
837                            else:
838                                print('Leaving the loopback MAC.')
839                        except Exception as e:
840                            print(e)
841                            continue
842                        break
843                wanted_interfaces[i]['dest_mac'] = dest_mac
844                wanted_interfaces[i]['loopback_dest'] = loopback_dest
845
846        config = ConfigCreator(cpu_topology, wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
847                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa or ignore_numa,
848                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
849        if dpdk_nic_bind.confirm('Print preview of generated config? (Y/n)', default = True):
850            config.create_config(print_config = True)
851        if dpdk_nic_bind.confirm('Save the config to file? (Y/n)', default = True):
852            print('Default filename is /etc/trex_cfg.yaml')
853            filename = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new file: ')
854            if not filename:
855                filename = '/etc/trex_cfg.yaml'
856            config.create_config(filename = filename)
857
858
859def parse_parent_cfg (parent_cfg):
860    parent_parser = argparse.ArgumentParser(add_help = False)
861    parent_parser.add_argument('-?', '-h', '--help', dest = 'help', action = 'store_true')
862    parent_parser.add_argument('--cfg', default='')
863    parent_parser.add_argument('--dump-interfaces', nargs='*', default=None)
864    parent_parser.add_argument('--no-ofed-check', action = 'store_true')
865    parent_parser.add_argument('--no-scapy-server', action = 'store_true')
866    parent_parser.add_argument('--no-watchdog', action = 'store_true')
867    parent_parser.add_argument('-i', action = 'store_true', dest = 'stl', default = False)
868    map_driver.parent_args, _ = parent_parser.parse_known_args(shlex.split(parent_cfg))
869    if map_driver.parent_args.help:
870        sys.exit(0)
871
872
873def process_options ():
874    parser = argparse.ArgumentParser(usage="""
875
876Examples:
877---------
878
879To return to Linux the DPDK bound interfaces (for ifconfig etc.)
880  sudo ./dpdk_set_ports.py -l
881
882To create TRex config file using interactive mode
883  sudo ./dpdk_set_ports.py -i
884
885To create a default config file (example1)
886  sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
887
888To create a default config file (example2)
889  sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
890
891To show interfaces status
892  sudo ./dpdk_set_ports.py -s
893
894To see more detailed info on interfaces (table):
895  sudo ./dpdk_set_ports.py -t
896
897    """,
898    description=" unbind dpdk interfaces ",
899    epilog=" written by hhaim");
900
901    parser.add_argument("-l", "--linux", action='store_true',
902                      help=""" Return all DPDK interfaces to Linux driver """,
903     )
904
905    parser.add_argument("--cfg",
906                      help=""" configuration file name  """,
907     )
908
909    parser.add_argument("--parent",
910                      help=argparse.SUPPRESS
911     )
912
913    parser.add_argument('--dump-pci-description', help=argparse.SUPPRESS, dest='dump_pci_desc', action='store_true')
914
915    parser.add_argument("-i", "--interactive", action='store_true',
916                      help=""" Create TRex config in interactive mode """,
917     )
918
919    parser.add_argument("-c", "--create", nargs='*', default=None, dest='create_interfaces', metavar='<interface>',
920                      help="""Try to create a configuration file by specifying needed interfaces by PCI address or Linux names: eth1 etc.""",
921     )
922
923    parser.add_argument("--ci", "--cores-include", nargs='*', default=[], dest='create_include', metavar='<cores>',
924                      help="""White list of cores to use. Make sure there is enough for each NUMA.""",
925     )
926
927    parser.add_argument("--ce", "--cores-exclude", nargs='*', default=[], dest='create_exclude', metavar='<cores>',
928                      help="""Black list of cores to exclude. Make sure there will be enough for each NUMA.""",
929     )
930
931    parser.add_argument("--no-ht", default=False, dest='no_ht', action='store_true',
932                      help="""Use only one thread of each Core in created config yaml (No Hyper-Threading).""",
933     )
934
935    parser.add_argument("--dest-macs", nargs='*', default=[], action='store',
936                      help="""Destination MACs to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
937     )
938
939    parser.add_argument("--force-macs", default=False, action='store_true',
940                      help="""Use MACs in created config file.""",
941     )
942
943    parser.add_argument("--ips", nargs='*', default=[], action='store',
944                      help="""IP addresses to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
945     )
946
947    parser.add_argument("--def-gws", nargs='*', default=[], action='store',
948                      help="""Default gateways to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
949     )
950
951    parser.add_argument("-o", default=None, action='store', metavar='PATH', dest = 'output_config',
952                      help="""Output the config to this file.""",
953     )
954
955    parser.add_argument("--prefix", default=None, action='store',
956                      help="""Advanced option: prefix to be used in TRex config in case of parallel instances.""",
957     )
958
959    parser.add_argument("--zmq-pub-port", default=None, action='store',
960                      help="""Advanced option: ZMQ Publisher port to be used in TRex config in case of parallel instances.""",
961     )
962
963    parser.add_argument("--zmq-rpc-port", default=None, action='store',
964                      help="""Advanced option: ZMQ RPC port to be used in TRex config in case of parallel instances.""",
965     )
966
967    parser.add_argument("--ignore-numa", default=False, action='store_true',
968                      help="""Advanced option: Ignore NUMAs for config creation. Use this option only if you have to, as it will reduce performance.""",
969     )
970
971    parser.add_argument("-s", "--show", action='store_true',
972                      help=""" show the status """,
973     )
974
975    parser.add_argument("-t", "--table", action='store_true',
976                      help=""" show table with NICs info """,
977     )
978
979    parser.add_argument('--version', action='version',
980                        version="0.2" )
981
982    map_driver.args = parser.parse_args();
983    if map_driver.args.parent :
984        parse_parent_cfg (map_driver.args.parent)
985        if map_driver.parent_args.cfg:
986            map_driver.cfg_file = map_driver.parent_args.cfg;
987    if  map_driver.args.cfg :
988        map_driver.cfg_file = map_driver.args.cfg;
989
990def main ():
991    try:
992        if os.getuid() != 0:
993            raise DpdkSetup('Please run this program as root/with sudo')
994
995        process_options ()
996
997        if map_driver.args.show:
998            dpdk_nic_bind.show_status()
999            return
1000
1001        if map_driver.args.table:
1002            dpdk_nic_bind.show_table()
1003            return
1004
1005        if map_driver.args.dump_pci_desc:
1006            dpdk_nic_bind.dump_pci_description()
1007            return
1008
1009        obj =CIfMap(map_driver.cfg_file);
1010
1011        if map_driver.args.create_interfaces is not None:
1012            obj.do_create();
1013        elif map_driver.args.interactive:
1014            obj.do_interactive_create();
1015        elif map_driver.args.linux:
1016            obj.do_return_to_linux();
1017        else:
1018            obj.do_run();
1019        print('')
1020    except DpdkSetup as e:
1021        print(e)
1022        exit(-1)
1023
1024
1025
1026if __name__ == '__main__':
1027    main()
1028