dpdk_setup_ports.py revision 7d8d95fb
1#! /usr/bin/python
2# hhaim
3import sys
4import os
5python_ver = 'python%s' % sys.version_info[0]
6sys.path.append(os.path.join('external_libs', 'pyyaml-3.11', python_ver))
7import yaml
8import dpdk_nic_bind
9import re
10import argparse
11import copy
12import shlex
13import traceback
14from collections import defaultdict, OrderedDict
15from distutils.util import strtobool
16import getpass
17
18class ConfigCreator(object):
19    mandatory_interface_fields = ['Slot_str', 'Device_str', 'NUMA']
20    _2hex_re = '[\da-fA-F]{2}'
21    mac_re = re.compile('^({0}:){{5}}{0}$'.format(_2hex_re))
22
23    # cpu_topology - dict: physical processor -> physical core -> logical processing unit (thread)
24    # interfaces - array of dicts per interface, should include "mandatory_interface_fields" values
25    def __init__(self, cpu_topology, interfaces, include_lcores = [], exclude_lcores = [], only_first_thread = False, zmq_rpc_port = None, zmq_pub_port = None, prefix = None, ignore_numa = False):
26        self.cpu_topology = copy.deepcopy(cpu_topology)
27        self.interfaces   = copy.deepcopy(interfaces)
28        del cpu_topology
29        del interfaces
30        assert isinstance(self.cpu_topology, dict), 'Type of cpu_topology should be dict, got: %s' % type(self.cpu_topology)
31        assert len(self.cpu_topology.keys()) > 0, 'cpu_topology should contain at least one processor'
32        assert isinstance(self.interfaces, list), 'Type of interfaces should be list, got: %s' % type(list)
33        assert len(self.interfaces) % 2 == 0, 'Should be even number of interfaces, got: %s' % len(self.interfaces)
34        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
35        assert isinstance(include_lcores, list), 'include_lcores should be list, got: %s' % type(include_lcores)
36        assert isinstance(exclude_lcores, list), 'exclude_lcores should be list, got: %s' % type(exclude_lcores)
37        assert len(self.interfaces) >= 2, 'Should be at least two interfaces, got: %s' % len(self.interfaces)
38        if only_first_thread:
39            for cores in self.cpu_topology.values():
40                for core in cores.keys():
41                    cores[core] = cores[core][:1]
42        include_lcores = [int(x) for x in include_lcores]
43        exclude_lcores = [int(x) for x in exclude_lcores]
44        self.has_zero_lcore = False
45        for numa, cores in self.cpu_topology.items():
46            for core, lcores in cores.items():
47                for lcore in copy.copy(lcores):
48                    if include_lcores and lcore not in include_lcores:
49                        cores[core].remove(lcore)
50                    if exclude_lcores and lcore in exclude_lcores:
51                        cores[core].remove(lcore)
52                if 0 in lcores:
53                    self.has_zero_lcore = True
54                    cores[core].remove(0)
55                    zero_lcore_numa = numa
56                    zero_lcore_core = core
57                    zero_lcore_siblings = cores[core]
58        if self.has_zero_lcore:
59            del self.cpu_topology[zero_lcore_numa][zero_lcore_core]
60            self.cpu_topology[zero_lcore_numa][zero_lcore_core] = zero_lcore_siblings
61        for interface in self.interfaces:
62            for mandatory_interface_field in ConfigCreator.mandatory_interface_fields:
63                if mandatory_interface_field not in interface:
64                    raise DpdkSetup("Expected '%s' field in interface dictionary, got: %s" % (mandatory_interface_field, interface))
65        Device_str = self._verify_devices_same_type(self.interfaces)
66        if '40Gb' in Device_str:
67            self.speed = 40
68        else:
69            self.speed = 10
70        lcores_per_numa = OrderedDict()
71        system_lcores = int(self.has_zero_lcore)
72        for numa, core in self.cpu_topology.items():
73            for lcores in core.values():
74                if numa not in lcores_per_numa:
75                    lcores_per_numa[numa] = []
76                lcores_per_numa[numa].extend(lcores)
77                system_lcores += len(lcores)
78        minimum_required_lcores = len(self.interfaces) / 2 + 2
79        if system_lcores < minimum_required_lcores:
80            raise DpdkSetup('Your system should have at least %s cores for %s interfaces, and it has: %s.' %
81                    (minimum_required_lcores, len(self.interfaces), system_lcores + (0 if self.has_zero_lcore else 1)))
82        interfaces_per_numa = defaultdict(int)
83        for i in range(0, len(self.interfaces), 2):
84            if self.interfaces[i]['NUMA'] != self.interfaces[i+1]['NUMA'] and not ignore_numa:
85                raise DpdkSetup('NUMA of each pair of interfaces should be the same. Got NUMA %s for client interface %s, NUMA %s for server interface %s' %
86                        (self.interfaces[i]['NUMA'], self.interfaces[i]['Slot_str'], self.interfaces[i+1]['NUMA'], self.interfaces[i+1]['Slot_str']))
87            interfaces_per_numa[self.interfaces[i]['NUMA']] += 2
88        self.lcores_per_numa     = lcores_per_numa
89        self.interfaces_per_numa = interfaces_per_numa
90        self.prefix              = prefix
91        self.zmq_pub_port        = zmq_pub_port
92        self.zmq_rpc_port        = zmq_rpc_port
93        self.ignore_numa         = ignore_numa
94
95    @staticmethod
96    def verify_mac(mac_string):
97        if not ConfigCreator.mac_re.match(mac_string):
98            raise DpdkSetup('MAC address should be in format of 12:34:56:78:9a:bc, got: %s' % mac_string)
99        return mac_string.lower()
100
101    @staticmethod
102    def _exit_if_bad_ip(ip):
103        if not ConfigCreator._verify_ip(ip):
104            raise DpdkSetup("Got bad IP %s" % ip)
105
106    @staticmethod
107    def _verify_ip(ip):
108        a = ip.split('.')
109        if len(a) != 4:
110            return False
111        for x in a:
112            if not x.isdigit():
113                return False
114            i = int(x)
115            if i < 0 or i > 255:
116                return False
117        return True
118
119    @staticmethod
120    def _verify_devices_same_type(interfaces_list):
121        Device_str = interfaces_list[0]['Device_str']
122        for interface in interfaces_list:
123            if Device_str != interface['Device_str']:
124                raise DpdkSetup('Interfaces should be of same type, got:\n\t* %s\n\t* %s' % (Device_str, interface['Device_str']))
125        return Device_str
126
127    def create_config(self, filename = None, print_config = False):
128        config_str = '### Config file generated by dpdk_setup_ports.py ###\n\n'
129        config_str += '- port_limit: %s\n' % len(self.interfaces)
130        config_str += '  version: 2\n'
131        config_str += "  interfaces: ['%s']\n" % "', '".join([interface['Slot_str'] for interface in self.interfaces])
132        if self.speed > 10:
133            config_str += '  port_bandwidth_gb: %s\n' % self.speed
134        if self.prefix:
135            config_str += '  prefix: %s\n' % self.prefix
136        if self.zmq_pub_port:
137            config_str += '  zmq_pub_port: %s\n' % self.zmq_pub_port
138        if self.zmq_rpc_port:
139            config_str += '  zmq_rpc_port: %s\n' % self.zmq_rpc_port
140        config_str += '  port_info:\n'
141        for index, interface in enumerate(self.interfaces):
142            if 'ip' in interface:
143                self._exit_if_bad_ip(interface['ip'])
144                self._exit_if_bad_ip(interface['def_gw'])
145                config_str += ' '*6 + '- ip: %s\n' % interface['ip']
146                config_str += ' '*8 + 'default_gw: %s\n' % interface['def_gw']
147            else:
148                config_str += ' '*6 + '- dest_mac: %s' % self.verify_mac(interface['dest_mac'])
149                if interface.get('loopback_dest'):
150                    config_str += " # MAC OF LOOPBACK TO IT'S DUAL INTERFACE\n"
151                else:
152                    config_str += '\n'
153                config_str += ' '*8 + 'src_mac:  %s\n' % self.verify_mac(interface['src_mac'])
154            if index % 2:
155                config_str += '\n' # dual if barrier
156        if not self.ignore_numa:
157            config_str += '  platform:\n'
158            if len(self.interfaces_per_numa.keys()) == 1 and -1 in self.interfaces_per_numa: # VM, use any cores, 1 core per dual_if
159                lcores_pool = sorted([lcore for lcores in self.lcores_per_numa.values() for lcore in lcores])
160                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool.pop())
161                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool.pop(0)
162                lcores_per_dual_if = int(len(lcores_pool) / len(self.interfaces))
163                config_str += ' '*6 + 'dual_if:\n'
164                for i in range(0, len(self.interfaces), 2):
165                    lcores_for_this_dual_if = [str(lcores_pool.pop(0)) for _ in range(lcores_per_dual_if)]
166                    config_str += ' '*8 + '- socket: 0\n'
167                    config_str += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
168            else:
169                # we will take common minimum among all NUMAs, to satisfy all
170                lcores_per_dual_if = 99
171                extra_lcores = 1 if self.has_zero_lcore else 2
172                # worst case 3 iterations, to ensure master and "rx" have cores left
173                while (lcores_per_dual_if * sum(self.interfaces_per_numa.values()) / 2) + extra_lcores > sum([len(lcores) for lcores in self.lcores_per_numa.values()]):
174                    lcores_per_dual_if -= 1
175                    for numa, cores in self.lcores_per_numa.items():
176                        if not self.interfaces_per_numa[numa]:
177                            continue
178                        lcores_per_dual_if = min(lcores_per_dual_if, int(2 * len(cores) / self.interfaces_per_numa[numa]))
179                lcores_pool = copy.deepcopy(self.lcores_per_numa)
180                # first, allocate lcores for dual_if section
181                dual_if_section = ' '*6 + 'dual_if:\n'
182                for i in range(0, len(self.interfaces), 2):
183                    numa = self.interfaces[i]['NUMA']
184                    dual_if_section += ' '*8 + '- socket: %s\n' % numa
185                    lcores_for_this_dual_if = [str(lcores_pool[numa].pop(0)) for _ in range(lcores_per_dual_if)]
186                    if not lcores_for_this_dual_if:
187                        raise DpdkSetup('Not enough cores at NUMA %s. This NUMA has %s processing units and %s interfaces.' % (numa, len(self.lcores_per_numa[numa]), self.interfaces_per_numa[numa]))
188                    dual_if_section += ' '*10 + 'threads: [%s]\n\n' % ','.join(lcores_for_this_dual_if)
189                # take the cores left to master and rx
190                lcores_pool_left = [lcore for lcores in lcores_pool.values() for lcore in lcores]
191                config_str += ' '*6 + 'master_thread_id: %s\n' % (0 if self.has_zero_lcore else lcores_pool_left.pop(0))
192                config_str += ' '*6 + 'latency_thread_id: %s\n' % lcores_pool_left.pop(0)
193                # add the dual_if section
194                config_str += dual_if_section
195
196        # verify config is correct YAML format
197        try:
198            yaml.safe_load(config_str)
199        except Exception as e:
200            raise DpdkSetup('Could not create correct yaml config.\nGenerated YAML:\n%s\nEncountered error:\n%s' % (config_str, e))
201
202        if print_config:
203            print(config_str)
204        if filename:
205            if os.path.exists(filename):
206                if not dpdk_nic_bind.confirm('File %s already exist, overwrite? (y/N)' % filename):
207                    print('Skipping.')
208                    return config_str
209            with open(filename, 'w') as f:
210                f.write(config_str)
211            print('Saved.')
212        return config_str
213
214
215class map_driver(object):
216    args=None;
217    cfg_file='/etc/trex_cfg.yaml'
218    parent_cfg = None
219    dump_interfaces = None
220
221class DpdkSetup(Exception):
222    pass
223
224class CIfMap:
225
226    def __init__(self, cfg_file):
227        self.m_cfg_file =cfg_file;
228        self.m_cfg_dict={};
229        self.m_devices={};
230
231    def dump_error (self,err):
232        s="""%s
233From this TRex version a configuration file must exist in /etc/ folder "
234The name of the configuration file should be /etc/trex_cfg.yaml "
235The minimum configuration file should include something like this
236- version       : 2 # version 2 of the configuration file
237  interfaces    : ["03:00.0","03:00.1","13:00.1","13:00.0"]  # list of the interfaces to bind run ./dpdk_nic_bind.py --status to see the list
238  port_limit      : 2 # number of ports to use valid is 2,4,6,8,10,12
239
240example of already bind devices
241
242$ ./dpdk_nic_bind.py --status
243
244Network devices using DPDK-compatible driver
245============================================
2460000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2470000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2480000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
2490000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
250
251Network devices using kernel driver
252===================================
2530000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*
254
255Other network devices
256=====================
257
258
259          """ % (err);
260        return s;
261
262
263    def raise_error  (self,err):
264        s= self.dump_error (err)
265        raise DpdkSetup(s)
266
267    def load_config_file (self):
268
269        fcfg=self.m_cfg_file
270
271        if not os.path.isfile(fcfg) :
272            self.raise_error ("There is no valid configuration file %s " % fcfg)
273
274        try:
275          stream = open(fcfg, 'r')
276          self.m_cfg_dict= yaml.safe_load(stream)
277        except Exception as e:
278          print(e);
279          raise e
280
281        stream.close();
282        cfg_dict = self.m_cfg_dict[0]
283        if 'version' not in cfg_dict:
284            self.raise_error ("Configuration file %s is old, should include version field\n" % fcfg )
285
286        if int(cfg_dict['version'])<2 :
287            self.raise_error ("Configuration file %s is old, should include version field with value greater than 2\n" % fcfg)
288
289        if 'interfaces' not in self.m_cfg_dict[0]:
290            self.raise_error ("Configuration file %s is old, should include interfaces field even number of elemets" % fcfg)
291
292        if_list=self.m_cfg_dict[0]['interfaces']
293        l=len(if_list);
294        if (l>20):
295            self.raise_error ("Configuration file %s should include interfaces field with maximum of number of elemets" % (fcfg,l))
296        if ((l % 2)==1):
297            self.raise_error ("Configuration file %s should include even number of interfaces " % (fcfg,l))
298        if 'port_limit' in cfg_dict and cfg_dict['port_limit'] > len(if_list):
299            self.raise_error ('Error: port_limit should not be higher than number of interfaces in config file: %s\n' % fcfg)
300
301
302    def do_bind_one (self,key,mellanox):
303        if mellanox:
304            drv="mlx5_core"
305        else:
306            drv="igb_uio"
307
308        cmd='%s dpdk_nic_bind.py --bind=%s %s ' % (sys.executable, drv,key)
309        print(cmd)
310        res=os.system(cmd);
311        if res!=0:
312            raise DpdkSetup('')
313
314
315    def pci_name_to_full_name (self,pci_name):
316          c='[0-9A-Fa-f]';
317          sp='[:]'
318          s_short=c+c+sp+c+c+'[.]'+c;
319          s_full=c+c+c+c+sp+s_short
320          re_full = re.compile(s_full)
321          re_short = re.compile(s_short)
322
323          if re_short.match(pci_name):
324              return '0000:'+pci_name
325
326          if re_full.match(pci_name):
327              return pci_name
328
329          err=" %s is not a valid pci address \n" %pci_name;
330          raise DpdkSetup(err)
331
332
333    def run_dpdk_lspci (self):
334        dpdk_nic_bind.get_nic_details()
335        self.m_devices= dpdk_nic_bind.devices
336
337    def do_run (self,only_check_all_mlx=False):
338        self.run_dpdk_lspci ()
339        if map_driver.dump_interfaces is None or (map_driver.dump_interfaces == [] and map_driver.parent_cfg):
340            self.load_config_file()
341            if_list=self.m_cfg_dict[0]['interfaces']
342        else:
343            if_list = map_driver.dump_interfaces
344            if not if_list:
345                for dev in self.m_devices.values():
346                    if dev.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
347                        if_list.append(dev['Slot'])
348
349        if_list = list(map(self.pci_name_to_full_name, if_list))
350
351
352        # check how many mellanox cards we have
353        Mellanox_cnt=0;
354        for key in if_list:
355            if key not in self.m_devices:
356                err=" %s does not exist " %key;
357                raise DpdkSetup(err)
358
359            if 'Vendor_str' not in self.m_devices[key]:
360                err=" %s does not have Vendor_str " %key;
361                raise DpdkSetup(err)
362
363            if self.m_devices[key]['Vendor_str'].find("Mellanox")>-1 :
364                Mellanox_cnt=Mellanox_cnt+1
365
366
367        if ((Mellanox_cnt>0) and (Mellanox_cnt!= len(if_list))):
368            err=" All driver should be from one vendor. you have at least one driver from Mellanox but not all ";
369            raise DpdkSetup(err)
370
371
372        if only_check_all_mlx:
373            if Mellanox_cnt >0:
374                exit(1);
375            else:
376                exit(0);
377
378        for key in if_list:
379            if key not in self.m_devices:
380                err=" %s does not exist " %key;
381                raise DpdkSetup(err)
382
383            if 'Driver_str' in self.m_devices[key]:
384                if self.m_devices[key]['Driver_str'] not in dpdk_nic_bind.dpdk_drivers :
385                    self.do_bind_one (key,(Mellanox_cnt>0))
386                    pass;
387            else:
388                self.do_bind_one (key,(Mellanox_cnt>0))
389                pass;
390
391        if if_list and map_driver.args.parent and dpdk_nic_bind.get_igb_uio_usage():
392            pid = dpdk_nic_bind.get_pid_using_pci(if_list)
393            if pid:
394                cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
395                print('Some or all of given interfaces are in use by following process:\npid: %s, cmd: %s' % (pid, cmdline))
396                if not dpdk_nic_bind.confirm('Ignore and proceed (y/N):'):
397                    sys.exit(1)
398            else:
399                print('WARNING: Some other program is using DPDK driver.\nIf it is TRex and you did not configure it for dual run, current command will fail.')
400
401    def do_return_to_linux(self):
402        if not self.m_devices:
403            self.run_dpdk_lspci()
404        dpdk_interfaces = []
405        for device in self.m_devices.values():
406            if device.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
407                dpdk_interfaces.append(device['Slot'])
408        if not dpdk_interfaces:
409            print('No DPDK bound interfaces.')
410            return
411        if dpdk_nic_bind.get_igb_uio_usage():
412            pid = dpdk_nic_bind.get_pid_using_pci(dpdk_interfaces)
413            if pid:
414                cmdline = dpdk_nic_bind.read_pid_cmdline(pid)
415                print('DPDK interfaces are in use. Unbinding them might cause following process to hang:\npid: %s, cmd: %s' % (pid, cmdline))
416                if not dpdk_nic_bind.confirm('Confirm (y/N):'):
417                    return
418        drivers_table = {
419            'rte_ixgbe_pmd': 'ixgbe',
420            'rte_igb_pmd': 'igb',
421            'rte_i40e_pmd': 'i40e',
422            'rte_em_pmd': 'e1000',
423            'rte_vmxnet3_pmd': 'vmxnet3',
424            'rte_virtio_pmd': 'virtio-pci',
425            'rte_enic_pmd': 'enic',
426        }
427        for pci, info in dpdk_nic_bind.get_info_from_trex(dpdk_interfaces).items():
428            if pci not in self.m_devices:
429                raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
430            dev = self.m_devices[pci]
431            if info['TRex_Driver'] not in drivers_table:
432                print('Got unknown driver %s, description: %s' % (info['TRex_Driver'], dev['Device_str']))
433            else:
434                print('Returning to Linux %s' % pci)
435                dpdk_nic_bind.bind_one(pci, drivers_table[info['TRex_Driver']], False)
436
437    def _get_cpu_topology(self):
438        cpu_topology_file = '/proc/cpuinfo'
439        # physical processor -> physical core -> logical processing units (threads)
440        cpu_topology = OrderedDict()
441        if not os.path.exists(cpu_topology_file):
442            raise DpdkSetup('File with CPU topology (%s) does not exist.' % cpu_topology_file)
443        with open(cpu_topology_file) as f:
444            for lcore in f.read().split('\n\n'):
445                if not lcore:
446                    continue
447                lcore_dict = OrderedDict()
448                for line in lcore.split('\n'):
449                    key, val = line.split(':', 1)
450                    lcore_dict[key.strip()] = val.strip()
451                if 'processor' not in lcore_dict:
452                    continue
453                numa = int(lcore_dict.get('physical id', -1))
454                if numa not in cpu_topology:
455                    cpu_topology[numa] = OrderedDict()
456                core = int(lcore_dict.get('core id', lcore_dict['processor']))
457                if core not in cpu_topology[numa]:
458                    cpu_topology[numa][core] = []
459                cpu_topology[numa][core].append(int(lcore_dict['processor']))
460        if not cpu_topology:
461            raise DpdkSetup('Could not determine CPU topology from %s' % cpu_topology_file)
462        return cpu_topology
463
464    # input: list of different descriptions of interfaces: index, pci, name etc.
465    # Binds to dpdk wanted interfaces, not bound to any driver.
466    # output: list of maps of devices in dpdk_* format (self.m_devices.values())
467    def _get_wanted_interfaces(self, input_interfaces, get_macs = True):
468        if type(input_interfaces) is not list:
469            raise DpdkSetup('type of input interfaces should be list')
470        if not len(input_interfaces):
471            raise DpdkSetup('Please specify interfaces to use in the config')
472        if len(input_interfaces) % 2:
473            raise DpdkSetup('Please specify even number of interfaces')
474        wanted_interfaces = []
475        sorted_pci = sorted(self.m_devices.keys())
476        for interface in input_interfaces:
477            dev = None
478            try:
479                interface = int(interface)
480                if interface < 0 or interface >= len(sorted_pci):
481                    raise DpdkSetup('Index of an interfaces should be in range 0:%s' % (len(sorted_pci) - 1))
482                dev = self.m_devices[sorted_pci[interface]]
483            except ValueError:
484                for d in self.m_devices.values():
485                    if interface in (d['Interface'], d['Slot'], d['Slot_str']):
486                        dev = d
487                        break
488            if not dev:
489                raise DpdkSetup('Could not find information about this interface: %s' % interface)
490            if dev in wanted_interfaces:
491                raise DpdkSetup('Interface %s is specified twice' % interface)
492            dev['Interface_argv'] = interface
493            wanted_interfaces.append(dev)
494
495        if get_macs:
496            unbound = []
497            dpdk_bound = []
498            for interface in wanted_interfaces:
499                if 'Driver_str' not in interface:
500                    unbound.append(interface['Slot'])
501                elif interface.get('Driver_str') in dpdk_nic_bind.dpdk_drivers:
502                    dpdk_bound.append(interface['Slot'])
503            if unbound or dpdk_bound:
504                for pci, info in dpdk_nic_bind.get_info_from_trex(unbound + dpdk_bound).items():
505                    if pci not in self.m_devices:
506                        raise DpdkSetup('Internal error: PCI %s is not found among devices' % pci)
507                    self.m_devices[pci].update(info)
508
509        return wanted_interfaces
510
511    def do_create(self):
512
513        ips = map_driver.args.ips
514        def_gws = map_driver.args.def_gws
515        dest_macs = map_driver.args.dest_macs
516        if map_driver.args.force_macs:
517            ip_config = False
518            if ips:
519                raise DpdkSetup("If using --force-macs, should not specify ips")
520            if def_gws:
521                raise DpdkSetup("If using --force-macs, should not specify default gateways")
522        elif ips:
523            ip_config = True
524            if not def_gws:
525                raise DpdkSetup("If specifying ips, must specify also def-gws")
526            if dest_macs:
527                raise DpdkSetup("If specifying ips, should not specify dest--macs")
528            if len(ips) != len(def_gws) or len(ips) != len(map_driver.args.create_interfaces):
529                raise DpdkSetup("Number of given IPs should equal number of given def-gws and number of interfaces")
530        else:
531            if dest_macs:
532                ip_config = False
533            else:
534                ip_config = True
535
536        # gather info about NICS from dpdk_nic_bind.py
537        if not self.m_devices:
538            self.run_dpdk_lspci()
539        wanted_interfaces = self._get_wanted_interfaces(map_driver.args.create_interfaces, get_macs = not ip_config)
540
541        for i, interface in enumerate(wanted_interfaces):
542            dual_index = i + 1 - (i % 2) * 2
543            if ip_config:
544                if isinstance(ips, list) and len(ips) > i:
545                    interface['ip'] = ips[i]
546                else:
547                    interface['ip'] = ".".join(list(str(i+1))*4)
548                if isinstance(def_gws, list) and len(def_gws) > i:
549                    interface['def_gw'] = def_gws[i]
550                else:
551                    interface['def_gw'] = ".".join(list(str(dual_index+1))*4)
552            else:
553                dual_if = wanted_interfaces[dual_index]
554                if 'MAC' not in interface:
555                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
556                if 'MAC' not in dual_if:
557                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % dual_if['Interface_argv'])
558                interface['src_mac'] = interface['MAC']
559                if isinstance(dest_macs, list) and len(dest_macs) > i:
560                    interface['dest_mac'] = dest_macs[i]
561                else:
562                    interface['dest_mac'] = dual_if['MAC']
563                    interface['loopback_dest'] = True
564
565        config = ConfigCreator(self._get_cpu_topology(), wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
566                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa,
567                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
568        config.create_config(filename = map_driver.args.o, print_config = map_driver.args.dump)
569
570    def do_interactive_create(self):
571        ignore_numa = False
572        cpu_topology = self._get_cpu_topology()
573        total_lcores = sum([len(lcores) for cores in cpu_topology.values() for lcores in cores.values()])
574        if total_lcores < 1:
575            raise DpdkSetup('Script could not determine number of cores of the system, exiting.')
576        elif total_lcores < 2:
577            if dpdk_nic_bind.confirm("You only have 1 core and can't run TRex at all. Ignore and continue? (y/N): "):
578                ignore_numa = True
579            else:
580                sys.exit(1)
581        elif total_lcores < 3:
582            if dpdk_nic_bind.confirm("You only have 2 cores and will be able to run only stateful without latency checks.\nIgnore and continue? (y/N): "):
583                ignore_numa = True
584            else:
585                sys.exit(1)
586
587        if map_driver.args.force_macs:
588            ip_based = False
589        elif dpdk_nic_bind.confirm("By default, IP based configuration file will be created. Do you want to use MAC based config? (y/N)"):
590            ip_based = False
591        else:
592            ip_based = True
593            ip_addr_digit = 1
594
595        if not self.m_devices:
596            self.run_dpdk_lspci()
597        dpdk_nic_bind.show_table(get_macs = not ip_based)
598        print('Please choose even number of interfaces from the list above, either by ID , PCI or Linux IF')
599        print('Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.')
600        print('Stateless can be in any order.')
601        numa = None
602        for dev in self.m_devices.values():
603            if numa is None:
604                numa = dev['NUMA']
605            elif numa != dev['NUMA']:
606                print('For performance, try to choose each pair of interfaces to be on the same NUMA.')
607                break
608        while True:
609            try:
610                input = dpdk_nic_bind.read_line('Enter list of interfaces separated by space (for example: 1 3) : ')
611                create_interfaces = input.replace(',', ' ').replace(';', ' ').split()
612                wanted_interfaces = self._get_wanted_interfaces(create_interfaces)
613                ConfigCreator._verify_devices_same_type(wanted_interfaces)
614            except Exception as e:
615                print(e)
616                continue
617            break
618        print('')
619
620        for interface in wanted_interfaces:
621            if interface['Active']:
622                print('Interface %s is active. Using it by TRex might close ssh connections etc.' % interface['Interface_argv'])
623                if not dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
624                    sys.exit(1)
625
626        for i, interface in enumerate(wanted_interfaces):
627            if not ip_based:
628                if 'MAC' not in interface:
629                    raise DpdkSetup('Could not determine MAC of interface: %s. Please verify with -t flag.' % interface['Interface_argv'])
630                interface['src_mac'] = interface['MAC']
631            dual_index = i + 1 - (i % 2) * 2
632            dual_int = wanted_interfaces[dual_index]
633            if not ignore_numa and interface['NUMA'] != dual_int['NUMA']:
634                print('NUMA is different at pair of interfaces: %s and %s. It will reduce performance.' % (interface['Interface_argv'], dual_int['Interface_argv']))
635                if dpdk_nic_bind.confirm('Ignore and continue? (y/N): '):
636                    ignore_numa = True
637                    print('')
638                else:
639                    return
640
641            if ip_based:
642                if ip_addr_digit % 2 == 0:
643                    dual_ip_digit = ip_addr_digit - 1
644                else:
645                    dual_ip_digit = ip_addr_digit + 1
646                ip = ".".join(list(str(ip_addr_digit))*4)
647                def_gw= ".".join(list(str(dual_ip_digit))*4)
648                ip_addr_digit += 1
649
650                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
651                if dpdk_nic_bind.confirm("Putting IP %s, default gw %s Change it?(y/N)." % (ip, def_gw)):
652                    while True:
653                        ip = dpdk_nic_bind.read_line('Please enter IP address for interface %s: ' % interface['Interface_argv'])
654                        if not ConfigCreator._verify_ip(ip):
655                            print ("Bad IP address format")
656                        else:
657                            break
658                    while True:
659                        def_gw = dpdk_nic_bind.read_line('Please enter default gateway for interface %s: ' % interface['Interface_argv'])
660                        if not ConfigCreator._verify_ip(def_gw):
661                            print ("Bad IP address format")
662                        else:
663                            break
664                wanted_interfaces[i]['ip'] = ip
665                wanted_interfaces[i]['def_gw'] = def_gw
666            else:
667                dest_mac = dual_int['MAC']
668                loopback_dest = True
669                print("For interface %s, assuming loopback to it's dual interface %s." % (interface['Interface_argv'], dual_int['Interface_argv']))
670                if dpdk_nic_bind.confirm("Destination MAC is %s. Change it to MAC of DUT? (y/N)." % dest_mac):
671                    while True:
672                        input_mac = dpdk_nic_bind.read_line('Please enter new destination MAC of interface %s: ' % interface['Interface_argv'])
673                        try:
674                            if input_mac:
675                                ConfigCreator._convert_mac(input_mac) # verify format
676                                dest_mac = input_mac
677                                loopback_dest = False
678                            else:
679                                print('Leaving the loopback MAC.')
680                        except Exception as e:
681                            print(e)
682                            continue
683                        break
684                wanted_interfaces[i]['dest_mac'] = dest_mac
685                wanted_interfaces[i]['loopback_dest'] = loopback_dest
686
687        config = ConfigCreator(cpu_topology, wanted_interfaces, include_lcores = map_driver.args.create_include, exclude_lcores = map_driver.args.create_exclude,
688                               only_first_thread = map_driver.args.no_ht, ignore_numa = map_driver.args.ignore_numa or ignore_numa,
689                               prefix = map_driver.args.prefix, zmq_rpc_port = map_driver.args.zmq_rpc_port, zmq_pub_port = map_driver.args.zmq_pub_port)
690        if dpdk_nic_bind.confirm('Print preview of generated config? (Y/n)', default = True):
691            config.create_config(print_config = True)
692        if dpdk_nic_bind.confirm('Save the config to file? (Y/n)', default = True):
693            print('Default filename is /etc/trex_cfg.yaml')
694            filename = dpdk_nic_bind.read_line('Press ENTER to confirm or enter new file: ')
695            if not filename:
696                filename = '/etc/trex_cfg.yaml'
697            config.create_config(filename = filename)
698
699
700def parse_parent_cfg (parent_cfg):
701    parent_parser = argparse.ArgumentParser(add_help = False)
702    parent_parser.add_argument('-?', '-h', '--help', dest = 'help', action = 'store_true')
703    parent_parser.add_argument('--cfg', default='')
704    parent_parser.add_argument('--dump-interfaces', nargs='*', default=None)
705    args, _ = parent_parser.parse_known_args(shlex.split(parent_cfg))
706    if args.help:
707        sys.exit(0)
708    return (args.cfg, args.dump_interfaces)
709
710def process_options ():
711    parser = argparse.ArgumentParser(usage="""
712
713Examples:
714---------
715
716To return to Linux the DPDK bound interfaces (for ifconfig etc.)
717  sudo ./dpdk_set_ports.py -l
718
719To create TRex config file using interactive mode
720  sudo ./dpdk_set_ports.py -i
721
722To create a default config file (example1)
723  sudo ./dpdk_setup_ports.py -c 02:00.0 02:00.1 -o /etc/trex_cfg.yaml
724
725To create a default config file (example2)
726  sudo ./dpdk_setup_ports.py -c eth1 eth2 --dest-macs 11:11:11:11:11:11 22:22:22:22:22:22 --dump
727
728To show interfaces status
729  sudo ./dpdk_set_ports.py -s
730
731To see more detailed info on interfaces (table):
732  sudo ./dpdk_set_ports.py -t
733
734    """,
735    description=" unbind dpdk interfaces ",
736    epilog=" written by hhaim");
737
738    parser.add_argument("-l", "--linux", action='store_true',
739                      help=""" Return all DPDK interfaces to Linux driver """,
740     )
741
742    parser.add_argument("--cfg",
743                      help=""" configuration file name  """,
744     )
745
746    parser.add_argument("--parent",
747                      help=argparse.SUPPRESS
748     )
749
750    parser.add_argument('--dump-pci-description', help=argparse.SUPPRESS, dest='dump_pci_desc', action='store_true')
751
752    parser.add_argument("-i", "--interactive", action='store_true',
753                      help=""" Create TRex config in interactive mode """,
754     )
755
756    parser.add_argument("-c", "--create", nargs='*', default=None, dest='create_interfaces', metavar='<interface>',
757                      help="""Try to create a configuration file by specifying needed interfaces by PCI address or Linux names: eth1 etc.""",
758     )
759
760    parser.add_argument("--ci", "--cores-include", nargs='*', default=[], dest='create_include', metavar='<cores>',
761                      help="""White list of cores to use. Make sure there is enough for each NUMA.""",
762     )
763
764    parser.add_argument("--ce", "--cores-exclude", nargs='*', default=[], dest='create_exclude', metavar='<cores>',
765                      help="""Black list of cores to exclude. Make sure there will be enough for each NUMA.""",
766     )
767
768    parser.add_argument("--dump", default=False, action='store_true',
769                      help="""Dump created config to screen.""",
770     )
771
772    parser.add_argument("--no-ht", default=False, dest='no_ht', action='store_true',
773                      help="""Use only one thread of each Core in created config yaml (No Hyper-Threading).""",
774     )
775
776    parser.add_argument("--dest-macs", nargs='*', default=[], action='store',
777                      help="""Destination MACs to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
778     )
779
780    parser.add_argument("--force-macs", default=False, action='store_true',
781                      help="""Use MACs in created config file.""",
782     )
783
784    parser.add_argument("--ips", nargs='*', default=[], action='store',
785                      help="""IP addresses to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
786     )
787
788    parser.add_argument("--def-gws", nargs='*', default=[], action='store',
789                      help="""Default gateways to be used in created yaml file. Without them, will assume loopback (0<->1, 2<->3 etc.)""",
790     )
791
792    parser.add_argument("-o", default=None, action='store', metavar='PATH',
793                      help="""Output the config to this file.""",
794     )
795
796    parser.add_argument("--prefix", default=None, action='store',
797                      help="""Advanced option: prefix to be used in TRex config in case of parallel instances.""",
798     )
799
800    parser.add_argument("--zmq-pub-port", default=None, action='store',
801                      help="""Advanced option: ZMQ Publisher port to be used in TRex config in case of parallel instances.""",
802     )
803
804    parser.add_argument("--zmq-rpc-port", default=None, action='store',
805                      help="""Advanced option: ZMQ RPC port to be used in TRex config in case of parallel instances.""",
806     )
807
808    parser.add_argument("--ignore-numa", default=False, action='store_true',
809                      help="""Advanced option: Ignore NUMAs for config creation. Use this option only if you have to, as it will reduce performance.""",
810     )
811
812    parser.add_argument("-s", "--show", action='store_true',
813                      help=""" show the status """,
814     )
815
816    parser.add_argument("-t", "--table", action='store_true',
817                      help=""" show table with NICs info """,
818     )
819
820    parser.add_argument('--version', action='version',
821                        version="0.2" )
822
823    map_driver.args = parser.parse_args();
824    if map_driver.args.parent :
825        map_driver.parent_cfg, map_driver.dump_interfaces = parse_parent_cfg (map_driver.args.parent)
826        if map_driver.parent_cfg != '':
827            map_driver.cfg_file = map_driver.parent_cfg;
828    if  map_driver.args.cfg :
829        map_driver.cfg_file = map_driver.args.cfg;
830
831def main ():
832    try:
833        if getpass.getuser() != 'root':
834            raise DpdkSetup('Please run this program as root/with sudo')
835
836        process_options ()
837
838        if map_driver.args.show:
839            dpdk_nic_bind.show_status()
840            return
841
842        if map_driver.args.table:
843            dpdk_nic_bind.show_table()
844            return
845
846        if map_driver.args.dump_pci_desc:
847            dpdk_nic_bind.dump_pci_description()
848            return
849
850        obj =CIfMap(map_driver.cfg_file);
851
852        if map_driver.args.create_interfaces is not None:
853            obj.do_create();
854        elif map_driver.args.interactive:
855            obj.do_interactive_create();
856        elif map_driver.args.linux:
857            obj.do_return_to_linux();
858        else:
859            obj.do_run();
860        print('')
861    except DpdkSetup as e:
862        print(e)
863        exit(-1)
864
865if __name__ == '__main__':
866    main()
867
868