ws_main.py revision af781231
1#!/usr/bin/env python
2# encoding: utf-8
3#
4# Hanoh Haim
5# Cisco Systems, Inc.
6
7VERSION='0.0.1'
8APPNAME='cxx_test'
9import os;
10import shutil;
11import copy;
12import re
13import uuid
14import subprocess
15
16# these variables are mandatory ('/' are converted automatically)
17top = '../'
18out = 'build_dpdk'
19
20
21b_path ="./build/linux_dpdk/"
22
23C_VER_FILE      = "version.c"
24H_VER_FILE      = "version.h"
25
26BUILD_NUM_FILE  = "../VERSION"
27USERS_ALLOWED_TO_RELEASE = ['hhaim']
28
29
30#######################################
31# utility for group source code
32###################################
33
34class SrcGroup:
35    ' group of source by directory '
36
37    def __init__(self,dir,src_list):
38      self.dir = dir;
39      self.src_list = src_list;
40      self.group_list = None;
41      assert (type(src_list)==list)
42      assert (type(dir)==str)
43
44
45
46    def file_list (self,top):
47        ' return  the long list of the files '
48        res=''
49        for file in self.src_list:
50            res= res + top+'/'+self.dir+'/'+file+'  ';
51
52        return res;
53
54    def __str__ (self):
55        return (self.file_list(''));
56
57    def __repr__ (self):
58        return (self.file_list(''));
59
60
61
62class SrcGroups:
63    ' group of source groups '
64
65    def __init__(self,list_group):
66      self.list_group = list_group;
67      assert (type(list_group)==list)
68
69
70    def file_list (self,top):
71          ' return  the long list of the files '
72          res=''
73          for o in self.list_group:
74              res += o.file_list(top);
75          return res;
76
77    def __str__ (self):
78          return (self.file_list(''));
79
80    def __repr__ (self):
81          return (self.file_list(''));
82
83
84def options(opt):
85    opt.load('compiler_cxx')
86    opt.load('compiler_c')
87    opt.add_option('--pkg-dir', '--pkg_dir', dest='pkg_dir', default=False, action='store', help="Destination folder for 'pkg' option.")
88    opt.add_option('--pkg-file', '--pkg_file', dest='pkg_file', default=False, action='store', help="Destination filename for 'pkg' option.")
89    opt.add_option('--publish-commit', '--publish_commit', dest='publish_commit', default=False, action='store', help="Specify commit id for 'publish_both' option (Please make sure it's good!)")
90
91def configure(conf):
92    conf.load('g++')
93    conf.load('gcc')
94
95def getstatusoutput(cmd):
96    """    Return (status, output) of executing cmd in a shell. Taken from Python3 subprocess.getstatusoutput"""
97    try:
98        data = subprocess.check_output(cmd, shell=True, universal_newlines=True, stderr=subprocess.STDOUT)
99        status = 0
100    except subprocess.CalledProcessError as ex:
101        data = ex.output
102        status = ex.returncode
103    if data[-1:] == '\n':
104        data = data[:-1]
105    return status, data
106
107main_src = SrcGroup(dir='src',
108        src_list=[
109             'utl_term_io.cpp',
110             'global_io_mode.cpp',
111             'main_dpdk.cpp',
112             'trex_watchdog.cpp',
113             'trex_client_config.cpp',
114             'debug.cpp',
115             'flow_stat.cpp',
116             'flow_stat_parser.cpp',
117             'bp_sim.cpp',
118             'latency.cpp',
119             'platform_cfg.cpp',
120             'pre_test.cpp',
121             'tuple_gen.cpp',
122             'rx_check.cpp',
123             'rx_check_header.cpp',
124             'test_pkt_gen.cpp',
125             'timer_wheel_pq.cpp',
126             'time_histogram.cpp',
127             'os_time.cpp',
128             'utl_cpuu.cpp',
129             'utl_json.cpp',
130             'utl_yaml.cpp',
131             'nat_check.cpp',
132             'nat_check_flow_table.cpp',
133             'msg_manager.cpp',
134             'publisher/trex_publisher.cpp',
135             'pal/linux_dpdk/pal_utl.cpp',
136             'pal/linux_dpdk/mbuf.cpp',
137             'pal/common/common_mbuf.cpp'
138             ]);
139
140cmn_src = SrcGroup(dir='src/common',
141    src_list=[
142        'basic_utils.cpp',
143        'captureFile.cpp',
144        'erf.cpp',
145        'pcap.cpp',
146        'base64.cpp'
147        ]);
148
149net_src = SrcGroup(dir='src/common/Network/Packet',
150        src_list=[
151           'CPktCmn.cpp',
152           'EthernetHeader.cpp',
153           'IPHeader.cpp',
154           'IPv6Header.cpp',
155           'TCPHeader.cpp',
156           'TCPOptions.cpp',
157           'UDPHeader.cpp',
158           'MacAddress.cpp',
159           'VLANHeader.cpp']);
160
161# JSON package
162json_src = SrcGroup(dir='external_libs/json',
163        src_list=[
164            'jsoncpp.cpp'
165           ])
166
167# RPC code
168rpc_server_src = SrcGroup(dir='src/rpc-server/',
169                          src_list=[
170                              'trex_rpc_server.cpp',
171                              'trex_rpc_req_resp_server.cpp',
172                              'trex_rpc_async_server.cpp',
173                              'trex_rpc_jsonrpc_v2_parser.cpp',
174                              'trex_rpc_cmds_table.cpp',
175                              'trex_rpc_cmd.cpp',
176                              'trex_rpc_zip.cpp',
177
178                              'commands/trex_rpc_cmd_test.cpp',
179                              'commands/trex_rpc_cmd_general.cpp',
180                              'commands/trex_rpc_cmd_stream.cpp',
181
182                          ])
183
184
185ef_src = SrcGroup(dir='src/common',
186    src_list=[
187        'ef/efence.cpp',
188        'ef/page.cpp',
189        'ef/print.cpp'
190        ]);
191
192
193# stateless code
194stateless_src = SrcGroup(dir='src/stateless/',
195                          src_list=['cp/trex_stream.cpp',
196                                    'cp/trex_stream_vm.cpp',
197                                    'cp/trex_stateless.cpp',
198                                    'cp/trex_stateless_port.cpp',
199                                    'cp/trex_streams_compiler.cpp',
200                                    'cp/trex_vm_splitter.cpp',
201                                    'cp/trex_dp_port_events.cpp',
202                                    'dp/trex_stateless_dp_core.cpp',
203                                    'messaging/trex_stateless_messaging.cpp',
204                                    'rx/trex_stateless_rx_core.cpp'
205                                    ])
206# JSON package
207json_src = SrcGroup(dir='external_libs/json',
208                    src_list=[
209                        'jsoncpp.cpp'
210                        ])
211
212yaml_src = SrcGroup(dir='external_libs/yaml-cpp/src/',
213        src_list=[
214            'aliasmanager.cpp',
215            'binary.cpp',
216            'contrib/graphbuilder.cpp',
217            'contrib/graphbuilderadapter.cpp',
218            'conversion.cpp',
219            'directives.cpp',
220            'emitfromevents.cpp',
221            'emitter.cpp',
222            'emitterstate.cpp',
223            'emitterutils.cpp',
224            'exp.cpp',
225            'iterator.cpp',
226            'node.cpp',
227            'nodebuilder.cpp',
228            'nodeownership.cpp',
229            'null.cpp',
230            'ostream.cpp',
231            'parser.cpp',
232            'regex.cpp',
233            'scanner.cpp',
234            'scanscalar.cpp',
235            'scantag.cpp',
236            'scantoken.cpp',
237            'simplekey.cpp',
238            'singledocparser.cpp',
239            'stream.cpp',
240            'tag.cpp']);
241
242
243version_src = SrcGroup(
244    dir='linux_dpdk',
245    src_list=[
246        'version.c',
247    ])
248
249
250dpdk_src = SrcGroup(dir='src/dpdk/',
251                src_list=[
252                 '../dpdk_funcs.c',
253                 'drivers/net/af_packet/rte_eth_af_packet.c',
254                 'drivers/net/cxgbe/base/t4_hw.c',
255                 'drivers/net/cxgbe/cxgbe_ethdev.c',
256                 'drivers/net/cxgbe/cxgbe_main.c',
257                 'drivers/net/cxgbe/sge.c',
258                 'drivers/net/e1000/base/e1000_80003es2lan.c',
259                 'drivers/net/e1000/base/e1000_82540.c',
260                 'drivers/net/e1000/base/e1000_82541.c',
261                 'drivers/net/e1000/base/e1000_82542.c',
262                 'drivers/net/e1000/base/e1000_82543.c',
263                 'drivers/net/e1000/base/e1000_82571.c',
264                 'drivers/net/e1000/base/e1000_82575.c',
265                 'drivers/net/e1000/base/e1000_api.c',
266                 'drivers/net/e1000/base/e1000_i210.c',
267                 'drivers/net/e1000/base/e1000_ich8lan.c',
268                 'drivers/net/e1000/base/e1000_mac.c',
269                 'drivers/net/e1000/base/e1000_manage.c',
270                 'drivers/net/e1000/base/e1000_mbx.c',
271                 'drivers/net/e1000/base/e1000_nvm.c',
272                 'drivers/net/e1000/base/e1000_osdep.c',
273                 'drivers/net/e1000/base/e1000_phy.c',
274                 'drivers/net/e1000/base/e1000_vf.c',
275                 'drivers/net/e1000/em_ethdev.c',
276                 'drivers/net/e1000/em_rxtx.c',
277                 'drivers/net/e1000/igb_ethdev.c',
278                 'drivers/net/e1000/igb_pf.c',
279                 'drivers/net/e1000/igb_rxtx.c',
280                 'drivers/net/enic/base/vnic_cq.c',
281                 'drivers/net/enic/base/vnic_dev.c',
282                 'drivers/net/enic/base/vnic_intr.c',
283                 'drivers/net/enic/base/vnic_rq.c',
284                 'drivers/net/enic/base/vnic_rss.c',
285                 'drivers/net/enic/base/vnic_wq.c',
286                 'drivers/net/enic/enic_clsf.c',
287                 'drivers/net/enic/enic_ethdev.c',
288                 'drivers/net/enic/enic_main.c',
289                 'drivers/net/enic/enic_res.c',
290                 'drivers/net/enic/enic_rxtx.c',
291                 'drivers/net/fm10k/base/fm10k_api.c',
292                 'drivers/net/fm10k/base/fm10k_common.c',
293                 'drivers/net/fm10k/base/fm10k_mbx.c',
294                 'drivers/net/fm10k/base/fm10k_pf.c',
295                 'drivers/net/fm10k/base/fm10k_tlv.c',
296                 'drivers/net/fm10k/base/fm10k_vf.c',
297                 'drivers/net/fm10k/fm10k_ethdev.c',
298                 'drivers/net/fm10k/fm10k_rxtx.c',
299                 'drivers/net/fm10k/fm10k_rxtx_vec.c',
300                 'drivers/net/ixgbe/base/ixgbe_82598.c',
301                 'drivers/net/ixgbe/base/ixgbe_82599.c',
302                 'drivers/net/ixgbe/base/ixgbe_api.c',
303                 'drivers/net/ixgbe/base/ixgbe_common.c',
304                 'drivers/net/ixgbe/base/ixgbe_dcb.c',
305                 'drivers/net/ixgbe/base/ixgbe_dcb_82598.c',
306                 'drivers/net/ixgbe/base/ixgbe_dcb_82599.c',
307                 'drivers/net/ixgbe/base/ixgbe_mbx.c',
308                 'drivers/net/ixgbe/base/ixgbe_phy.c',
309                 'drivers/net/ixgbe/base/ixgbe_vf.c',
310                 'drivers/net/ixgbe/base/ixgbe_x540.c',
311                 'drivers/net/ixgbe/base/ixgbe_x550.c',
312                 'drivers/net/ixgbe/ixgbe_ethdev.c',
313                 'drivers/net/ixgbe/ixgbe_fdir.c',
314                 'drivers/net/ixgbe/ixgbe_pf.c',
315                 'drivers/net/ixgbe/ixgbe_rxtx.c',
316                 'drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c',
317                 'drivers/net/i40e/base/i40e_adminq.c',
318                 'drivers/net/i40e/base/i40e_common.c',
319                 'drivers/net/i40e/base/i40e_dcb.c',
320                 'drivers/net/i40e/base/i40e_diag.c',
321                 'drivers/net/i40e/base/i40e_hmc.c',
322                 'drivers/net/i40e/base/i40e_lan_hmc.c',
323                 'drivers/net/i40e/base/i40e_nvm.c',
324#                 'drivers/net/i40e/i40e_ethdev_vf.c',
325                 'drivers/net/i40e/i40e_pf.c',
326                 'drivers/net/i40e/i40e_rxtx.c',
327                 'drivers/net/i40e/i40e_rxtx_vec.c',
328                 'drivers/net/i40e/i40e_fdir.c',
329                 'drivers/net/i40e/i40e_ethdev.c',
330                 'drivers/net/null/rte_eth_null.c',
331                 'drivers/net/ring/rte_eth_ring.c',
332                 'drivers/net/virtio/virtio_ethdev.c',
333                 'drivers/net/virtio/virtio_pci.c',
334                 'drivers/net/virtio/virtio_rxtx.c',
335                 'drivers/net/virtio/virtio_rxtx_simple.c',
336                 'drivers/net/virtio/virtqueue.c',
337                 '/drivers/net/vmxnet3/vmxnet3_ethdev.c',
338                 '/drivers/net/vmxnet3/vmxnet3_rxtx.c',
339                 'lib/librte_cfgfile/rte_cfgfile.c',
340                 'lib/librte_eal/common/arch/x86/rte_cpuflags.c',
341                 'lib/librte_eal/common/arch/x86/rte_spinlock.c',
342                 'lib/librte_eal/common/eal_common_cpuflags.c',
343                 'lib/librte_eal/common/eal_common_dev.c',
344                 'lib/librte_eal/common/eal_common_devargs.c',
345                 'lib/librte_eal/common/eal_common_errno.c',
346                 'lib/librte_eal/common/eal_common_hexdump.c',
347                 'lib/librte_eal/common/eal_common_launch.c',
348                 'lib/librte_eal/common/eal_common_lcore.c',
349                 'lib/librte_eal/common/eal_common_log.c',
350                 'lib/librte_eal/common/eal_common_memory.c',
351                 'lib/librte_eal/common/eal_common_memzone.c',
352                 'lib/librte_eal/common/eal_common_options.c',
353                 'lib/librte_eal/common/eal_common_pci.c',
354                 'lib/librte_eal/common/eal_common_pci_uio.c',
355                 'lib/librte_eal/common/eal_common_string_fns.c',
356                 'lib/librte_eal/common/eal_common_tailqs.c',
357                 'lib/librte_eal/common/eal_common_thread.c',
358                 'lib/librte_eal/common/eal_common_timer.c',
359                 'lib/librte_eal/common/malloc_elem.c',
360                 'lib/librte_eal/common/malloc_heap.c',
361                 'lib/librte_eal/common/rte_keepalive.c',
362                 'lib/librte_eal/common/rte_malloc.c',
363                 'lib/librte_eal/linuxapp/eal/eal.c',
364                 'lib/librte_eal/linuxapp/eal/eal_alarm.c',
365                 'lib/librte_eal/linuxapp/eal/eal_debug.c',
366                 'lib/librte_eal/linuxapp/eal/eal_hugepage_info.c',
367                 'lib/librte_eal/linuxapp/eal/eal_interrupts.c',
368                 'lib/librte_eal/linuxapp/eal/eal_ivshmem.c',
369                 'lib/librte_eal/linuxapp/eal/eal_lcore.c',
370                 'lib/librte_eal/linuxapp/eal/eal_log.c',
371                 'lib/librte_eal/linuxapp/eal/eal_memory.c',
372                 'lib/librte_eal/linuxapp/eal/eal_pci.c',
373                 'lib/librte_eal/linuxapp/eal/eal_pci_uio.c',
374                 'lib/librte_eal/linuxapp/eal/eal_pci_vfio.c',
375                 'lib/librte_eal/linuxapp/eal/eal_thread.c',
376                 'lib/librte_eal/linuxapp/eal/eal_timer.c',
377                 'lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c',
378                 'lib/librte_eal/linuxapp/eal/eal_vfio.c',
379                 'lib/librte_ether/rte_ethdev.c',
380                 'lib/librte_hash/rte_cuckoo_hash.c',
381                 'lib/librte_kvargs/rte_kvargs.c',
382                 'lib/librte_mbuf/rte_mbuf.c',
383                 'lib/librte_mempool/rte_mempool.c',
384                 'lib/librte_mempool/rte_mempool_ops.c',
385                 'lib/librte_mempool/rte_mempool_ring.c',
386                 'lib/librte_pipeline/rte_pipeline.c',
387                 'lib/librte_ring/rte_ring.c',
388            ]);
389
390bp_dpdk =SrcGroups([
391                dpdk_src
392                ]);
393
394# this is the library dp going to falcon (and maybe other platforms)
395bp =SrcGroups([
396                main_src,
397                cmn_src ,
398                net_src ,
399                yaml_src,
400                rpc_server_src,
401                json_src,
402                stateless_src,
403                version_src
404                ]);
405
406l2fwd_main_src = SrcGroup(dir='src',
407        src_list=[
408             'l2fwd/main.c'
409             ]);
410
411
412l2fwd =SrcGroups([
413                l2fwd_main_src]);
414
415
416# common flags for both new and old configurations
417common_flags = ['-DWIN_UCODE_SIM',
418                '-D_BYTE_ORDER',
419                '-D_LITTLE_ENDIAN',
420                '-DLINUX',
421                '-g',
422                '-Wno-format',
423                '-Wno-deprecated-declarations',
424                '-DRTE_DPDK',
425                '-D__STDC_LIMIT_MACROS',
426                '-D__STDC_FORMAT_MACROS',
427                '-include','../src/pal/linux_dpdk/dpdk0716/rte_config.h'
428               ]
429
430common_flags_new = common_flags + [
431                    '-march=native',
432                    '-DRTE_MACHINE_CPUFLAG_SSE',
433                    '-DRTE_MACHINE_CPUFLAG_SSE2',
434                    '-DRTE_MACHINE_CPUFLAG_SSE3',
435                    '-DRTE_MACHINE_CPUFLAG_SSSE3',
436                    '-DRTE_MACHINE_CPUFLAG_SSE4_1',
437                    '-DRTE_MACHINE_CPUFLAG_SSE4_2',
438                    '-DRTE_MACHINE_CPUFLAG_AES',
439                    '-DRTE_MACHINE_CPUFLAG_PCLMULQDQ',
440                    '-DRTE_MACHINE_CPUFLAG_AVX',
441                    '-DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE3,RTE_CPUFLAG_SSE,RTE_CPUFLAG_SSE2,RTE_CPUFLAG_SSSE3,RTE_CPUFLAG_SSE4_1,RTE_CPUFLAG_SSE4_2,RTE_CPUFLAG_AES,RTE_CPUFLAG_PCLMULQDQ,RTE_CPUFLAG_AVX',
442                   ]
443
444common_flags_old = common_flags + [
445                      '-march=corei7',
446                      '-DUCS_210',
447                      '-mtune=generic',
448                      '-DRTE_MACHINE_CPUFLAG_SSE',
449                      '-DRTE_COMPILE_TIME_CPUFLAGS=RTE_CPUFLAG_SSE',
450                      ];
451
452
453
454
455includes_path =''' ../src/pal/linux_dpdk/
456                   ../src/pal/common/
457                   ../src/
458
459                   ../src/rpc-server/
460                   ../src/stateless/cp/
461                   ../src/stateless/dp/
462                   ../src/stateless/rx/
463                   ../src/stateless/messaging/
464
465                   ../external_libs/yaml-cpp/include/
466                   ../external_libs/zmq/include/
467                   ../external_libs/json/
468
469../src/dpdk/drivers/net/af_packet/
470../src/dpdk/drivers/net/bnx2x/
471../src/dpdk/drivers/net/bonding/
472../src/dpdk/drivers/net/cxgbe/
473../src/dpdk/drivers/net/cxgbe/base/
474../src/dpdk/drivers/net/e1000/
475../src/dpdk/drivers/net/e1000/base/
476../src/dpdk/drivers/net/fm10k/
477../src/dpdk/drivers/net/fm10k/base/
478../src/dpdk/drivers/net/i40e/
479../src/dpdk/drivers/net/i40e/base/
480../src/dpdk/drivers/net/ixgbe/
481../src/dpdk/drivers/net/ixgbe/base/
482../src/dpdk/drivers/net/mlx4/
483../src/dpdk/drivers/net/mlx5/
484../src/dpdk/drivers/net/mpipe/
485../src/dpdk/drivers/net/null/
486../src/dpdk/drivers/net/pcap/
487../src/dpdk/drivers/net/ring/
488../src/dpdk/drivers/net/szedata2/
489../src/dpdk/drivers/net/virtio/
490../src/dpdk/drivers/net/xenvirt/
491../src/dpdk/lib/librte_acl/
492../src/dpdk/lib/librte_cfgfile/
493../src/dpdk/lib/librte_compat/
494../src/dpdk/lib/librte_distributor/
495../src/dpdk/lib/librte_eal/
496../src/dpdk/lib/librte_eal/common/
497../src/dpdk/lib/librte_eal/common/include/
498../src/dpdk/lib/librte_eal/common/include/arch/
499../src/dpdk/lib/librte_eal/common/include/arch/x86/
500../src/dpdk/lib/librte_eal/common/include/generic/
501../src/dpdk/lib/librte_eal/linuxapp/
502../src/dpdk/lib/librte_eal/linuxapp/eal/
503../src/dpdk/lib/librte_eal/linuxapp/eal/include/
504../src/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/
505../src/dpdk/lib/librte_eal/linuxapp/igb_uio/
506../src/dpdk/lib/librte_eal/linuxapp/xen_dom0/
507../src/dpdk/lib/librte_ether/
508../src/dpdk/lib/librte_hash/
509../src/dpdk/lib/librte_kvargs/
510../src/dpdk/lib/librte_mbuf/
511../src/dpdk/lib/librte_mempool/
512../src/dpdk/lib/librte_net/
513../src/dpdk/lib/librte_pipeline/
514../src/dpdk/lib/librte_ring/
515              ''';
516
517dpdk_includes_path =''' ../src/
518                        ../src/pal/linux_dpdk/
519                        ../src/pal/linux_dpdk/dpdk
520../src/dpdk/drivers/
521../src/dpdk/drivers/net/
522../src/dpdk/drivers/net/af_packet/
523../src/dpdk/drivers/net/bnx2x/
524../src/dpdk/drivers/net/bonding/
525../src/dpdk/drivers/net/cxgbe/
526../src/dpdk/drivers/net/cxgbe/base/
527../src/dpdk/drivers/net/e1000/
528../src/dpdk/drivers/net/e1000/base/
529../src/dpdk/drivers/net/enic/
530../src/dpdk/drivers/net/enic/base/
531../src/dpdk/drivers/net/fm10k/
532../src/dpdk/drivers/net/fm10k/base/
533../src/dpdk/drivers/net/i40e/
534../src/dpdk/drivers/net/i40e/base/
535../src/dpdk/drivers/net/ixgbe/
536../src/dpdk/drivers/net/ixgbe/base/
537../src/dpdk/drivers/net/mlx4/
538../src/dpdk/drivers/net/mlx5/
539../src/dpdk/drivers/net/mpipe/
540../src/dpdk/drivers/net/null/
541../src/dpdk/drivers/net/pcap/
542../src/dpdk/drivers/net/ring/
543../src/dpdk/drivers/net/virtio/
544../src/dpdk/drivers/net/vmxnet3/
545../src/dpdk/drivers/net/vmxnet3/base
546../src/dpdk/drivers/net/xenvirt/
547../src/dpdk/lib/
548../src/dpdk/lib/librte_acl/
549../src/dpdk/lib/librte_cfgfile/
550../src/dpdk/lib/librte_compat/
551../src/dpdk/lib/librte_distributor/
552../src/dpdk/lib/librte_eal/
553../src/dpdk/lib/librte_eal/common/
554../src/dpdk/lib/librte_eal/common/include/
555../src/dpdk/lib/librte_eal/common/include/arch/
556../src/dpdk/lib/librte_eal/common/include/arch/x86/
557../src/dpdk/lib/librte_eal/common/include/generic/
558../src/dpdk/lib/librte_eal/linuxapp/
559../src/dpdk/lib/librte_eal/linuxapp/eal/
560../src/dpdk/lib/librte_eal/linuxapp/eal/include/
561../src/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/
562../src/dpdk/lib/librte_eal/linuxapp/igb_uio/
563../src/dpdk/lib/librte_eal/linuxapp/xen_dom0/
564../src/dpdk/lib/librte_ether/
565../src/dpdk/lib/librte_hash/
566../src/dpdk/lib/librte_kvargs/
567../src/dpdk/lib/librte_mbuf/
568../src/dpdk/lib/librte_mempool/
569../src/dpdk/lib/librte_pipeline/
570../src/dpdk/lib/librte_ring/
571../src/dpdk/lib/librte_net/
572../src/dpdk/lib/librte_port/
573../src/dpdk/lib/librte_pipeline/
574../src/dpdk/lib/librte_table/
575../src/dpdk/
576''';
577
578
579DPDK_FLAGS=['-D_GNU_SOURCE', '-DPF_DRIVER', '-DX722_SUPPORT', '-DX722_A0_SUPPORT', '-DVF_DRIVER', '-DINTEGRATED_VF'];
580
581client_external_libs = [
582        'enum34-1.0.4',
583        'jsonrpclib-pelix-0.2.5',
584        'pyyaml-3.11',
585        'pyzmq-14.5.0',
586        'scapy-2.3.1',
587        'texttable-0.8.4',
588        ]
589
590
591RELEASE_    = "release"
592DEBUG_      = "debug"
593PLATFORM_64 = "64"
594PLATFORM_32 = "32"
595
596
597class build_option:
598
599    def __init__(self,platform,debug_mode,is_pie):
600      self.mode     = debug_mode;   ##debug,release
601      self.platform = platform; #['32','64']
602      self.is_pie = is_pie
603
604    def __str__(self):
605       s=self.mode+","+self.platform;
606       return (s);
607
608    def lib_name(self,lib_name_p,full_path):
609        if full_path:
610            return b_path+lib_name_p;
611        else:
612            return lib_name_p;
613    #private functions
614    def toLib (self,name,full_path = True):
615        lib_n = "lib"+name+".a";
616        return (self.lib_name(lib_n,full_path));
617
618    def toExe(self,name,full_path = True):
619        return (self.lib_name(name,full_path));
620
621    def is64Platform (self):
622        return ( self.platform == PLATFORM_64);
623
624    def isRelease (self):
625        return ( self.mode  == RELEASE_);
626
627    def isPIE (self):
628        return self.is_pie
629
630    def update_executable_name (self,name):
631        return self.update_name(name,"-")
632
633    def update_non_exe_name (self,name):
634        return self.update_name(name,"_")
635
636    def update_name (self,name,delimiter):
637        trg = copy.copy(name);
638        if self.is64Platform():
639            trg += delimiter + "64";
640        else:
641            trg += delimiter + "32";
642
643        if self.isRelease () :
644            trg += "";
645        else:
646            trg +=  delimiter + "debug";
647
648        if self.isPIE():
649            trg += delimiter + "o"
650        return trg;
651
652    def get_target (self):
653        return self.update_executable_name("_t-rex");
654
655    def get_target_l2fwd (self):
656        return self.update_executable_name("l2fwd");
657
658    def get_dpdk_target (self):
659        return self.update_executable_name("dpdk");
660
661    def get_common_flags (self):
662        if self.isPIE():
663            flags = copy.copy(common_flags_old)
664        else:
665            flags = copy.copy(common_flags_new);
666
667        if self.is64Platform () :
668            flags += ['-m64'];
669        else:
670            flags += ['-m32'];
671
672        if self.isRelease () :
673            flags += ['-O3'];
674        else:
675            flags += ['-O0','-D_DEBUG'];
676
677        return (flags)
678
679    def get_cxx_flags (self):
680        flags = self.get_common_flags()
681
682        # support c++ 2011
683        flags += ['-std=c++0x']
684
685        flags += ['-Wall',
686                  '-Werror',
687                  '-Wno-literal-suffix',
688                  '-Wno-sign-compare',
689                  '-Wno-strict-aliasing']
690
691        return (flags)
692
693    def get_c_flags (self):
694        flags = self.get_common_flags()
695
696        # for C no special flags yet
697        return (flags)
698
699    def get_link_flags(self):
700        base_flags = ['-rdynamic'];
701        if self.is64Platform():
702            base_flags += ['-m64'];
703            base_flags += ['-lrt'];
704        else:
705            base_flags += ['-lrt'];
706
707        return base_flags;
708
709
710
711build_types = [
712               build_option(debug_mode= DEBUG_, platform = PLATFORM_64, is_pie = False),
713               build_option(debug_mode= RELEASE_,platform = PLATFORM_64, is_pie = False),
714               build_option(debug_mode= DEBUG_, platform = PLATFORM_64, is_pie = True),
715               build_option(debug_mode= RELEASE_,platform = PLATFORM_64, is_pie = True),
716              ]
717
718
719def build_prog (bld, build_obj):
720
721    zmq_lib_path='external_libs/zmq/'
722    bld.read_shlib( name='zmq' , paths=[top+zmq_lib_path] )
723
724    #rte_libs =[
725    #         'dpdk'];
726
727    #rte_libs1 = rte_libs+rte_libs+rte_libs;
728
729    #for obj in rte_libs:
730    #    bld.read_shlib( name=obj , paths=[top+rte_lib_path] )
731
732    # add electric fence only for debug image
733    debug_file_list='';
734    if not build_obj.isRelease ():
735        #debug
736        debug_file_list +=ef_src.file_list(top)
737
738
739    bld.objects(
740      features='c ',
741      includes = dpdk_includes_path,
742
743      cflags   = (build_obj.get_c_flags()+DPDK_FLAGS ),
744      source   = bp_dpdk.file_list(top),
745      target=build_obj.get_dpdk_target()
746      );
747
748    bld.program(features='cxx cxxprogram',
749                includes =includes_path,
750                cxxflags =(build_obj.get_cxx_flags()+['-std=gnu++11',]),
751                linkflags = build_obj.get_link_flags() ,
752                lib=['pthread','dl', 'z'],
753                use =[build_obj.get_dpdk_target(),'zmq'],
754                source = bp.file_list(top) + debug_file_list,
755                target = build_obj.get_target())
756
757
758
759def build_type(bld,build_obj):
760    build_prog(bld, build_obj);
761
762
763def post_build(bld):
764    print("copy objects")
765    exec_p ="../scripts/"
766    for obj in build_types:
767        install_single_system(bld, exec_p, obj);
768
769def build(bld):
770    bld.add_pre_fun(pre_build)
771    bld.add_post_fun(post_build);
772    for obj in build_types:
773        build_type(bld,obj);
774
775
776def build_info(bld):
777    pass;
778
779def install_single_system (bld, exec_p, build_obj):
780    o='build_dpdk/linux_dpdk/';
781    src_file =  os.path.realpath(o+build_obj.get_target())
782    if os.path.exists(src_file):
783        dest_file = exec_p +build_obj.get_target()
784        print(dest_file)
785        if not os.path.lexists(dest_file):
786            relative_path = os.path.relpath(src_file, exec_p)
787            os.symlink(relative_path, dest_file);
788
789
790def pre_build(bld):
791    print("update version files")
792    create_version_files ()
793
794
795def write_file (file_name,s):
796    f=open(file_name,'w')
797    f.write(s)
798    f.close()
799
800
801def get_build_num ():
802    s='';
803    if os.path.isfile(BUILD_NUM_FILE):
804        f=open(BUILD_NUM_FILE,'r');
805        s+=f.readline().rstrip();
806        f.close();
807    return s;
808
809def create_version_files ():
810    git_sha="N/A"
811    try:
812      r=getstatusoutput("git log --pretty=format:'%H' -n 1")
813      if r[0]==0:
814          git_sha=r[1]
815    except :
816        pass;
817
818
819    s =''
820    s +="#ifndef __TREX_VER_FILE__           \n"
821    s +="#define __TREX_VER_FILE__           \n"
822    s +="#ifdef __cplusplus                  \n"
823    s +=" extern \"C\" {                        \n"
824    s +=" #endif                             \n";
825    s +='#define  VERSION_USER  "%s"          \n' % os.environ.get('USER', 'unknown')
826    s +='extern const char * get_build_date(void);  \n'
827    s +='extern const char * get_build_time(void);  \n'
828    s +='#define VERSION_UIID      "%s"       \n' % uuid.uuid1()
829    s +='#define VERSION_GIT_SHA   "%s"       \n' % git_sha
830    s +='#define VERSION_BUILD_NUM "%s"       \n' % get_build_num()
831    s +="#ifdef __cplusplus                  \n"
832    s +=" }                        \n"
833    s +=" #endif                             \n";
834    s +="#endif \n"
835
836    write_file (H_VER_FILE ,s)
837
838    s ='#include "version.h"          \n'
839    s +='#define VERSION_UIID1      "%s"       \n' % uuid.uuid1()
840    s +="const char * get_build_date(void){ \n"
841    s +="    return (__DATE__);       \n"
842    s +="}      \n"
843    s +=" \n"
844    s +="const char * get_build_time(void){ \n"
845    s +="    return (__TIME__ );       \n"
846    s +="}      \n"
847
848    write_file (C_VER_FILE,s)
849
850def build_test(bld):
851    create_version_files ()
852
853def _copy_single_system (bld, exec_p, build_obj):
854    o='build_dpdk/linux_dpdk/';
855    src_file =  os.path.realpath(o+build_obj.get_target())
856    print(src_file)
857    if os.path.exists(src_file):
858        dest_file = exec_p +build_obj.get_target()
859        print(dest_file)
860        os.system("cp %s %s " %(src_file,dest_file));
861        os.system("chmod +x %s " %(dest_file));
862
863def _copy_single_system1 (bld, exec_p, build_obj):
864    o='../scripts/';
865    src_file =  os.path.realpath(o+build_obj.get_target()[1:])
866    print(src_file)
867    if os.path.exists(src_file):
868        dest_file = exec_p +build_obj.get_target()[1:]
869        os.system("cp %s %s " %(src_file,dest_file));
870        os.system("chmod +x %s " %(dest_file));
871
872
873def copy_single_system (bld, exec_p, build_obj):
874    _copy_single_system (bld, exec_p, build_obj)
875
876def copy_single_system1 (bld, exec_p, build_obj):
877    _copy_single_system1 (bld, exec_p, build_obj)
878
879
880files_list=[
881            'libzmq.so.3',
882            'trex-cfg',
883            'bp-sim-64',
884            'bp-sim-64-debug',
885            't-rex-debug-gdb',
886            'stl-sim',
887            'find_python.sh',
888            'run_regression',
889            'run_functional_tests',
890            'release_notes.pdf',
891            'dpdk_nic_bind.py',
892            'dpdk_setup_ports.py',
893            'doc_process.py',
894            'trex_daemon_server',
895            'master_daemon.py',
896            'trex-console',
897            'daemon_server'
898            ];
899
900files_dir=['cap2','avl','cfg','ko','automation', 'external_libs', 'python-lib','stl','api','exp']
901
902
903class Env(object):
904    @staticmethod
905    def get_env(name) :
906        s= os.environ.get(name);
907        if s == None:
908            print("You should define $ %s" % name)
909            raise Exception("Env error");
910        return (s);
911
912    @staticmethod
913    def get_release_path () :
914        s= Env().get_env('TREX_LOCAL_PUBLISH_PATH');
915        s +=get_build_num ()+"/"
916        return  s;
917
918    @staticmethod
919    def get_remote_release_path () :
920        s= Env().get_env('TREX_REMOTE_PUBLISH_PATH');
921        return  s;
922
923    @staticmethod
924    def get_local_web_server () :
925        s= Env().get_env('TREX_WEB_SERVER');
926        return  s;
927
928    # extral web
929    @staticmethod
930    def get_trex_ex_web_key() :
931        s= Env().get_env('TREX_EX_WEB_KEY');
932        return  s;
933
934    @staticmethod
935    def get_trex_ex_web_path() :
936        s= Env().get_env('TREX_EX_WEB_PATH');
937        return  s;
938
939    @staticmethod
940    def get_trex_ex_web_user() :
941        s= Env().get_env('TREX_EX_WEB_USER');
942        return  s;
943
944    @staticmethod
945    def get_trex_ex_web_srv() :
946        s= Env().get_env('TREX_EX_WEB_SRV');
947        return  s;
948
949    @staticmethod
950    def get_trex_regression_workspace():
951        return Env().get_env('TREX_REGRESSION_WORKSPACE')
952
953
954def check_release_permission():
955    if os.getenv('USER') not in USERS_ALLOWED_TO_RELEASE:
956        raise Exception('You are not allowed to release TRex. Please contact Hanoch.')
957
958# build package in parent dir. can provide custom name and folder with --pkg-dir and --pkg-file
959def pkg(self):
960    build_num = get_build_num()
961    pkg_dir = self.options.pkg_dir
962    if not pkg_dir:
963        pkg_dir = os.pardir
964    pkg_file = self.options.pkg_file
965    if not pkg_file:
966        pkg_file = '%s.tar.gz' % build_num
967    tmp_path = os.path.join(pkg_dir, '_%s' % pkg_file)
968    dst_path = os.path.join(pkg_dir, pkg_file)
969    build_path = os.path.join(os.pardir, build_num)
970
971    # clean old dir if exists
972    os.system('rm -rf %s' % build_path)
973    release(self, build_path + '/')
974    os.system("cp %s/%s.tar.gz %s" % (build_path, build_num, tmp_path))
975    os.system("mv %s %s" % (tmp_path, dst_path))
976
977    # clean new dir
978    os.system('rm -rf %s' % build_path)
979
980
981def release(bld, custom_dir = None):
982    """ release to local folder  """
983    if custom_dir:
984        exec_p = custom_dir
985    else:
986        check_release_permission()
987        exec_p = Env().get_release_path()
988    print("copy images and libs")
989    os.system(' mkdir -p '+exec_p);
990
991    for obj in build_types:
992        copy_single_system (bld,exec_p,obj);
993        copy_single_system1 (bld,exec_p,obj)
994
995    for obj in files_list:
996        src_file =  '../scripts/'+obj
997        dest_file = exec_p +'/'+obj
998        os.system("cp %s %s " %(src_file,dest_file));
999
1000    for obj in files_dir:
1001        src_file =  '../scripts/'+obj+'/'
1002        dest_file = exec_p +'/'+obj+'/'
1003        os.system("cp -rv %s %s " %(src_file,dest_file));
1004        os.system("chmod 755 %s " %(dest_file));
1005
1006    rel=get_build_num ()
1007
1008    # create client package
1009    os.system('mkdir -p %s/trex_client/external_libs' % exec_p)
1010    for ext_lib in client_external_libs:
1011        os.system('cp ../scripts/external_libs/%s %s/trex_client/external_libs/ -r' % (ext_lib, exec_p))
1012    os.system('cp ../scripts/automation/trex_control_plane/stf %s/trex_client/ -r' % exec_p)
1013    os.system('cp ../scripts/automation/trex_control_plane/stl %s/trex_client/ -r' % exec_p)
1014    with open('%s/trex_client/stl/examples/stl_path.py' % exec_p) as f:
1015        stl_path_content = f.read()
1016    if 'STL_PROFILES_PATH' not in stl_path_content:
1017        raise Exception('Could not find STL_PROFILES_PATH in stl/examples/stl_path.py')
1018    stl_path_content = re.sub('STL_PROFILES_PATH.*?\n', "STL_PROFILES_PATH = os.path.join(os.pardir, 'profiles')\n", stl_path_content)
1019    with open('%s/trex_client/stl/examples/stl_path.py' % exec_p, 'w') as f:
1020        f.write(stl_path_content)
1021    os.system('cp ../scripts/stl %s/trex_client/stl/profiles -r' % exec_p)
1022    shutil.make_archive(os.path.join(exec_p, 'trex_client_%s' % rel), 'gztar', exec_p, 'trex_client')
1023    os.system('rm -r %s/trex_client' % exec_p)
1024
1025    os.system('cd %s/..;tar --exclude="*.pyc" -zcvf %s/%s.tar.gz %s' %(exec_p,os.getcwd(),rel,rel))
1026    os.system("mv %s/%s.tar.gz %s" % (os.getcwd(),rel,exec_p));
1027
1028
1029def publish(bld, custom_source = None):
1030    check_release_permission()
1031    exec_p = Env().get_release_path()
1032    rel=get_build_num ()
1033
1034    release_name ='%s.tar.gz' % (rel);
1035    if custom_source:
1036        from_ = custom_source
1037    else:
1038        from_ = exec_p+'/'+release_name;
1039    os.system("rsync -av %s %s:%s/%s " %(from_,Env().get_local_web_server(),Env().get_remote_release_path (), release_name))
1040    os.system("ssh %s 'cd %s;rm be_latest; ln -P %s be_latest'  " %(Env().get_local_web_server(),Env().get_remote_release_path (),release_name))
1041    os.system("ssh %s 'cd %s;rm latest; ln -P %s latest'  " %(Env().get_local_web_server(),Env().get_remote_release_path (),release_name))
1042
1043
1044def publish_ext(bld, custom_source = None):
1045    check_release_permission()
1046    exec_p = Env().get_release_path()
1047    rel=get_build_num ()
1048
1049    release_name ='%s.tar.gz' % (rel);
1050    if custom_source:
1051        from_ = custom_source
1052    else:
1053        from_ = exec_p+'/'+release_name;
1054    cmd='rsync -avz --progress -e "ssh -i %s" --rsync-path=/usr/bin/rsync %s %s@%s:%s/release/%s' % (Env().get_trex_ex_web_key(),from_, Env().get_trex_ex_web_user(),Env().get_trex_ex_web_srv(),Env().get_trex_ex_web_path() ,release_name)
1055    print(cmd)
1056    os.system( cmd )
1057    os.system("ssh -i %s -l %s %s 'cd %s/release/;rm be_latest; ln -P %s be_latest'  " %(Env().get_trex_ex_web_key(),Env().get_trex_ex_web_user(),Env().get_trex_ex_web_srv(),Env().get_trex_ex_web_path(),release_name))
1058    os.system("ssh -i %s -l %s %s 'cd %s/release/;rm latest; ln -P %s latest'  " %(Env().get_trex_ex_web_key(),Env().get_trex_ex_web_user(),Env().get_trex_ex_web_srv(),Env().get_trex_ex_web_path(),release_name))
1059
1060# publish latest passed regression package (or custom commit from  --publish_commit option) as be_latest to trex-tgn.cisco.com and internal wiki
1061def publish_both(self):
1062    check_release_permission()
1063    packages_dir = Env().get_env('TREX_LOCAL_PUBLISH_PATH') + '/experiment/packages'
1064    publish_commit = self.options.publish_commit
1065    if publish_commit:
1066        package_file = '%s/%s.tar.gz' % (packages_dir, publish_commit)
1067    else:
1068        last_passed_commit_file = Env().get_trex_regression_workspace() + '/reports/last_passed_commit'
1069        with open(last_passed_commit_file) as f:
1070            last_passed_commit = f.read().strip()
1071        package_file = '%s/%s.tar.gz' % (packages_dir, last_passed_commit)
1072    publish(self, custom_source = package_file)
1073    publish_ext(self, custom_source = package_file)
1074
1075# print detailed latest passed regression commit + brief info of 5 commits before it
1076def show(bld):
1077    last_passed_commit_file = Env().get_trex_regression_workspace() + '/reports/last_passed_commit'
1078    with open(last_passed_commit_file) as f:
1079        last_passed_commit = f.read().strip()
1080
1081    # last passed nightly
1082    command = 'timeout 10 git show %s --quiet' % last_passed_commit
1083    result, output = getstatusoutput(command)
1084    if result == 0:
1085        print('Last passed regression commit:\n%s\n' % output)
1086    else:
1087        raise Exception('Error getting commit info with command: %s' % command)
1088
1089    # brief list of 5 commits before passed
1090    result, output = getstatusoutput('git --version')
1091    if result != 0 or output.startswith('git version 1'):
1092        # old format, no color etc.
1093        command = "timeout 10 git log --no-merges -n 5 --pretty=format:'%%h  %%an  %%ci  %%s' %s^@" % last_passed_commit
1094    else:
1095        # new format, with color, padding, truncating etc.
1096        command = "timeout 10 git log --no-merges -n 5 --pretty=format:'%%C(auto)%%h%%Creset  %%<(10,trunc)%%an  %%ci  %%<(100,trunc)%%s' %s^@ " % last_passed_commit
1097    result, output = getstatusoutput(command)
1098    if result == 0:
1099        print(output)
1100    else:
1101        raise Exception('Error getting commits info with command: %s' % command)
1102
1103def test (bld):
1104    r=getstatusoutput("git log --pretty=format:'%H' -n 1")
1105    if r[0]==0:
1106        print(r[1])
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120