trex_unit_test.py revision 138686f3
1#!/usr/bin/env python
2
3__copyright__ = "Copyright 2014"
4
5"""
6Name:
7     trex_unit_test.py
8
9
10Description:
11
12    This script creates the functionality to test the performance of the TRex traffic generator
13    The tested scenario is a TRex TG directly connected to a Cisco router.
14
15::
16
17    Topology:
18
19       -------                         --------
20      |       | Tx---1gig/10gig----Rx |        |
21      | TRex  |                       | router |
22      |       | Rx---1gig/10gig----Tx |        |
23       -------                         --------
24
25"""
26
27import os
28import sys
29import outer_packages
30
31import nose
32from nose.plugins import Plugin
33from nose.selector import Selector
34import CustomLogger
35import misc_methods
36from rednose import RedNose
37import termstyle
38from trex import CTRexScenario
39from trex_stf_lib.trex_client import *
40from trex_stf_lib.trex_exceptions import *
41from trex_stl_lib.api import *
42from trex_stl_lib.utils.GAObjClass import GAmanager
43import trex
44import socket
45from pprint import pprint
46import time
47from distutils.dir_util import mkpath
48
49# nose overrides
50
51# option to select wanted test by name without file, class etc.
52def new_Selector_wantMethod(self, method, orig_Selector_wantMethod = Selector.wantMethod):
53    result = orig_Selector_wantMethod(self, method)
54    return result and (not CTRexScenario.test or CTRexScenario.test in getattr(method, '__name__', ''))
55
56Selector.wantMethod = new_Selector_wantMethod
57
58def new_Selector_wantFunction(self, function, orig_Selector_wantFunction = Selector.wantFunction):
59    result = orig_Selector_wantFunction(self, function)
60    return result and (not CTRexScenario.test or CTRexScenario.test in getattr(function, '__name__', ''))
61
62Selector.wantFunction = new_Selector_wantFunction
63
64# override nose's strange representation of setUpClass errors
65def __suite_repr__(self):
66    if hasattr(self.context, '__module__'): # inside class, setUpClass etc.
67        class_repr = nose.suite._strclass(self.context)
68    else:                                   # outside of class, setUpModule etc.
69        class_repr = nose.suite._strclass(self.__class__)
70    return '%s.%s' % (class_repr, getattr(self.context, '__name__', self.context))
71
72nose.suite.ContextSuite.__repr__ = __suite_repr__
73nose.suite.ContextSuite.__str__  = __suite_repr__
74
75# /nose overrides
76
77def check_trex_path(trex_path):
78    if os.path.isfile('%s/trex_daemon_server' % trex_path):
79        return os.path.abspath(trex_path)
80
81def check_setup_path(setup_path):
82    if os.path.isfile('%s/config.yaml' % setup_path):
83        return os.path.abspath(setup_path)
84
85
86def get_trex_path():
87    latest_build_path = check_trex_path(os.getenv('TREX_UNDER_TEST'))    # TREX_UNDER_TEST is env var pointing to <trex-core>/scripts
88    if not latest_build_path:
89        latest_build_path = check_trex_path(os.path.join(os.pardir, os.pardir))
90    if not latest_build_path:
91        raise Exception('Could not determine trex_under_test folder, try setting env.var. TREX_UNDER_TEST')
92    return latest_build_path
93
94
95def address_to_ip(address):
96    for i in range(5):
97        try:
98            return socket.gethostbyname(address)
99        except:
100            continue
101    return socket.gethostbyname(address)
102
103
104class CTRexTestConfiguringPlugin(Plugin):
105    def options(self, parser, env = os.environ):
106        super(CTRexTestConfiguringPlugin, self).options(parser, env)
107        parser.add_option('--cfg', '--trex-scenario-config', action='store',
108                            dest='config_path',
109                            help='Specify path to folder with config.yaml and benchmark.yaml')
110        parser.add_option('--skip-clean', '--skip_clean', action='store_true',
111                            dest='skip_clean_config',
112                            help='Skip the clean configuration replace on the platform.')
113        parser.add_option('--load-image', '--load_image', action='store_true', default = False,
114                            dest='load_image',
115                            help='Install image specified in config file on router.')
116        parser.add_option('--log-path', '--log_path', action='store',
117                            dest='log_path',
118                            help='Specify path for the tests` log to be saved at. Once applied, logs capturing by nose will be disabled.') # Default is CURRENT/WORKING/PATH/trex_log/trex_log.log')
119        parser.add_option('--json-verbose', '--json_verbose', action="store_true", default = False,
120                            dest="json_verbose",
121                            help="Print JSON-RPC commands.")
122        parser.add_option('--telnet-verbose', '--telnet_verbose', action="store_true", default = False,
123                            dest="telnet_verbose",
124                            help="Print telnet commands and responces.")
125        parser.add_option('--server-logs', '--server_logs', action="store_true", default = False,
126                            dest="server_logs",
127                            help="Print server side (TRex and trex_daemon) logs per test.")
128        parser.add_option('--kill-running', '--kill_running', action="store_true", default = False,
129                            dest="kill_running",
130                            help="Kills running TRex process on remote server (useful for regression).")
131        parser.add_option('--func', '--functional', action="store_true", default = False,
132                            dest="functional",
133                            help="Run functional tests.")
134        parser.add_option('--stl', '--stateless', action="store_true", default = False,
135                            dest="stateless",
136                            help="Run stateless tests.")
137        parser.add_option('--stf', '--stateful', action="store_true", default = False,
138                            dest="stateful",
139                            help="Run stateful tests.")
140        parser.add_option('--pkg', action="store",
141                            dest="pkg",
142                            help="Run with given TRex package. Make sure the path available at server machine.")
143        parser.add_option('--collect', action="store_true", default = False,
144                            dest="collect",
145                            help="Alias to --collect-only.")
146        parser.add_option('--warmup', action="store_true", default = False,
147                            dest="warmup",
148                            help="Warm up the system for stateful: run 30 seconds 9k imix test without check of results.")
149        parser.add_option('--test-client-package', '--test_client_package', action="store_true", default = False,
150                            dest="test_client_package",
151                            help="Includes tests of client package.")
152        parser.add_option('--long', action="store_true", default = False,
153                            dest="long",
154                            help="Flag of long tests (stability).")
155        parser.add_option('--ga', action="store_true", default = False,
156                            dest="ga",
157                            help="Flag to send benchmarks to GA.")
158        parser.add_option('--no-daemon', action="store_true", default = False,
159                            dest="no_daemon",
160                            help="Flag that specifies to use running stl server, no need daemons.")
161        parser.add_option('--debug-image', action="store_true", default = False,
162                            dest="debug_image",
163                            help="Flag that specifies to use t-rex-64-debug as TRex executable.")
164        parser.add_option('--trex-args', action='store', default = '',
165                            dest="trex_args",
166                            help="Additional TRex arguments (--no-watchdog etc.).")
167        parser.add_option('-t', '--test', action='store', default = '', dest='test',
168                            help='Test name to run (without file, class etc.)')
169
170
171    def configure(self, options, conf):
172        self.collect_only   = options.collect_only
173        self.functional     = options.functional
174        self.stateless      = options.stateless
175        self.stateful       = options.stateful
176        self.pkg            = options.pkg
177        self.json_verbose   = options.json_verbose
178        self.telnet_verbose = options.telnet_verbose
179        self.no_daemon      = options.no_daemon
180        CTRexScenario.test  = options.test
181        if self.collect_only or self.functional:
182            return
183        if CTRexScenario.setup_dir and options.config_path:
184            raise Exception('Please either define --cfg or use env. variable SETUP_DIR, not both.')
185        if not options.config_path and CTRexScenario.setup_dir:
186            options.config_path = CTRexScenario.setup_dir
187        if not options.config_path:
188            raise Exception('Please specify path to config.yaml using --cfg parameter or env. variable SETUP_DIR')
189        self.configuration = misc_methods.load_complete_config_file(os.path.join(options.config_path, 'config.yaml'))
190        self.configuration.trex['trex_name'] = address_to_ip(self.configuration.trex['trex_name']) # translate hostname to ip
191        self.benchmark     = misc_methods.load_benchmark_config_file(os.path.join(options.config_path, 'benchmark.yaml'))
192        self.enabled       = True
193        self.modes         = self.configuration.trex.get('modes', [])
194        self.kill_running  = options.kill_running
195        self.load_image    = options.load_image
196        self.clean_config  = False if options.skip_clean_config else True
197        self.server_logs   = options.server_logs
198        if options.log_path:
199            self.loggerPath = options.log_path
200        # initialize CTRexScenario global testing class, to be used by all tests
201        CTRexScenario.configuration = self.configuration
202        CTRexScenario.no_daemon     = options.no_daemon
203        CTRexScenario.benchmark     = self.benchmark
204        CTRexScenario.modes         = set(self.modes)
205        CTRexScenario.server_logs   = self.server_logs
206        CTRexScenario.debug_image   = options.debug_image
207        if not self.no_daemon:
208            CTRexScenario.trex      = CTRexClient(trex_host   = self.configuration.trex['trex_name'],
209                                                  verbose     = self.json_verbose,
210                                                  debug_image = options.debug_image,
211                                                  trex_args   = options.trex_args)
212            if not CTRexScenario.trex.check_master_connectivity():
213                print('Could not connect to master daemon')
214                sys.exit(-1)
215        if options.ga and CTRexScenario.setup_name:
216            CTRexScenario.GAManager  = GAmanager(GoogleID       = 'UA-75220362-4',
217                                                 UserID         = CTRexScenario.setup_name,
218                                                 QueueSize      = 100,
219                                                 Timeout        = 5, # seconds
220                                                 UserPermission = 1,
221                                                 BlockingMode   = 1,
222                                                 appName        = 'TRex',
223                                                 appVer         = '1.11.232')
224
225
226    def begin (self):
227        client = CTRexScenario.trex
228        if self.pkg and not CTRexScenario.is_copied:
229            if client.master_daemon.is_trex_daemon_running() and client.get_trex_cmds() and not self.kill_running:
230                print("Can't update TRex, it's running")
231                sys.exit(-1)
232            print('Updating TRex to %s' % self.pkg)
233            if not client.master_daemon.update_trex(self.pkg):
234                print('Failed updating TRex')
235                sys.exit(-1)
236            else:
237                print('Updated')
238            CTRexScenario.is_copied = True
239        if self.functional or self.collect_only:
240            return
241        if not self.no_daemon:
242            print('Restarting TRex daemon server')
243            res = client.restart_trex_daemon()
244            if not res:
245                print('Could not restart TRex daemon server')
246                sys.exit(-1)
247            print('Restarted.')
248
249            if self.kill_running:
250                client.kill_all_trexes()
251            else:
252                if client.get_trex_cmds():
253                    print('TRex is already running')
254                    sys.exit(-1)
255
256        if self.stateless:
257            cores = self.configuration.trex.get('trex_cores', 1)
258            if 'virt_nics' in self.modes and cores > 1:
259                raise Exception('Number of cores should be 1 with virtual NICs')
260            if not self.no_daemon:
261                client.start_stateless(c = cores)
262            CTRexScenario.stl_trex = STLClient(username = 'TRexRegression',
263                                               server = self.configuration.trex['trex_name'],
264                                               verbose_level = self.json_verbose)
265        if 'loopback' not in self.modes:
266            CTRexScenario.router_cfg = dict(config_dict      = self.configuration.router,
267                                            forceImageReload = self.load_image,
268                                            silent_mode      = not self.telnet_verbose,
269                                            forceCleanConfig = self.clean_config,
270                                            tftp_config_dict = self.configuration.tftp)
271        try:
272            CustomLogger.setup_custom_logger('TRexLogger', self.loggerPath)
273        except AttributeError:
274            CustomLogger.setup_custom_logger('TRexLogger')
275
276    def finalize(self, result):
277        if self.functional or self.collect_only:
278            return
279        CTRexScenario.is_init = False
280        if self.stateful:
281            CTRexScenario.trex = None
282        if self.stateless:
283            if self.no_daemon:
284                if CTRexScenario.stl_trex and CTRexScenario.stl_trex.is_connected():
285                    CTRexScenario.stl_trex.disconnect()
286            else:
287                CTRexScenario.trex.force_kill(False)
288            CTRexScenario.stl_trex = None
289
290
291def save_setup_info():
292    try:
293        if CTRexScenario.setup_name and CTRexScenario.trex_version:
294            setup_info = ''
295            for key, value in CTRexScenario.trex_version.items():
296                setup_info += '{0:8}: {1}\n'.format(key, value)
297            cfg = CTRexScenario.configuration
298            setup_info += 'Server: %s, Modes: %s' % (cfg.trex.get('trex_name'), cfg.trex.get('modes'))
299            if cfg.router:
300                setup_info += '\nRouter: Model: %s, Image: %s' % (cfg.router.get('model'), CTRexScenario.router_image)
301            if CTRexScenario.debug_image:
302                setup_info += '\nDebug image: %s' % CTRexScenario.debug_image
303
304            with open('%s/report_%s.info' % (CTRexScenario.report_dir, CTRexScenario.setup_name), 'w') as f:
305                f.write(setup_info)
306    except Exception as err:
307        print('Error saving setup info: %s ' % err)
308
309
310if __name__ == "__main__":
311
312    # setting defaults. By default we run all the test suite
313    specific_tests              = False
314    CTRexScenario.report_dir    = 'reports'
315    need_to_copy                = False
316    setup_dir                   = os.getenv('SETUP_DIR', '').rstrip('/')
317    CTRexScenario.setup_dir     = check_setup_path(setup_dir)
318    CTRexScenario.scripts_path  = get_trex_path()
319    if not CTRexScenario.setup_dir:
320        CTRexScenario.setup_dir = check_setup_path(os.path.join('setups', setup_dir))
321
322
323    nose_argv = ['', '-s', '-v', '--exe', '--rednose', '--detailed-errors']
324    test_client_package = False
325    if '--test-client-package' in sys.argv:
326        test_client_package = True
327
328    if '--collect' in sys.argv:
329        sys.argv.append('--collect-only')
330    if '--collect-only' in sys.argv: # this is a user trying simply to view the available tests. no need xunit.
331        CTRexScenario.is_test_list   = True
332        xml_arg                      = ''
333    else:
334        xml_name                     = 'unit_test.xml'
335        if CTRexScenario.setup_dir:
336            CTRexScenario.setup_name = os.path.basename(CTRexScenario.setup_dir)
337            xml_name = 'report_%s.xml' % CTRexScenario.setup_name
338        xml_arg= '--xunit-file=%s/%s' % (CTRexScenario.report_dir, xml_name)
339        mkpath(CTRexScenario.report_dir)
340
341    sys_args = sys.argv[:]
342    for i, arg in enumerate(sys.argv):
343        if 'log-path' in arg:
344            nose_argv += ['--nologcapture']
345        else:
346            for tests_type in CTRexScenario.test_types.keys():
347                if tests_type in arg:
348                    specific_tests = True
349                    CTRexScenario.test_types[tests_type].append(arg[arg.find(tests_type):])
350                    sys_args.remove(arg)
351
352    if not specific_tests:
353        for key in ('--func', '--functional'):
354            if key in sys_args:
355                CTRexScenario.test_types['functional_tests'].append('functional_tests')
356                sys_args.remove(key)
357        for key in ('--stf', '--stateful'):
358            if key in sys_args:
359                CTRexScenario.test_types['stateful_tests'].append('stateful_tests')
360                sys_args.remove(key)
361        for key in ('--stl', '--stateless'):
362            if key in sys_args:
363                CTRexScenario.test_types['stateless_tests'].append('stateless_tests')
364                sys_args.remove(key)
365        # Run all of the tests or just the selected ones
366        if not sum([len(x) for x in CTRexScenario.test_types.values()]):
367            for key in CTRexScenario.test_types.keys():
368                CTRexScenario.test_types[key].append(key)
369
370    nose_argv += sys_args
371
372    addplugins = [RedNose(), CTRexTestConfiguringPlugin()]
373    result = True
374    try:
375        if len(CTRexScenario.test_types['functional_tests']):
376            additional_args = ['--func'] + CTRexScenario.test_types['functional_tests']
377            if xml_arg:
378                additional_args += ['--with-xunit', xml_arg.replace('.xml', '_functional.xml')]
379            result = nose.run(argv = nose_argv + additional_args, addplugins = addplugins)
380        if len(CTRexScenario.test_types['stateful_tests']):
381            additional_args = ['--stf']
382            if '--warmup' in sys.argv:
383                additional_args.append('stateful_tests/trex_imix_test.py:CTRexIMIX_Test.test_warm_up')
384            additional_args += CTRexScenario.test_types['stateful_tests']
385            if not test_client_package:
386                additional_args.extend(['-a', '!client_package'])
387            if xml_arg:
388                additional_args += ['--with-xunit', xml_arg.replace('.xml', '_stateful.xml')]
389            result = nose.run(argv = nose_argv + additional_args, addplugins = addplugins) and result
390        if len(CTRexScenario.test_types['stateless_tests']):
391            additional_args = ['--stl', 'stateless_tests/stl_general_test.py:STLBasic_Test.test_connectivity'] + CTRexScenario.test_types['stateless_tests']
392            if not test_client_package:
393                additional_args.extend(['-a', '!client_package'])
394            if xml_arg:
395                additional_args += ['--with-xunit', xml_arg.replace('.xml', '_stateless.xml')]
396            result = nose.run(argv = nose_argv + additional_args, addplugins = addplugins) and result
397    #except Exception as e:
398    #    result = False
399    #    print(e)
400    finally:
401        save_setup_info()
402
403    if not CTRexScenario.is_test_list:
404        if result == True:
405            print(termstyle.green("""
406                 ..::''''::..
407               .;''        ``;.
408              ::    ::  ::    ::
409             ::     ::  ::     ::
410             ::     ::  ::     ::
411             :: .:' ::  :: `:. ::
412             ::  :          :  ::
413              :: `:.      .:' ::
414               `;..``::::''..;'
415                 ``::,,,,::''
416
417               ___  ___   __________
418              / _ \/ _ | / __/ __/ /
419             / ___/ __ |_\ \_\ \/_/
420            /_/  /_/ |_/___/___(_)
421
422        """))
423            sys.exit(0)
424        else:
425            print(termstyle.red("""
426           /\_/\
427          ( o.o )
428           > ^ <
429
430This cat is sad, test failed.
431        """))
432            sys.exit(-1)
433
434
435
436
437
438
439
440
441
442
443
444