trex_general_test.py revision 48cb48c6
1#!/router/bin/python
2
3__copyright__ = "Copyright 2014"
4
5"""
6Name:
7     trex_general_test.py
8
9
10Description:
11
12    This script creates the functionality to test the performance of the TRex traffic generator
13    The tested scenario is a TRex TG directly connected to a Cisco router.
14
15::
16
17    Topology:
18
19       -------                         --------
20      |       | Tx---1gig/10gig----Rx |        |
21      | TRex  |                       | router |
22      |       | Rx---1gig/10gig----Tx |        |
23       -------                         --------
24
25"""
26from nose.plugins import Plugin
27from nose.plugins.skip import SkipTest
28import trex
29from trex import CTRexScenario
30import misc_methods
31import sys
32import os
33# from CPlatformUnderTest import *
34from CPlatform import *
35import termstyle
36import threading
37from .tests_exceptions import *
38from platform_cmd_link import *
39import unittest
40from glob import glob
41
42def setUpModule(module):
43    pass
44
45def tearDownModule(module):
46    pass
47
48class CTRexGeneral_Test(unittest.TestCase):
49    """This class defines the general stateful testcase of the TRex traffic generator"""
50    def __init__ (self, *args, **kwargs):
51        sys.stdout.flush()
52        unittest.TestCase.__init__(self, *args, **kwargs)
53        if CTRexScenario.is_test_list:
54            return
55        # Point test object to scenario global object
56        self.configuration         = CTRexScenario.configuration
57        self.benchmark             = CTRexScenario.benchmark
58        self.trex                  = CTRexScenario.trex
59        self.stl_trex              = CTRexScenario.stl_trex
60        self.trex_crashed          = CTRexScenario.trex_crashed
61        self.modes                 = CTRexScenario.modes
62        self.GAManager             = CTRexScenario.GAManager
63        self.no_daemon             = CTRexScenario.no_daemon
64        self.skipping              = False
65        self.fail_reasons          = []
66        if not hasattr(self, 'unsupported_modes'):
67            self.unsupported_modes   = []
68        self.is_loopback           = True if 'loopback' in self.modes else False
69        self.is_virt_nics          = True if 'virt_nics' in self.modes else False
70        self.is_VM                 = True if 'VM' in self.modes else False
71
72        if not CTRexScenario.is_init:
73            if self.trex and not self.no_daemon: # stateful
74                CTRexScenario.trex_version = self.trex.get_trex_version()
75            if not self.is_loopback:
76                # initilize the scenario based on received configuration, once per entire testing session
77                CTRexScenario.router = CPlatform(CTRexScenario.router_cfg['silent_mode'])
78                device_cfg           = CDeviceCfg()
79                device_cfg.set_platform_config(CTRexScenario.router_cfg['config_dict'])
80                device_cfg.set_tftp_config(CTRexScenario.router_cfg['tftp_config_dict'])
81                CTRexScenario.router.load_platform_data_from_file(device_cfg)
82                CTRexScenario.router.launch_connection(device_cfg)
83                if CTRexScenario.router_cfg['forceImageReload']:
84                    running_image = CTRexScenario.router.get_running_image_details()['image']
85                    print('Current router image: %s' % running_image)
86                    needed_image = device_cfg.get_image_name()
87                    if not CTRexScenario.router.is_image_matches(needed_image):
88                        print('Setting router image: %s' % needed_image)
89                        CTRexScenario.router.config_tftp_server(device_cfg)
90                        CTRexScenario.router.load_platform_image(needed_image)
91                        CTRexScenario.router.set_boot_image(needed_image)
92                        CTRexScenario.router.reload_platform(device_cfg)
93                        CTRexScenario.router.launch_connection(device_cfg)
94                        running_image = CTRexScenario.router.get_running_image_details()['image'] # verify image
95                        if not CTRexScenario.router.is_image_matches(needed_image):
96                            self.fail('Unable to set router image: %s, current image is: %s' % (needed_image, running_image))
97                    else:
98                        print('Matches needed image: %s' % needed_image)
99                    CTRexScenario.router_image = running_image
100
101            if self.modes:
102                print(termstyle.green('\t!!!\tRunning with modes: %s, not suitable tests will be skipped.\t!!!' % list(self.modes)))
103
104            CTRexScenario.is_init = True
105            print(termstyle.green("Done instantiating TRex scenario!\n"))
106
107#           raise RuntimeError('CTRexScenario class is not initialized!')
108        self.router = CTRexScenario.router
109
110
111
112#   def assert_dict_eq (self, dict, key, val, error=''):
113#           v1 = int(dict[key]))
114#           self.assertEqual(v1, int(val), error)
115#
116#   def assert_dict_gt (self, d, key, val, error=''):
117#           v1 = int(dict[key])
118#           self.assert_gt(v1, int(val), error)
119
120    def assertEqual(self, v1, v2, s):
121        if v1 != v2:
122            error='ERROR '+str(v1)+' !=  '+str(v2)+ '   '+s;
123            self.fail(error)
124
125    def assert_gt(self, v1, v2, s):
126        if not v1 > v2:
127            error='ERROR {big} <  {small}      {str}'.format(big = v1, small = v2, str = s)
128            self.fail(error)
129
130    def check_results_eq (self,res,name,val):
131        if res is None:
132            self.fail('TRex results cannot be None !')
133            return
134
135        if name not in res:
136            self.fail('TRex results does not include key %s' % name)
137            return
138
139        if res[name] != float(val):
140            self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val))
141
142    def check_CPU_benchmark (self, trex_res, err = 25, minimal_cpu = 10, maximal_cpu = 85):
143        cpu_util          = trex_res.get_avg_steady_state_value('trex-global.data.m_cpu_util_raw')
144        trex_tx_bps       = trex_res.get_avg_steady_state_value('trex-global.data.m_tx_bps')
145        expected_norm_cpu = self.get_benchmark_param('bw_per_core')
146        cores             = self.get_benchmark_param('cores')
147        ports_count       = trex_res.get_ports_count()
148        if not (cpu_util and ports_count and cores):
149            print("Can't calculate CPU benchmark, need to divide by zero: cpu util: %s, ports: %s, cores: %s" % (cpu_util, ports_count, cores))
150            test_norm_cpu = -1
151        else:
152            test_norm_cpu = trex_tx_bps / (cpu_util * ports_count * cores * 2.5e6)
153
154        if '1G' in self.modes:
155            minimal_cpu /= 10.0
156
157        if not self.is_virt_nics:
158            if cpu_util > maximal_cpu:
159                self.fail("CPU is too high (%s%%), probably queue full." % cpu_util )
160            #if cpu_util < minimal_cpu:
161            #    self.fail("CPU is too low (%s%%), can't verify performance in such low CPU%%." % cpu_util )
162
163        print("TRex CPU utilization: %g%%, norm_cpu is : %g Gb/core" % (round(cpu_util, 2), round(test_norm_cpu, 2)))
164        if test_norm_cpu < 0:
165            return
166
167        if not expected_norm_cpu:
168            expected_norm_cpu = 1
169
170        calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100)
171        print('Err percent: %s' % calc_error_precent)
172        #if calc_error_precent > err and cpu_util > 10:
173        #    self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu)))
174
175        # report benchmarks
176        if self.GAManager:
177            try:
178                pass
179                #setup_test = '%s.%s' % (CTRexScenario.setup_name, self.get_name())
180                #self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core', value = int(test_norm_cpu))
181                #self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core_exp', value = int(expected_norm_cpu))
182                #self.GAManager.emptyAndReportQ()
183            except Exception as e:
184                print('Sending GA failed: %s' % e)
185
186    def check_results_gt (self, res, name, val):
187        if res is None:
188            self.fail('TRex results canot be None !')
189            return
190
191        if name not in res:
192            self.fail('TRex results does not include key %s' % name)
193            return
194
195        if res[name]< float(val):
196            self.fail('TRex results[%s]<%f and not as expected greater than %f ' % (name, res[name], val))
197
198    def check_for_trex_crash(self):
199        pass
200
201    def get_benchmark_param (self, param, sub_param = None, test_name = None,default=None):
202        if not test_name:
203            test_name = self.get_name()
204        if test_name not in self.benchmark:
205            if default ==None:
206               self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param))
207            else:
208                return default
209        if sub_param:
210            return self.benchmark[test_name][param].get(sub_param)
211        else:
212            return self.benchmark[test_name].get(param)
213
214    def check_general_scenario_results (self, trex_res, check_latency = True):
215
216        try:
217            # check history size is enough
218            if len(trex_res._history) < 5:
219                self.fail('TRex results list is too short. Increase the test duration or check unexpected stopping.')
220
221            # check if test is valid
222            if not trex_res.is_done_warmup():
223                self.fail('TRex did not reach warm-up situtaion. Results are not valid.')
224
225            # check that BW is not much more than expected
226            trex_exp_bps = int(trex_res.get_expected_tx_rate().get('m_tx_expected_bps') / 1e6)
227            trex_cur_bps = int(max(trex_res.get_value_list('trex-global.data.m_tx_bps')) / 1e6)
228
229            if trex_exp_bps is None:
230                self.fail('Expected rate is None!')
231            if trex_cur_bps is None:
232                self.fail('Current rate is None!')
233
234            if trex_exp_bps * 1.05 < trex_cur_bps:
235                self.fail('Got BW (%sMbps) that is %s%% more than expected (%sMbps)!' % (trex_cur_bps, round(100.0 * trex_cur_bps / trex_exp_bps - 100, 2), trex_exp_bps))
236
237            # check TRex number of drops
238            trex_tx_pckt    = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
239            trex_drops      = trex_res.get_total_drops()
240            trex_drop_rate  = trex_res.get_drop_rate()
241            if ( trex_drops > 0.001 * trex_tx_pckt) and (trex_drop_rate > 0.0):     # deliberately mask kickoff drops when TRex first initiated
242                self.fail('Number of packet drops larger than 0.1% of all traffic')
243
244            # check queue full, queue drop, allocation error
245            m_total_alloc_error = trex_res.get_last_value("trex-global.data.m_total_alloc_error")
246            m_total_queue_full = trex_res.get_last_value("trex-global.data.m_total_queue_full")
247            m_total_queue_drop = trex_res.get_last_value("trex-global.data.m_total_queue_drop")
248            self.assert_gt(1000, m_total_alloc_error, 'Got allocation errors. (%s), please review multiplier and templates configuration.' % m_total_alloc_error)
249            self.assert_gt(1000, m_total_queue_drop, 'Too much queue_drop (%s), please review multiplier.' % m_total_queue_drop)
250
251            if self.is_VM:
252                allowed_queue_full = 10000 + trex_tx_pckt / 100
253            else:
254                allowed_queue_full = 1000 + trex_tx_pckt / 1000
255            self.assert_gt(allowed_queue_full, m_total_queue_full, 'Too much queue_full (%s), please review multiplier.' % m_total_queue_full)
256
257            # # check TRex expected counters
258            #trex_exp_rate = trex_res.get_expected_tx_rate().get('m_tx_expected_bps')
259            #assert trex_exp_rate is not None
260            #trex_exp_gbps = trex_exp_rate/(10**9)
261
262            if check_latency:
263                # check that max latency does not exceed 1 msec
264                if self.configuration.trex['trex_name'] == '10.56.217.210': # temporary workaround for latency issue in kiwi02, remove it ASAP. http://trex-tgn.cisco.com/youtrack/issue/trex-194
265                    allowed_latency = 8000
266                elif self.is_VM:
267                    allowed_latency = 9999999
268                else: # no excuses, check 1ms
269                    allowed_latency = 1000
270                if max(trex_res.get_max_latency().values()) > allowed_latency:
271                    self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency)
272
273                # check that avg latency does not exceed 1 msec
274                if self.is_VM:
275                    allowed_latency = 9999999
276                else: # no excuses, check 1ms
277                    allowed_latency = 1000
278                if max(trex_res.get_avg_latency().values()) > allowed_latency:
279                    self.fail('LatencyError: Average latency exceeds %s (usec)' % allowed_latency)
280
281                ports_names = trex_res.get_last_value('trex-latecny-v2.data', 'port\-\d+')
282                if not ports_names:
283                    raise AbnormalResultError('Could not find ports info in TRex results, path: trex-latecny-v2.data.port-*')
284                for port_name in ports_names:
285                    path = 'trex-latecny-v2.data.%s.hist.cnt' % port_name
286                    lat_count = trex_res.get_last_value(path)
287                    if lat_count == 0:
288                        self.fail('LatencyError: Number of latency packets received on %s is 0' % port_name)
289
290            if not self.is_loopback:
291                # check router number of drops --> deliberately masked- need to be figured out!!!!!
292                pkt_drop_stats = self.router.get_drop_stats()
293#               assert pkt_drop_stats['total_drops'] < 20
294
295                # check for trex-router packet consistency
296                # TODO: check if it's ok
297                print('router drop stats: %s' % pkt_drop_stats)
298                print('TRex drop stats: %s' % trex_drops)
299                #self.assertEqual(pkt_drop_stats, trex_drops, "TRex's and router's drop stats don't match.")
300
301        except KeyError as e:
302            self.fail(e)
303            #assert False
304
305        # except AssertionError as e:
306        #     e.args += ('TRex has crashed!')
307        #     raise
308
309    @staticmethod
310    def unzip_client_package():
311        client_pkg_files = glob('%s/trex_client*.tar.gz' % CTRexScenario.scripts_path)
312        if not len(client_pkg_files):
313            raise Exception('Could not find client package')
314        if len(client_pkg_files) > 1:
315            raise Exception('Found more than one client packages')
316        if not os.path.exists('%s/trex_client' % CTRexScenario.scripts_path):
317            print('\nUnzipping package')
318            return_code, _, stderr = misc_methods.run_command("tar -xzf %s -C %s" % (client_pkg_files[0], CTRexScenario.scripts_path))
319            if return_code:
320                raise Exception('Could not untar the client package: %s' % stderr)
321        else:
322            print('\nClient package is untarred')
323
324    # We encountered error, don't fail the test immediately
325    def fail(self, reason = 'Unknown error'):
326        print('Error: %s' % reason)
327        self.fail_reasons.append(reason)
328
329    # skip running of the test, counts as 'passed' but prints 'skipped'
330    def skip(self, message = 'Unknown reason'):
331        print('Skip: %s' % message)
332        self.skipping = True
333        raise SkipTest(message)
334
335    # get name of currently running test
336    def get_name(self):
337        return self._testMethodName
338
339    def setUp(self):
340        test_setup_modes_conflict = self.modes & set(self.unsupported_modes)
341        if test_setup_modes_conflict:
342            self.skip("The test can't run with following modes of given setup: %s " % test_setup_modes_conflict)
343        if not self.stl_trex and not self.trex.is_idle():
344            print('Warning: TRex is not idle at setUp, trying to stop it.')
345            self.trex.force_kill(confirm = False)
346        if not self.is_loopback:
347            print('')
348            if not self.stl_trex: # stateful
349                self.router.load_clean_config()
350            self.router.clear_counters()
351            self.router.clear_packet_drop_stats()
352
353    ########################################################################
354    ####                DO NOT ADD TESTS TO THIS FILE                   ####
355    ####    Added tests here will held once for EVERY test sub-class    ####
356    ########################################################################
357
358    # masked example to such test. uncomment to watch how it affects #
359#   def test_isInitialized(self):
360#       assert CTRexScenario.is_init == True
361    def tearDown(self):
362        if not self.stl_trex and not self.trex.is_idle():
363            print('Warning: TRex is not idle at tearDown, trying to stop it.')
364            self.trex.force_kill(confirm = False)
365        if not self.skipping:
366            # print server logs of test run
367            if self.trex and CTRexScenario.server_logs and not self.no_daemon:
368                try:
369                    print(termstyle.green('\n>>>>>>>>>>>>>>> Daemon log <<<<<<<<<<<<<<<'))
370                    daemon_log = self.trex.get_trex_daemon_log()
371                    log_size = len(daemon_log)
372                    print(''.join(daemon_log[CTRexScenario.daemon_log_lines:]))
373                    CTRexScenario.daemon_log_lines = log_size
374                except Exception as e:
375                    print("Can't get TRex daemon log:", e)
376                try:
377                    print(termstyle.green('>>>>>>>>>>>>>>>> Trex log <<<<<<<<<<<<<<<<'))
378                    print(''.join(self.trex.get_trex_log()))
379                except Exception as e:
380                    print("Can't get TRex log:", e)
381            if len(self.fail_reasons):
382                sys.stdout.flush()
383                raise Exception('Test failed. Reasons:\n%s' % '\n'.join(self.fail_reasons))
384        sys.stdout.flush()
385
386    def check_for_trex_crash(self):
387        pass
388