trex_general_test.py revision f35edfa4
1#!/router/bin/python
2
3__copyright__ = "Copyright 2014"
4
5"""
6Name:
7     trex_general_test.py
8
9
10Description:
11
12    This script creates the functionality to test the performance of the T-Rex traffic generator
13    The tested scenario is a T-Rex TG directly connected to a Cisco router.
14
15::
16
17    Topology:
18
19       -------                         --------
20      |       | Tx---1gig/10gig----Rx |        |
21      | T-Rex |                       | router |
22      |       | Rx---1gig/10gig----Tx |        |
23       -------                         --------
24
25"""
26from nose.plugins import Plugin
27from nose.plugins.skip import SkipTest
28import trex
29from trex import CTRexScenario
30import misc_methods
31import sys
32import os
33# from CPlatformUnderTest import *
34from CPlatform import *
35import termstyle
36import threading
37from .tests_exceptions import *
38from platform_cmd_link import *
39import unittest
40from glob import glob
41
42def setUpModule(module):
43    pass
44
45def tearDownModule(module):
46    pass
47
48class CTRexGeneral_Test(unittest.TestCase):
49    """This class defines the general stateful testcase of the T-Rex traffic generator"""
50    def __init__ (self, *args, **kwargs):
51        sys.stdout.flush()
52        unittest.TestCase.__init__(self, *args, **kwargs)
53        if CTRexScenario.is_test_list:
54            return
55        # Point test object to scenario global object
56        self.configuration         = CTRexScenario.configuration
57        self.benchmark             = CTRexScenario.benchmark
58        self.trex                  = CTRexScenario.trex
59        self.trex_crashed          = CTRexScenario.trex_crashed
60        self.modes                 = CTRexScenario.modes
61        self.skipping              = False
62        self.fail_reasons          = []
63        if not hasattr(self, 'unsupported_modes'):
64            self.unsupported_modes   = []
65        self.is_loopback           = True if 'loopback' in self.modes else False
66        self.is_virt_nics          = True if 'virt_nics' in self.modes else False
67        self.is_VM                 = True if 'VM' in self.modes else False
68
69        if not CTRexScenario.is_init:
70            if self.trex: # stateful
71                CTRexScenario.trex_version = self.trex.get_trex_version()
72            if not self.is_loopback:
73                # initilize the scenario based on received configuration, once per entire testing session
74                CTRexScenario.router = CPlatform(CTRexScenario.router_cfg['silent_mode'])
75                device_cfg           = CDeviceCfg()
76                device_cfg.set_platform_config(CTRexScenario.router_cfg['config_dict'])
77                device_cfg.set_tftp_config(CTRexScenario.router_cfg['tftp_config_dict'])
78                CTRexScenario.router.load_platform_data_from_file(device_cfg)
79                CTRexScenario.router.launch_connection(device_cfg)
80                running_image = CTRexScenario.router.get_running_image_details()['image']
81                print('Current router image: %s' % running_image)
82                if CTRexScenario.router_cfg['forceImageReload']:
83                    needed_image = device_cfg.get_image_name()
84                    if not CTRexScenario.router.is_image_matches(needed_image):
85                        print('Setting router image: %s' % needed_image)
86                        CTRexScenario.router.config_tftp_server(device_cfg)
87                        CTRexScenario.router.load_platform_image(needed_image)
88                        CTRexScenario.router.set_boot_image(needed_image)
89                        CTRexScenario.router.reload_platform(device_cfg)
90                        CTRexScenario.router.launch_connection(device_cfg)
91                        running_image = CTRexScenario.router.get_running_image_details()['image'] # verify image
92                        if not CTRexScenario.router.is_image_matches(needed_image):
93                            self.fail('Unable to set router image: %s, current image is: %s' % (needed_image, running_image))
94                    else:
95                        print('Matches needed image: %s' % needed_image)
96                CTRexScenario.router_image = running_image
97
98            if self.modes:
99                print(termstyle.green('\t!!!\tRunning with modes: %s, not suitable tests will be skipped.\t!!!' % list(self.modes)))
100
101            CTRexScenario.is_init = True
102            print(termstyle.green("Done instantiating T-Rex scenario!\n"))
103
104#           raise RuntimeError('CTRexScenario class is not initialized!')
105        self.router = CTRexScenario.router
106
107
108
109#   def assert_dict_eq (self, dict, key, val, error=''):
110#           v1 = int(dict[key]))
111#           self.assertEqual(v1, int(val), error)
112#
113#   def assert_dict_gt (self, d, key, val, error=''):
114#           v1 = int(dict[key])
115#           self.assert_gt(v1, int(val), error)
116
117    def assertEqual(self, v1, v2, s):
118        if v1 != v2:
119            error='ERROR '+str(v1)+' !=  '+str(v2)+ '   '+s;
120            self.fail(error)
121
122    def assert_gt(self, v1, v2, s):
123        if not v1 > v2:
124            error='ERROR {big} <  {small}      {str}'.format(big = v1, small = v2, str = s)
125            self.fail(error)
126
127    def check_results_eq (self,res,name,val):
128        if res is None:
129            self.fail('TRex results cannot be None !')
130            return
131
132        if name not in res:
133            self.fail('TRex results does not include key %s' % name)
134            return
135
136        if res[name] != float(val):
137            self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val))
138
139    def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = 30, maximal_cpu = 85):
140            #cpu_util = float(trex_res.get_last_value("trex-global.data.m_cpu_util"))
141            cpu_util = sum(trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]) / 3.0 # mean of 3 values before last
142
143            if '1G' in self.modes:
144                minimal_cpu /= 10.0
145
146            if not self.is_virt_nics:
147                if cpu_util > maximal_cpu:
148                    self.fail("CPU is too high (%s%%), probably queue full." % cpu_util )
149                if cpu_util < minimal_cpu:
150                    self.fail("CPU is too low (%s%%), can't verify performance in such low CPU%%." % cpu_util )
151
152            cores = self.get_benchmark_param('cores')
153            ports_count = trex_res.get_ports_count()
154            trex_tx_bps  = sum(trex_res.get_value_list("trex-global.data.m_tx_bps")[-4:-1]) / 3.0
155            # x2 because each thread uses 2 ports and another x2 because each core can use 2 threads
156            test_norm_cpu = 2 * 2 * (100.0 / cpu_util) * trex_tx_bps / (ports_count * cores * 1e6)
157
158            print("TRex CPU utilization: %g%%, norm_cpu is : %g Mb/core" % (round(cpu_util), round(test_norm_cpu)))
159
160            expected_norm_cpu = self.get_benchmark_param('bw_per_core')
161            if not expected_norm_cpu:
162                expected_norm_cpu = 1
163            calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100)
164            print('Err percent: %s' % calc_error_precent)
165            if calc_error_precent > err:
166                self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu)))
167
168
169    def check_results_gt (self, res, name, val):
170        if res is None:
171            self.fail('TRex results canot be None !')
172            return
173
174        if name not in res:
175            self.fail('TRex results does not include key %s' % name)
176            return
177
178        if res[name]< float(val):
179            self.fail('TRex results[%s]<%f and not as expected greater than %f ' % (name, res[name], val))
180
181    def check_for_trex_crash(self):
182        pass
183
184    def get_benchmark_param (self, param, sub_param = None, test_name = None):
185        if not test_name:
186            test_name = self.get_name()
187        if test_name not in self.benchmark:
188            self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param))
189        if sub_param:
190            return self.benchmark[test_name][param].get(sub_param)
191        else:
192            return self.benchmark[test_name].get(param)
193
194    def check_general_scenario_results (self, trex_res, check_latency = True):
195
196        try:
197            # check if test is valid
198            if not trex_res.is_done_warmup():
199                self.fail('T-Rex did not reach warm-up situtaion. Results are not valid.')
200
201            # check history size is enough
202            if len(trex_res._history) < 5:
203                self.fail('T-Rex results list is too short. Increase the test duration or check unexpected stopping.')
204
205            # check T-Rex number of drops
206            trex_tx_pckt    = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
207            trex_drops      = trex_res.get_total_drops()
208            trex_drop_rate  = trex_res.get_drop_rate()
209            if ( trex_drops > 0.001 * trex_tx_pckt) and (trex_drop_rate > 0.0):     # deliberately mask kickoff drops when T-Rex first initiated
210                self.fail('Number of packet drops larger than 0.1% of all traffic')
211
212            # check queue full, queue drop, allocation error
213            m_total_alloc_error = trex_res.get_last_value("trex-global.data.m_total_alloc_error")
214            m_total_queue_full = trex_res.get_last_value("trex-global.data.m_total_queue_full")
215            m_total_queue_drop = trex_res.get_last_value("trex-global.data.m_total_queue_drop")
216            self.assert_gt(1000, m_total_alloc_error, 'Got allocation errors. (%s), please review multiplier and templates configuration.' % m_total_alloc_error)
217            self.assert_gt(1000, m_total_queue_drop, 'Too much queue_drop (%s), please review multiplier.' % m_total_queue_drop)
218
219            if self.is_VM:
220                allowed_queue_full = 10000 + trex_tx_pckt / 100
221            else:
222                allowed_queue_full = 1000 + trex_tx_pckt / 1000
223            self.assert_gt(allowed_queue_full, m_total_queue_full, 'Too much queue_full (%s), please review multiplier.' % m_total_queue_full)
224
225            # # check T-Rex expected counters
226            #trex_exp_rate = trex_res.get_expected_tx_rate().get('m_tx_expected_bps')
227            #assert trex_exp_rate is not None
228            #trex_exp_gbps = trex_exp_rate/(10**9)
229
230            if check_latency:
231                # check that max latency does not exceed 1 msec
232                if self.configuration.trex['trex_name'] == '10.56.217.210': # temporary workaround for latency issue in kiwi02, remove it ASAP. http://trex-tgn.cisco.com/youtrack/issue/trex-194
233                    allowed_latency = 8000
234                elif self.is_VM:
235                    allowed_latency = 9999999
236                else: # no excuses, check 1ms
237                    allowed_latency = 1000
238                if max(trex_res.get_max_latency().values()) > allowed_latency:
239                    self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency)
240
241                # check that avg latency does not exceed 1 msec
242                if self.is_VM:
243                    allowed_latency = 9999999
244                else: # no excuses, check 1ms
245                    allowed_latency = 1000
246                if max(trex_res.get_avg_latency().values()) > allowed_latency:
247                    self.fail('LatencyError: Average latency exceeds %s (usec)' % allowed_latency)
248
249            if not self.is_loopback:
250                # check router number of drops --> deliberately masked- need to be figured out!!!!!
251                pkt_drop_stats = self.router.get_drop_stats()
252#               assert pkt_drop_stats['total_drops'] < 20
253
254                # check for trex-router packet consistency
255                # TODO: check if it's ok
256                print('router drop stats: %s' % pkt_drop_stats)
257                print('TRex drop stats: %s' % trex_drops)
258                #self.assertEqual(pkt_drop_stats, trex_drops, "TRex's and router's drop stats don't match.")
259
260        except KeyError as e:
261            self.fail(e)
262            #assert False
263
264        # except AssertionError as e:
265        #     e.args += ('T-Rex has crashed!')
266        #     raise
267
268    def unzip_client_package(self):
269        client_pkg_files = glob('%s/trex_client*.tar.gz' % CTRexScenario.scripts_path)
270        if not len(client_pkg_files):
271            raise Exception('Could not find client package')
272        if len(client_pkg_files) > 1:
273            raise Exception('Found more than one client packages')
274        client_pkg_name = os.path.basename(client_pkg_files[0])
275        if not os.path.exists('%s/trex_client' % CTRexScenario.scripts_path):
276            print('\nUnzipping package')
277            return_code, _, stderr = misc_methods.run_command("sh -ec 'cd %s; tar -xzf %s'" % (CTRexScenario.scripts_path, client_pkg_name))
278            if return_code:
279                raise Exception('Could not untar the client package: %s' % stderr)
280        else:
281            print('\nClient package is untarred')
282
283    # We encountered error, don't fail the test immediately
284    def fail(self, reason = 'Unknown error'):
285        print('Error: %s' % reason)
286        self.fail_reasons.append(reason)
287
288    # skip running of the test, counts as 'passed' but prints 'skipped'
289    def skip(self, message = 'Unknown reason'):
290        print('Skip: %s' % message)
291        self.skipping = True
292        raise SkipTest(message)
293
294    # get name of currently running test
295    def get_name(self):
296        return self._testMethodName
297
298    def setUp(self):
299        test_setup_modes_conflict = self.modes & set(self.unsupported_modes)
300        if test_setup_modes_conflict:
301            self.skip("The test can't run with following modes of given setup: %s " % test_setup_modes_conflict)
302        if self.trex and not self.trex.is_idle():
303            print('Warning: TRex is not idle at setUp, trying to stop it.')
304            self.trex.force_kill(confirm = False)
305        if not self.is_loopback:
306            print('')
307            if self.trex: # stateful
308                self.router.load_clean_config()
309            self.router.clear_counters()
310            self.router.clear_packet_drop_stats()
311
312    ########################################################################
313    ####                DO NOT ADD TESTS TO THIS FILE                   ####
314    ####    Added tests here will held once for EVERY test sub-class    ####
315    ########################################################################
316
317    # masked example to such test. uncomment to watch how it affects #
318#   def test_isInitialized(self):
319#       assert CTRexScenario.is_init == True
320    def tearDown(self):
321        if self.trex and not self.trex.is_idle():
322            print('Warning: TRex is not idle at tearDown, trying to stop it.')
323            self.trex.force_kill(confirm = False)
324        if not self.skipping:
325            # print server logs of test run
326            if self.trex and CTRexScenario.server_logs:
327                try:
328                    print(termstyle.green('\n>>>>>>>>>>>>>>> Daemon log <<<<<<<<<<<<<<<'))
329                    daemon_log = self.trex.get_trex_daemon_log()
330                    log_size = len(daemon_log)
331                    print(''.join(daemon_log[CTRexScenario.daemon_log_lines:]))
332                    CTRexScenario.daemon_log_lines = log_size
333                except Exception as e:
334                    print("Can't get TRex daemon log:", e)
335                try:
336                    print(termstyle.green('>>>>>>>>>>>>>>>> Trex log <<<<<<<<<<<<<<<<'))
337                    print(''.join(self.trex.get_trex_log()))
338                except Exception as e:
339                    print("Can't get TRex log:", e)
340            if len(self.fail_reasons):
341                sys.stdout.flush()
342                raise Exception('The test is failed, reasons:\n%s' % '\n'.join(self.fail_reasons))
343        sys.stdout.flush()
344
345    def check_for_trex_crash(self):
346        pass
347