12258ea30SYaroslav Brustinov#!/router/bin/python
22258ea30SYaroslav Brustinov
32258ea30SYaroslav Brustinov__copyright__ = "Copyright 2014"
42258ea30SYaroslav Brustinov
52258ea30SYaroslav Brustinov"""
62258ea30SYaroslav BrustinovName:
72258ea30SYaroslav Brustinov     trex_general_test.py
82258ea30SYaroslav Brustinov
92258ea30SYaroslav Brustinov
102258ea30SYaroslav BrustinovDescription:
112258ea30SYaroslav Brustinov
12d279c8c1SYaroslav Brustinov    This script creates the functionality to test the performance of the TRex traffic generator
13d279c8c1SYaroslav Brustinov    The tested scenario is a TRex TG directly connected to a Cisco router.
142258ea30SYaroslav Brustinov
152258ea30SYaroslav Brustinov::
162258ea30SYaroslav Brustinov
172258ea30SYaroslav Brustinov    Topology:
182258ea30SYaroslav Brustinov
192258ea30SYaroslav Brustinov       -------                         --------
202258ea30SYaroslav Brustinov      |       | Tx---1gig/10gig----Rx |        |
21d279c8c1SYaroslav Brustinov      | TRex  |                       | router |
222258ea30SYaroslav Brustinov      |       | Rx---1gig/10gig----Tx |        |
232258ea30SYaroslav Brustinov       -------                         --------
242258ea30SYaroslav Brustinov
252258ea30SYaroslav Brustinov"""
262258ea30SYaroslav Brustinovfrom nose.plugins import Plugin
272258ea30SYaroslav Brustinovfrom nose.plugins.skip import SkipTest
282258ea30SYaroslav Brustinovimport trex
292258ea30SYaroslav Brustinovfrom trex import CTRexScenario
302258ea30SYaroslav Brustinovimport misc_methods
31420216e5SHanoh Haimimport pprint
322258ea30SYaroslav Brustinovimport sys
332258ea30SYaroslav Brustinovimport os
342258ea30SYaroslav Brustinov# from CPlatformUnderTest import *
352258ea30SYaroslav Brustinovfrom CPlatform import *
362258ea30SYaroslav Brustinovimport termstyle
372258ea30SYaroslav Brustinovimport threading
38501fb3b4SYaroslav Brustinovfrom .tests_exceptions import *
392258ea30SYaroslav Brustinovfrom platform_cmd_link import *
402258ea30SYaroslav Brustinovimport unittest
4182a3c6f2SYaroslav Brustinovfrom glob import glob
428c3c9822SYaroslav Brustinovfrom datetime import datetime, timedelta
432258ea30SYaroslav Brustinov
442258ea30SYaroslav Brustinovdef setUpModule(module):
452258ea30SYaroslav Brustinov    pass
462258ea30SYaroslav Brustinov
472258ea30SYaroslav Brustinovdef tearDownModule(module):
482258ea30SYaroslav Brustinov    pass
492258ea30SYaroslav Brustinov
502258ea30SYaroslav Brustinovclass CTRexGeneral_Test(unittest.TestCase):
51d279c8c1SYaroslav Brustinov    """This class defines the general stateful testcase of the TRex traffic generator"""
522258ea30SYaroslav Brustinov    def __init__ (self, *args, **kwargs):
53ba14992cSYaroslav Brustinov        sys.stdout.flush()
542258ea30SYaroslav Brustinov        unittest.TestCase.__init__(self, *args, **kwargs)
552258ea30SYaroslav Brustinov        if CTRexScenario.is_test_list:
562258ea30SYaroslav Brustinov            return
572258ea30SYaroslav Brustinov        # Point test object to scenario global object
582258ea30SYaroslav Brustinov        self.configuration         = CTRexScenario.configuration
592258ea30SYaroslav Brustinov        self.benchmark             = CTRexScenario.benchmark
602258ea30SYaroslav Brustinov        self.trex                  = CTRexScenario.trex
6189b608aeSYaroslav Brustinov        self.stl_trex              = CTRexScenario.stl_trex
622258ea30SYaroslav Brustinov        self.trex_crashed          = CTRexScenario.trex_crashed
632258ea30SYaroslav Brustinov        self.modes                 = CTRexScenario.modes
6489b608aeSYaroslav Brustinov        self.GAManager             = CTRexScenario.GAManager
65420216e5SHanoh Haim        self.elk                   = CTRexScenario.elk
66fad99b60SYaroslav Brustinov        self.no_daemon             = CTRexScenario.no_daemon
672258ea30SYaroslav Brustinov        self.skipping              = False
682258ea30SYaroslav Brustinov        self.fail_reasons          = []
692258ea30SYaroslav Brustinov        if not hasattr(self, 'unsupported_modes'):
702258ea30SYaroslav Brustinov            self.unsupported_modes   = []
712258ea30SYaroslav Brustinov        self.is_loopback           = True if 'loopback' in self.modes else False
722258ea30SYaroslav Brustinov        self.is_virt_nics          = True if 'virt_nics' in self.modes else False
738bd778d0SYaroslav Brustinov        self.is_vf_nics            = True if 'vf_nics' in self.modes else False
742258ea30SYaroslav Brustinov        self.is_VM                 = True if 'VM' in self.modes else False
752258ea30SYaroslav Brustinov
76c376394cSYaroslav Brustinov        if not CTRexScenario.is_init:
77fad99b60SYaroslav Brustinov            if self.trex and not self.no_daemon: # stateful
782258ea30SYaroslav Brustinov                CTRexScenario.trex_version = self.trex.get_trex_version()
7921760765SYaroslav Brustinov            #update elk const object
8021760765SYaroslav Brustinov            if self.elk:
8121760765SYaroslav Brustinov                timediff  = timedelta(hours=2) # workaround to get IL timezone
8221760765SYaroslav Brustinov                date_str  = CTRexScenario.trex_version['Date'].strip()
8321760765SYaroslav Brustinov                timestamp = datetime.strptime(date_str, '%b %d %Y , %H:%M:%S') - timediff
8421760765SYaroslav Brustinov
8521760765SYaroslav Brustinov                img               = CTRexScenario.elk_info['info']['image']
8621760765SYaroslav Brustinov                img['sha']        = CTRexScenario.trex_version['Git SHA']
8721760765SYaroslav Brustinov                img['build_time'] = timestamp.strftime("%Y-%m-%d %H:%M:%S")
8821760765SYaroslav Brustinov                img['version']    = CTRexScenario.trex_version['Version']
8921760765SYaroslav Brustinov
9021760765SYaroslav Brustinov                setup = CTRexScenario.elk_info['info']['setup']
9121760765SYaroslav Brustinov                if self.is_loopback :
9221760765SYaroslav Brustinov                    setup['dut'] = 'loopback'
9321760765SYaroslav Brustinov                else:
9421760765SYaroslav Brustinov                    setup['dut'] = 'router'
9521760765SYaroslav Brustinov
9621760765SYaroslav Brustinov                if self.is_VM:
9721760765SYaroslav Brustinov                    setup['baremetal'] = False
9821760765SYaroslav Brustinov                    setup['hypervisor'] = 'ESXi'       #TBD
9921760765SYaroslav Brustinov                else:
10021760765SYaroslav Brustinov                    setup['baremetal'] = True
1012258ea30SYaroslav Brustinov            if not self.is_loopback:
1022258ea30SYaroslav Brustinov                # initilize the scenario based on received configuration, once per entire testing session
1032258ea30SYaroslav Brustinov                CTRexScenario.router = CPlatform(CTRexScenario.router_cfg['silent_mode'])
1042258ea30SYaroslav Brustinov                device_cfg           = CDeviceCfg()
1052258ea30SYaroslav Brustinov                device_cfg.set_platform_config(CTRexScenario.router_cfg['config_dict'])
1062258ea30SYaroslav Brustinov                device_cfg.set_tftp_config(CTRexScenario.router_cfg['tftp_config_dict'])
1072258ea30SYaroslav Brustinov                CTRexScenario.router.load_platform_data_from_file(device_cfg)
1082258ea30SYaroslav Brustinov                CTRexScenario.router.launch_connection(device_cfg)
1092258ea30SYaroslav Brustinov                if CTRexScenario.router_cfg['forceImageReload']:
110c376394cSYaroslav Brustinov                    image_d = CTRexScenario.router.get_running_image_details();
111120746edSHanoh Haim                    running_image = image_d['image']
112cacad4a8SYaroslav Brustinov                    print('Current router image: %s' % running_image)
113c376394cSYaroslav Brustinov                    if self.elk:
114c376394cSYaroslav Brustinov                        setup['dut'] = image_d.get('model','router');
115c376394cSYaroslav Brustinov                        print('Current router model : %s' % setup['dut'])
1162258ea30SYaroslav Brustinov                    needed_image = device_cfg.get_image_name()
1172258ea30SYaroslav Brustinov                    if not CTRexScenario.router.is_image_matches(needed_image):
118501fb3b4SYaroslav Brustinov                        print('Setting router image: %s' % needed_image)
1192258ea30SYaroslav Brustinov                        CTRexScenario.router.config_tftp_server(device_cfg)
1202258ea30SYaroslav Brustinov                        CTRexScenario.router.load_platform_image(needed_image)
1212258ea30SYaroslav Brustinov                        CTRexScenario.router.set_boot_image(needed_image)
1222258ea30SYaroslav Brustinov                        CTRexScenario.router.reload_platform(device_cfg)
1232258ea30SYaroslav Brustinov                        CTRexScenario.router.launch_connection(device_cfg)
1242258ea30SYaroslav Brustinov                        running_image = CTRexScenario.router.get_running_image_details()['image'] # verify image
1252258ea30SYaroslav Brustinov                        if not CTRexScenario.router.is_image_matches(needed_image):
1262258ea30SYaroslav Brustinov                            self.fail('Unable to set router image: %s, current image is: %s' % (needed_image, running_image))
1272258ea30SYaroslav Brustinov                    else:
128501fb3b4SYaroslav Brustinov                        print('Matches needed image: %s' % needed_image)
129cacad4a8SYaroslav Brustinov                    CTRexScenario.router_image = running_image
1302258ea30SYaroslav Brustinov
1312258ea30SYaroslav Brustinov            if self.modes:
132501fb3b4SYaroslav Brustinov                print(termstyle.green('\t!!!\tRunning with modes: %s, not suitable tests will be skipped.\t!!!' % list(self.modes)))
1332258ea30SYaroslav Brustinov
1342258ea30SYaroslav Brustinov            CTRexScenario.is_init = True
135d279c8c1SYaroslav Brustinov            print(termstyle.green("Done instantiating TRex scenario!\n"))
1362258ea30SYaroslav Brustinov
1372258ea30SYaroslav Brustinov#           raise RuntimeError('CTRexScenario class is not initialized!')
1382258ea30SYaroslav Brustinov        self.router = CTRexScenario.router
1392258ea30SYaroslav Brustinov
140420216e5SHanoh Haim    def get_elk_obj (self):
141420216e5SHanoh Haim        obj=trex.copy_elk_info ()
142420216e5SHanoh Haim        return (obj);
1432258ea30SYaroslav Brustinov
1442258ea30SYaroslav Brustinov#   def assert_dict_eq (self, dict, key, val, error=''):
1452258ea30SYaroslav Brustinov#           v1 = int(dict[key]))
1462258ea30SYaroslav Brustinov#           self.assertEqual(v1, int(val), error)
1472258ea30SYaroslav Brustinov#
1482258ea30SYaroslav Brustinov#   def assert_dict_gt (self, d, key, val, error=''):
1492258ea30SYaroslav Brustinov#           v1 = int(dict[key])
1502258ea30SYaroslav Brustinov#           self.assert_gt(v1, int(val), error)
1512258ea30SYaroslav Brustinov
1522258ea30SYaroslav Brustinov    def assertEqual(self, v1, v2, s):
1532258ea30SYaroslav Brustinov        if v1 != v2:
1542258ea30SYaroslav Brustinov            error='ERROR '+str(v1)+' !=  '+str(v2)+ '   '+s;
1552258ea30SYaroslav Brustinov            self.fail(error)
1562258ea30SYaroslav Brustinov
1572258ea30SYaroslav Brustinov    def assert_gt(self, v1, v2, s):
1582258ea30SYaroslav Brustinov        if not v1 > v2:
1592258ea30SYaroslav Brustinov            error='ERROR {big} <  {small}      {str}'.format(big = v1, small = v2, str = s)
1602258ea30SYaroslav Brustinov            self.fail(error)
1612258ea30SYaroslav Brustinov
1622258ea30SYaroslav Brustinov    def check_results_eq (self,res,name,val):
1632258ea30SYaroslav Brustinov        if res is None:
1642258ea30SYaroslav Brustinov            self.fail('TRex results cannot be None !')
1652258ea30SYaroslav Brustinov            return
1662258ea30SYaroslav Brustinov
1672258ea30SYaroslav Brustinov        if name not in res:
1682258ea30SYaroslav Brustinov            self.fail('TRex results does not include key %s' % name)
1692258ea30SYaroslav Brustinov            return
1702258ea30SYaroslav Brustinov
1712258ea30SYaroslav Brustinov        if res[name] != float(val):
1722258ea30SYaroslav Brustinov            self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val))
1732258ea30SYaroslav Brustinov
174a76479bcSYaroslav Brustinov    def check_CPU_benchmark (self, trex_res, err = 25, minimal_cpu = 10, maximal_cpu = 85):
175a76479bcSYaroslav Brustinov        cpu_util          = trex_res.get_avg_steady_state_value('trex-global.data.m_cpu_util_raw')
176a76479bcSYaroslav Brustinov        trex_tx_bps       = trex_res.get_avg_steady_state_value('trex-global.data.m_tx_bps')
177420216e5SHanoh Haim        trex_tx_pps       = trex_res.get_avg_steady_state_value('trex-global.data.m_tx_pps')
178a76479bcSYaroslav Brustinov        expected_norm_cpu = self.get_benchmark_param('bw_per_core')
179a76479bcSYaroslav Brustinov        cores             = self.get_benchmark_param('cores')
180a76479bcSYaroslav Brustinov        ports_count       = trex_res.get_ports_count()
181e1538ec8SHanoh Haim        total_dp_cores    = cores * (ports_count/2);
182af419f1dSYaroslav Brustinov        if not (cpu_util and ports_count and cores):
183af419f1dSYaroslav Brustinov            print("Can't calculate CPU benchmark, need to divide by zero: cpu util: %s, ports: %s, cores: %s" % (cpu_util, ports_count, cores))
184af419f1dSYaroslav Brustinov            test_norm_cpu = -1
185af419f1dSYaroslav Brustinov        else:
186af419f1dSYaroslav Brustinov            test_norm_cpu = trex_tx_bps / (cpu_util * ports_count * cores * 2.5e6)
187a76479bcSYaroslav Brustinov
188a76479bcSYaroslav Brustinov        if '1G' in self.modes:
189a76479bcSYaroslav Brustinov            minimal_cpu /= 10.0
190a76479bcSYaroslav Brustinov
191a76479bcSYaroslav Brustinov        if not self.is_virt_nics:
192a76479bcSYaroslav Brustinov            if cpu_util > maximal_cpu:
193a76479bcSYaroslav Brustinov                self.fail("CPU is too high (%s%%), probably queue full." % cpu_util )
194a76479bcSYaroslav Brustinov            #if cpu_util < minimal_cpu:
195a76479bcSYaroslav Brustinov            #    self.fail("CPU is too low (%s%%), can't verify performance in such low CPU%%." % cpu_util )
196a76479bcSYaroslav Brustinov
197a76479bcSYaroslav Brustinov        print("TRex CPU utilization: %g%%, norm_cpu is : %g Gb/core" % (round(cpu_util, 2), round(test_norm_cpu, 2)))
198af419f1dSYaroslav Brustinov        if test_norm_cpu < 0:
199af419f1dSYaroslav Brustinov            return
200a76479bcSYaroslav Brustinov
201a76479bcSYaroslav Brustinov        if not expected_norm_cpu:
202a76479bcSYaroslav Brustinov            expected_norm_cpu = 1
203a76479bcSYaroslav Brustinov
204a76479bcSYaroslav Brustinov        calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100)
205a76479bcSYaroslav Brustinov        print('Err percent: %s' % calc_error_precent)
206a76479bcSYaroslav Brustinov        #if calc_error_precent > err and cpu_util > 10:
207a76479bcSYaroslav Brustinov        #    self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu)))
208a76479bcSYaroslav Brustinov
209420216e5SHanoh Haim        trex_tx_gbps       = trex_tx_bps/1e9
210420216e5SHanoh Haim        trex_tx_mpps       = trex_tx_pps/1e6
211420216e5SHanoh Haim
212e1538ec8SHanoh Haim        trex_tx_gbps_pc  = trex_tx_gbps*100.0/(cpu_util*total_dp_cores);
213e1538ec8SHanoh Haim        trex_tx_mpps_pc  = trex_tx_mpps*100.0/(cpu_util*total_dp_cores)
214420216e5SHanoh Haim
215420216e5SHanoh Haim        trex_tx_pckt    = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
216420216e5SHanoh Haim        trex_drops      = trex_res.get_total_drops()
217420216e5SHanoh Haim        trex_drop_precent = trex_drops *100.0/trex_tx_pckt;
218420216e5SHanoh Haim
219a76479bcSYaroslav Brustinov        # report benchmarks
220420216e5SHanoh Haim        if self.elk:
221420216e5SHanoh Haim            elk_obj = self.get_elk_obj()
222420216e5SHanoh Haim            print("Reporting to elk")
223420216e5SHanoh Haim            elk_obj['test']={ "name" : self.get_name(),
224420216e5SHanoh Haim                        "type"  : "stateful",
225e1538ec8SHanoh Haim                        "cores" : total_dp_cores,
226420216e5SHanoh Haim                        "cpu%"  : cpu_util,
227420216e5SHanoh Haim                        "mpps" :  (trex_tx_mpps),
228420216e5SHanoh Haim                        "streams_count" :1,
229420216e5SHanoh Haim                        "mpps_pc" :  (trex_tx_mpps_pc),
230420216e5SHanoh Haim                        "gbps_pc" :  (trex_tx_gbps_pc),
231420216e5SHanoh Haim                        "gbps" :  (trex_tx_gbps),
232420216e5SHanoh Haim                        "kcps"  : (trex_res.get_last_value("trex-global.data.m_tx_cps")/1000.0),
233420216e5SHanoh Haim                        "avg-pktsize" : round((1000.0*trex_tx_gbps/(8.0*trex_tx_mpps))),
234420216e5SHanoh Haim                        "latecny" : { "min" : min(trex_res.get_min_latency().values()),
235420216e5SHanoh Haim                                      "max" : max(trex_res.get_max_latency().values()),
236420216e5SHanoh Haim                                      "avr" : max(trex_res.get_avg_latency().values()),
237420216e5SHanoh Haim                                      "jitter" : max(trex_res.get_jitter_latency().values()),
238420216e5SHanoh Haim                                      "max-win" : max(trex_res.get_avg_window_latency ().values()),
239420216e5SHanoh Haim                                      "drop-rate" :trex_drop_precent
240420216e5SHanoh Haim                                     }
241420216e5SHanoh Haim                };
242420216e5SHanoh Haim            pprint.pprint(elk_obj['test']);
243420216e5SHanoh Haim            self.elk.perf.push_data(elk_obj)
244420216e5SHanoh Haim
2452258ea30SYaroslav Brustinov
2462258ea30SYaroslav Brustinov    def check_results_gt (self, res, name, val):
2472258ea30SYaroslav Brustinov        if res is None:
2482258ea30SYaroslav Brustinov            self.fail('TRex results canot be None !')
2492258ea30SYaroslav Brustinov            return
2502258ea30SYaroslav Brustinov
2512258ea30SYaroslav Brustinov        if name not in res:
2522258ea30SYaroslav Brustinov            self.fail('TRex results does not include key %s' % name)
2532258ea30SYaroslav Brustinov            return
2542258ea30SYaroslav Brustinov
2552258ea30SYaroslav Brustinov        if res[name]< float(val):
2562258ea30SYaroslav Brustinov            self.fail('TRex results[%s]<%f and not as expected greater than %f ' % (name, res[name], val))
2572258ea30SYaroslav Brustinov
2582258ea30SYaroslav Brustinov    def check_for_trex_crash(self):
2592258ea30SYaroslav Brustinov        pass
2602258ea30SYaroslav Brustinov
261f34b7915SHanoh Haim    def get_benchmark_param (self, param, sub_param = None, test_name = None,default=None):
2622258ea30SYaroslav Brustinov        if not test_name:
2632258ea30SYaroslav Brustinov            test_name = self.get_name()
2642258ea30SYaroslav Brustinov        if test_name not in self.benchmark:
265f34b7915SHanoh Haim            if default ==None:
266f34b7915SHanoh Haim               self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param))
267f34b7915SHanoh Haim            else:
268f34b7915SHanoh Haim                return default
2692258ea30SYaroslav Brustinov        if sub_param:
2702258ea30SYaroslav Brustinov            return self.benchmark[test_name][param].get(sub_param)
2712258ea30SYaroslav Brustinov        else:
2722258ea30SYaroslav Brustinov            return self.benchmark[test_name].get(param)
2732258ea30SYaroslav Brustinov
2742258ea30SYaroslav Brustinov    def check_general_scenario_results (self, trex_res, check_latency = True):
2752258ea30SYaroslav Brustinov
2762258ea30SYaroslav Brustinov        try:
27748cb48c6SYaroslav Brustinov            # check history size is enough
27848cb48c6SYaroslav Brustinov            if len(trex_res._history) < 5:
27948cb48c6SYaroslav Brustinov                self.fail('TRex results list is too short. Increase the test duration or check unexpected stopping.')
28048cb48c6SYaroslav Brustinov
2812258ea30SYaroslav Brustinov            # check if test is valid
2822258ea30SYaroslav Brustinov            if not trex_res.is_done_warmup():
283d279c8c1SYaroslav Brustinov                self.fail('TRex did not reach warm-up situtaion. Results are not valid.')
2842258ea30SYaroslav Brustinov
28548cb48c6SYaroslav Brustinov            # check that BW is not much more than expected
28663c6f35aSYaroslav Brustinov            trex_exp_bps = trex_res.get_expected_tx_rate().get('m_tx_expected_bps') / 1e6
28763c6f35aSYaroslav Brustinov            trex_cur_bps = max(trex_res.get_value_list('trex-global.data.m_tx_bps')) / 1e6
28848cb48c6SYaroslav Brustinov
28963c6f35aSYaroslav Brustinov            assert trex_exp_bps > 0, 'Expected BPS is zero: %s' % trex_exp_bps
29048cb48c6SYaroslav Brustinov
29163c6f35aSYaroslav Brustinov            if trex_exp_bps * 1.05 + 10 < trex_cur_bps:
29263c6f35aSYaroslav Brustinov                msg = 'Got BW (%gMbps) that is %g%% more than expected (%gMbps)!' % (round(trex_cur_bps, 2), round(100.0 * trex_cur_bps / trex_exp_bps - 100, 2), round(trex_exp_bps, 2))
29363c6f35aSYaroslav Brustinov                print('WARNING: %s' % msg)
29463c6f35aSYaroslav Brustinov                #self.fail(msg)
2952258ea30SYaroslav Brustinov
296d279c8c1SYaroslav Brustinov            # check TRex number of drops
2972258ea30SYaroslav Brustinov            trex_tx_pckt    = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
2982258ea30SYaroslav Brustinov            trex_drops      = trex_res.get_total_drops()
2992258ea30SYaroslav Brustinov            trex_drop_rate  = trex_res.get_drop_rate()
300d279c8c1SYaroslav Brustinov            if ( trex_drops > 0.001 * trex_tx_pckt) and (trex_drop_rate > 0.0):     # deliberately mask kickoff drops when TRex first initiated
3012258ea30SYaroslav Brustinov                self.fail('Number of packet drops larger than 0.1% of all traffic')
3022258ea30SYaroslav Brustinov
3032258ea30SYaroslav Brustinov            # check queue full, queue drop, allocation error
3042258ea30SYaroslav Brustinov            m_total_alloc_error = trex_res.get_last_value("trex-global.data.m_total_alloc_error")
3052258ea30SYaroslav Brustinov            m_total_queue_full = trex_res.get_last_value("trex-global.data.m_total_queue_full")
3062258ea30SYaroslav Brustinov            m_total_queue_drop = trex_res.get_last_value("trex-global.data.m_total_queue_drop")
3072258ea30SYaroslav Brustinov            self.assert_gt(1000, m_total_alloc_error, 'Got allocation errors. (%s), please review multiplier and templates configuration.' % m_total_alloc_error)
3082258ea30SYaroslav Brustinov            self.assert_gt(1000, m_total_queue_drop, 'Too much queue_drop (%s), please review multiplier.' % m_total_queue_drop)
3092258ea30SYaroslav Brustinov
3102258ea30SYaroslav Brustinov            if self.is_VM:
3112258ea30SYaroslav Brustinov                allowed_queue_full = 10000 + trex_tx_pckt / 100
3122258ea30SYaroslav Brustinov            else:
3132258ea30SYaroslav Brustinov                allowed_queue_full = 1000 + trex_tx_pckt / 1000
3142258ea30SYaroslav Brustinov            self.assert_gt(allowed_queue_full, m_total_queue_full, 'Too much queue_full (%s), please review multiplier.' % m_total_queue_full)
3152258ea30SYaroslav Brustinov
316d279c8c1SYaroslav Brustinov            # # check TRex expected counters
3172258ea30SYaroslav Brustinov            #trex_exp_rate = trex_res.get_expected_tx_rate().get('m_tx_expected_bps')
3182258ea30SYaroslav Brustinov            #assert trex_exp_rate is not None
3192258ea30SYaroslav Brustinov            #trex_exp_gbps = trex_exp_rate/(10**9)
3202258ea30SYaroslav Brustinov
3212258ea30SYaroslav Brustinov            if check_latency:
3223fa5b5bcSYaroslav Brustinov                # check that max latency does not exceed 1 msec
3233fa5b5bcSYaroslav Brustinov                if self.configuration.trex['trex_name'] == '10.56.217.210': # temporary workaround for latency issue in kiwi02, remove it ASAP. http://trex-tgn.cisco.com/youtrack/issue/trex-194
3243fa5b5bcSYaroslav Brustinov                    allowed_latency = 8000
325b901fb41SIdo Barnea                elif self.is_VM or self.is_virt_nics:
3263fa5b5bcSYaroslav Brustinov                    allowed_latency = 9999999
3273fa5b5bcSYaroslav Brustinov                else: # no excuses, check 1ms
3283fa5b5bcSYaroslav Brustinov                    allowed_latency = 1000
3292258ea30SYaroslav Brustinov                if max(trex_res.get_max_latency().values()) > allowed_latency:
3302258ea30SYaroslav Brustinov                    self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency)
3316c31c053SIdo Barnea
3323fa5b5bcSYaroslav Brustinov                # check that avg latency does not exceed 1 msec
333b901fb41SIdo Barnea                if self.is_VM or self.is_virt_nics:
3343fa5b5bcSYaroslav Brustinov                    allowed_latency = 9999999
3353fa5b5bcSYaroslav Brustinov                else: # no excuses, check 1ms
3363fa5b5bcSYaroslav Brustinov                    allowed_latency = 1000
3372258ea30SYaroslav Brustinov                if max(trex_res.get_avg_latency().values()) > allowed_latency:
3382258ea30SYaroslav Brustinov                    self.fail('LatencyError: Average latency exceeds %s (usec)' % allowed_latency)
3392258ea30SYaroslav Brustinov
3406c31c053SIdo Barnea                ports_names = trex_res.get_last_value('trex-latecny-v2.data', 'port\-\d+')
3416c31c053SIdo Barnea                if not ports_names:
3426c31c053SIdo Barnea                    raise AbnormalResultError('Could not find ports info in TRex results, path: trex-latecny-v2.data.port-*')
3436c31c053SIdo Barnea                for port_name in ports_names:
344feda9e92SIdo Barnea                    path = 'trex-latecny-v2.data.%s.hist.cnt' % port_name
3456c31c053SIdo Barnea                    lat_count = trex_res.get_last_value(path)
3466c31c053SIdo Barnea                    if lat_count == 0:
3476c31c053SIdo Barnea                        self.fail('LatencyError: Number of latency packets received on %s is 0' % port_name)
3486c31c053SIdo Barnea
3492258ea30SYaroslav Brustinov            if not self.is_loopback:
3502258ea30SYaroslav Brustinov                # check router number of drops --> deliberately masked- need to be figured out!!!!!
3512258ea30SYaroslav Brustinov                pkt_drop_stats = self.router.get_drop_stats()
3522258ea30SYaroslav Brustinov#               assert pkt_drop_stats['total_drops'] < 20
3532258ea30SYaroslav Brustinov
3542258ea30SYaroslav Brustinov                # check for trex-router packet consistency
3552258ea30SYaroslav Brustinov                # TODO: check if it's ok
356501fb3b4SYaroslav Brustinov                print('router drop stats: %s' % pkt_drop_stats)
357501fb3b4SYaroslav Brustinov                print('TRex drop stats: %s' % trex_drops)
3582258ea30SYaroslav Brustinov                #self.assertEqual(pkt_drop_stats, trex_drops, "TRex's and router's drop stats don't match.")
3592258ea30SYaroslav Brustinov
3602258ea30SYaroslav Brustinov        except KeyError as e:
3612258ea30SYaroslav Brustinov            self.fail(e)
3622258ea30SYaroslav Brustinov            #assert False
3632258ea30SYaroslav Brustinov
3642258ea30SYaroslav Brustinov        # except AssertionError as e:
365d279c8c1SYaroslav Brustinov        #     e.args += ('TRex has crashed!')
3662258ea30SYaroslav Brustinov        #     raise
3672258ea30SYaroslav Brustinov
3688e230286SYaroslav Brustinov    @staticmethod
3698e230286SYaroslav Brustinov    def unzip_client_package():
370e75dc535SYaroslav Brustinov        client_pkg_files = glob('%s/trex_client*.tar.gz' % CTRexScenario.scripts_path)
371e75dc535SYaroslav Brustinov        if not len(client_pkg_files):
372e75dc535SYaroslav Brustinov            raise Exception('Could not find client package')
373e75dc535SYaroslav Brustinov        if len(client_pkg_files) > 1:
374e75dc535SYaroslav Brustinov            raise Exception('Found more than one client packages')
37582a3c6f2SYaroslav Brustinov        if not os.path.exists('%s/trex_client' % CTRexScenario.scripts_path):
37682a3c6f2SYaroslav Brustinov            print('\nUnzipping package')
377e75dc535SYaroslav Brustinov            return_code, _, stderr = misc_methods.run_command("tar -xzf %s -C %s" % (client_pkg_files[0], CTRexScenario.scripts_path))
378e75dc535SYaroslav Brustinov            if return_code:
379e75dc535SYaroslav Brustinov                raise Exception('Could not untar the client package: %s' % stderr)
38082a3c6f2SYaroslav Brustinov        else:
38182a3c6f2SYaroslav Brustinov            print('\nClient package is untarred')
38282a3c6f2SYaroslav Brustinov
3832258ea30SYaroslav Brustinov    # We encountered error, don't fail the test immediately
3842258ea30SYaroslav Brustinov    def fail(self, reason = 'Unknown error'):
385501fb3b4SYaroslav Brustinov        print('Error: %s' % reason)
3862258ea30SYaroslav Brustinov        self.fail_reasons.append(reason)
3872258ea30SYaroslav Brustinov
3882258ea30SYaroslav Brustinov    # skip running of the test, counts as 'passed' but prints 'skipped'
3892258ea30SYaroslav Brustinov    def skip(self, message = 'Unknown reason'):
390501fb3b4SYaroslav Brustinov        print('Skip: %s' % message)
3912258ea30SYaroslav Brustinov        self.skipping = True
3922258ea30SYaroslav Brustinov        raise SkipTest(message)
3932258ea30SYaroslav Brustinov
3942258ea30SYaroslav Brustinov    # get name of currently running test
3952258ea30SYaroslav Brustinov    def get_name(self):
3962258ea30SYaroslav Brustinov        return self._testMethodName
3972258ea30SYaroslav Brustinov
3982258ea30SYaroslav Brustinov    def setUp(self):
3992258ea30SYaroslav Brustinov        test_setup_modes_conflict = self.modes & set(self.unsupported_modes)
4002258ea30SYaroslav Brustinov        if test_setup_modes_conflict:
4012258ea30SYaroslav Brustinov            self.skip("The test can't run with following modes of given setup: %s " % test_setup_modes_conflict)
40289b608aeSYaroslav Brustinov        if not self.stl_trex and not self.trex.is_idle():
403501fb3b4SYaroslav Brustinov            print('Warning: TRex is not idle at setUp, trying to stop it.')
4042258ea30SYaroslav Brustinov            self.trex.force_kill(confirm = False)
4052258ea30SYaroslav Brustinov        if not self.is_loopback:
406501fb3b4SYaroslav Brustinov            print('')
407d3c7c398SYaroslav Brustinov            if not self.stl_trex and CTRexScenario.router_cfg['forceCleanConfig']:
4082258ea30SYaroslav Brustinov                self.router.load_clean_config()
4092258ea30SYaroslav Brustinov            self.router.clear_counters()
4102258ea30SYaroslav Brustinov            self.router.clear_packet_drop_stats()
4112258ea30SYaroslav Brustinov
4122258ea30SYaroslav Brustinov    ########################################################################
4132258ea30SYaroslav Brustinov    ####                DO NOT ADD TESTS TO THIS FILE                   ####
4142258ea30SYaroslav Brustinov    ####    Added tests here will held once for EVERY test sub-class    ####
4152258ea30SYaroslav Brustinov    ########################################################################
4162258ea30SYaroslav Brustinov
4172258ea30SYaroslav Brustinov    # masked example to such test. uncomment to watch how it affects #
4182258ea30SYaroslav Brustinov#   def test_isInitialized(self):
4192258ea30SYaroslav Brustinov#       assert CTRexScenario.is_init == True
4202258ea30SYaroslav Brustinov    def tearDown(self):
42189b608aeSYaroslav Brustinov        if not self.stl_trex and not self.trex.is_idle():
422501fb3b4SYaroslav Brustinov            print('Warning: TRex is not idle at tearDown, trying to stop it.')
4232258ea30SYaroslav Brustinov            self.trex.force_kill(confirm = False)
4242258ea30SYaroslav Brustinov        if not self.skipping:
4252258ea30SYaroslav Brustinov            # print server logs of test run
426fad99b60SYaroslav Brustinov            if self.trex and CTRexScenario.server_logs and not self.no_daemon:
4272258ea30SYaroslav Brustinov                try:
428501fb3b4SYaroslav Brustinov                    print(termstyle.green('\n>>>>>>>>>>>>>>> Daemon log <<<<<<<<<<<<<<<'))
4292258ea30SYaroslav Brustinov                    daemon_log = self.trex.get_trex_daemon_log()
4302258ea30SYaroslav Brustinov                    log_size = len(daemon_log)
431501fb3b4SYaroslav Brustinov                    print(''.join(daemon_log[CTRexScenario.daemon_log_lines:]))
4322258ea30SYaroslav Brustinov                    CTRexScenario.daemon_log_lines = log_size
4332258ea30SYaroslav Brustinov                except Exception as e:
434501fb3b4SYaroslav Brustinov                    print("Can't get TRex daemon log:", e)
4352258ea30SYaroslav Brustinov                try:
436501fb3b4SYaroslav Brustinov                    print(termstyle.green('>>>>>>>>>>>>>>>> Trex log <<<<<<<<<<<<<<<<'))
437501fb3b4SYaroslav Brustinov                    print(''.join(self.trex.get_trex_log()))
4382258ea30SYaroslav Brustinov                except Exception as e:
439501fb3b4SYaroslav Brustinov                    print("Can't get TRex log:", e)
4402258ea30SYaroslav Brustinov            if len(self.fail_reasons):
441ba14992cSYaroslav Brustinov                sys.stdout.flush()
4426c31c053SIdo Barnea                raise Exception('Test failed. Reasons:\n%s' % '\n'.join(self.fail_reasons))
443ba14992cSYaroslav Brustinov        sys.stdout.flush()
4442258ea30SYaroslav Brustinov
4452258ea30SYaroslav Brustinov    def check_for_trex_crash(self):
4462258ea30SYaroslav Brustinov        pass
447