stl_performance_test.py revision 34777033
1import os
2from .stl_general_test import CStlGeneral_Test, CTRexScenario
3from trex_stl_lib.api import *
4
5def avg (values):
6    return (sum(values) / float(len(values)))
7
8# performance report object
9class PerformanceReport(object):
10    GOLDEN_NORMAL  = 1
11    GOLDEN_FAIL    = 2
12    GOLDEN_BETTER  = 3
13
14    def __init__ (self,
15                  scenario,
16                  machine_name,
17                  core_count,
18                  avg_cpu,
19                  avg_gbps,
20                  avg_mpps,
21                  avg_gbps_per_core,
22                  avg_mpps_per_core,
23                  ):
24
25        self.scenario                = scenario
26        self.machine_name            = machine_name
27        self.core_count              = core_count
28        self.avg_cpu                 = avg_cpu
29        self.avg_gbps                = avg_gbps
30        self.avg_mpps                = avg_mpps
31        self.avg_gbps_per_core       = avg_gbps_per_core
32        self.avg_mpps_per_core       = avg_mpps_per_core
33
34    def show (self):
35
36        print("\n")
37        print("scenario:                                {0}".format(self.scenario))
38        print("machine name:                            {0}".format(self.machine_name))
39        print("DP core count:                           {0}".format(self.core_count))
40        print("average CPU:                             {0}".format(self.avg_cpu))
41        print("average Gbps:                            {0}".format(self.avg_gbps))
42        print("average Mpps:                            {0}".format(self.avg_mpps))
43        print("average pkt size (bytes):                {0}".format( (self.avg_gbps * 1000 / 8) / self.avg_mpps))
44        print("average Gbps per core (at 100% CPU):     {0}".format(self.avg_gbps_per_core))
45        print("average Mpps per core (at 100% CPU):     {0}".format(self.avg_mpps_per_core))
46
47
48    def check_golden (self, golden_mpps):
49        if self.avg_mpps_per_core < golden_mpps['min']:
50            return self.GOLDEN_FAIL
51
52        if self.avg_mpps_per_core > golden_mpps['max']:
53            return self.GOLDEN_BETTER
54
55        return self.GOLDEN_NORMAL
56
57
58
59class STLPerformance_Test(CStlGeneral_Test):
60    """Tests for stateless client"""
61
62    def setUp(self):
63        CStlGeneral_Test.setUp(self)
64
65        self.c = CTRexScenario.stl_trex
66        self.c.connect()
67        self.c.reset()
68
69
70
71    def tearDown (self):
72        CStlGeneral_Test.tearDown(self)
73
74
75    def build_perf_profile_vm (self, pkt_size, cache_size = None):
76        size = pkt_size - 4; # HW will add 4 bytes ethernet FCS
77        src_ip = '16.0.0.1'
78        dst_ip = '48.0.0.1'
79
80        base_pkt = Ether()/IP(src=src_ip,dst=dst_ip)/UDP(dport=12,sport=1025)
81        pad = max(0, size - len(base_pkt)) * 'x'
82
83        vm = STLScVmRaw( [   STLVmFlowVar ( "ip_src",  min_value="10.0.0.1", max_value="10.0.0.255", size=4, step=1,op="inc"),
84                             STLVmWrFlowVar (fv_name="ip_src", pkt_offset= "IP.src" ),
85                             STLVmFixIpv4(offset = "IP")
86                         ],
87                         cache_size = cache_size
88                        );
89
90        pkt = STLPktBuilder(pkt = base_pkt/pad, vm = vm)
91        return STLStream(packet = pkt, mode = STLTXCont())
92
93
94    def build_perf_profile_syn_attack (self, pkt_size):
95        size = pkt_size - 4; # HW will add 4 bytes ethernet FCS
96
97        # TCP SYN
98        base_pkt  = Ether()/IP(dst="48.0.0.1")/TCP(dport=80,flags="S")
99        pad = max(0, size - len(base_pkt)) * 'x'
100
101        # vm
102        vm = STLScVmRaw( [ STLVmFlowVar(name="ip_src",
103                                              min_value="16.0.0.0",
104                                              max_value="18.0.0.254",
105                                              size=4, op="random"),
106
107                            STLVmFlowVar(name="src_port",
108                                              min_value=1025,
109                                              max_value=65000,
110                                              size=2, op="random"),
111
112                           STLVmWrFlowVar(fv_name="ip_src", pkt_offset= "IP.src" ),
113
114                           STLVmFixIpv4(offset = "IP"), # fix checksum
115
116                           STLVmWrFlowVar(fv_name="src_port",
117                                                pkt_offset= "TCP.sport") # fix udp len
118
119                          ]
120                       )
121
122        pkt = STLPktBuilder(pkt = base_pkt,
123                            vm = vm)
124
125        return STLStream(packet = pkt,
126                         random_seed = 0x1234,# can be remove. will give the same random value any run
127                         mode = STLTXCont())
128
129
130
131    # single CPU, VM, no cache, 64 bytes
132    def test_performance_vm_single_cpu (self):
133        setup_cfg = self.get_benchmark_param('cfg')
134        scenario_cfg = {}
135
136        scenario_cfg['name']        = "VM - 64 bytes, single CPU"
137        scenario_cfg['streams']     = self.build_perf_profile_vm(64)
138        scenario_cfg['core_count']  = 1
139
140        scenario_cfg['mult']                 = setup_cfg['mult']
141        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
142
143
144
145        self.execute_single_scenario(scenario_cfg)
146
147
148    # single CPU, VM, cached, 64 bytes
149    def test_performance_vm_single_cpu_cached (self):
150        setup_cfg = self.get_benchmark_param('cfg')
151        scenario_cfg = {}
152
153        scenario_cfg['name']        = "VM - 64 bytes, single CPU, cache size 1024"
154        scenario_cfg['streams']     = self.build_perf_profile_vm(64, cache_size = 1024)
155        scenario_cfg['core_count']  = 1
156
157        scenario_cfg['mult']                 = setup_cfg['mult']
158        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
159
160        self.execute_single_scenario(scenario_cfg)
161
162
163    # single CPU, syn attack, 64 bytes
164    def test_performance_syn_attack_single_cpu (self):
165        setup_cfg = self.get_benchmark_param('cfg')
166        scenario_cfg = {}
167
168        scenario_cfg['name']        = "syn attack - 64 bytes, single CPU"
169        scenario_cfg['streams']     = self.build_perf_profile_syn_attack(64)
170        scenario_cfg['core_count']  = 1
171
172        scenario_cfg['mult']                 = setup_cfg['mult']
173        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
174
175        self.execute_single_scenario(scenario_cfg)
176
177
178    # two CPUs, VM, no cache, 64 bytes
179    def test_performance_vm_multi_cpus (self):
180        setup_cfg = self.get_benchmark_param('cfg')
181        scenario_cfg = {}
182
183        scenario_cfg['name']        = "VM - 64 bytes, two CPUs"
184        scenario_cfg['streams']     = self.build_perf_profile_vm(64)
185
186        scenario_cfg['core_count']           = setup_cfg['core_count']
187        scenario_cfg['mult']                 = setup_cfg['mult']
188        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
189
190        self.execute_single_scenario(scenario_cfg)
191
192
193
194    # two CPUs, VM, cached, 64 bytes
195    def test_performance_vm_multi_cpus_cached (self):
196        setup_cfg = self.get_benchmark_param('cfg')
197        scenario_cfg = {}
198
199        scenario_cfg['name']        = "VM - 64 bytes, single CPU, cache size 1024"
200        scenario_cfg['streams']     = self.build_perf_profile_vm(64, cache_size = 1024)
201
202
203        scenario_cfg['core_count']           = setup_cfg['core_count']
204        scenario_cfg['mult']                 = setup_cfg['mult']
205        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
206
207        self.execute_single_scenario(scenario_cfg)
208
209
210    # two CPUs, syn attack, 64 bytes
211    def test_performance_syn_attack_multi_cpus (self):
212        setup_cfg = self.get_benchmark_param('cfg')
213        scenario_cfg = {}
214
215        scenario_cfg['name']        = "syn attack - 64 bytes, two CPUs"
216        scenario_cfg['streams']     = self.build_perf_profile_syn_attack(64)
217
218        scenario_cfg['core_count']           = setup_cfg['core_count']
219        scenario_cfg['mult']                 = setup_cfg['mult']
220        scenario_cfg['mpps_per_core_golden'] = setup_cfg['mpps_per_core_golden']
221
222        self.execute_single_scenario(scenario_cfg)
223
224
225
226############################################# test's infra functions ###########################################
227
228    def execute_single_scenario (self, scenario_cfg, iterations = 4):
229        golden = scenario_cfg['mpps_per_core_golden']
230
231
232        for i in range(iterations, -1, -1):
233            report = self.execute_single_scenario_iteration(scenario_cfg)
234            rc = report.check_golden(golden)
235
236            if rc == PerformanceReport.GOLDEN_NORMAL:
237                return
238
239            if rc == PerformanceReport.GOLDEN_BETTER:
240                return
241
242            print("\n*** Measured Mpps per core '{0}' is lower than expected golden '{1} - re-running scenario...{2} attempts left".format(report.avg_mpps_per_core, scenario_cfg['mpps_per_core_golden'], i))
243
244        assert 0, "performance failure"
245
246
247
248
249    def execute_single_scenario_iteration (self, scenario_cfg):
250
251        print("\nExecuting performance scenario: '{0}'\n".format(scenario_cfg['name']))
252
253        self.c.reset(ports = [0])
254        self.c.add_streams(ports = [0], streams = scenario_cfg['streams'])
255
256        # use one core
257        core_mask = (2 ** scenario_cfg['core_count']) - 1
258        self.c.start(ports = [0], mult = scenario_cfg['mult'], core_mask = [core_mask])
259
260        # stablize
261        print("Step 1 - waiting for stabilization... (10 seconds)")
262        for _ in range(10):
263            time.sleep(1)
264            sys.stdout.write('.')
265            sys.stdout.flush()
266
267        print("\n")
268
269        samples = {'cpu' : [], 'bps': [], 'pps': []}
270
271        # let the server gather samples
272        print("Step 2 - Waiting for samples... (40 seconds)")
273
274        for i in range(0, 1):
275
276            # sample bps/pps
277            for _ in range(0, 20):
278                stats = self.c.get_stats(ports = 0)
279                samples['bps'].append(stats[0]['tx_bps'])
280                samples['pps'].append(stats[0]['tx_pps'])
281                time.sleep(1)
282                sys.stdout.write('.')
283                sys.stdout.flush()
284
285            # sample CPU per core
286            rc = self.c._transmit('get_utilization')
287            if not rc:
288                raise Exception(rc)
289
290            data = rc.data()['cpu']
291            # filter
292            data = [s for s in data if s['ports'][0] == 0]
293
294            assert len(data) == scenario_cfg['core_count'] , "sampling info does not match core count"
295
296            for s in data:
297                samples['cpu'] += s['history']
298
299
300        stats = self.c.get_stats(ports = 0)
301        self.c.stop(ports = [0])
302
303
304
305        avg_values = {k:avg(v) for k, v in samples.iteritems()}
306        avg_cpu  = avg_values['cpu'] * scenario_cfg['core_count']
307        avg_gbps = avg_values['bps'] / 1e9
308        avg_mpps = avg_values['pps'] / 1e6
309
310        avg_gbps_per_core = avg_gbps * (100.0 / avg_cpu)
311        avg_mpps_per_core = avg_mpps * (100.0 / avg_cpu)
312
313        report = PerformanceReport(scenario            =  scenario_cfg['name'],
314                                   machine_name        =  os.uname()[1],
315                                   core_count          =  scenario_cfg['core_count'],
316                                   avg_cpu             =  avg_cpu,
317                                   avg_gbps            =  avg_gbps,
318                                   avg_mpps            =  avg_mpps,
319                                   avg_gbps_per_core   =  avg_gbps_per_core,
320                                   avg_mpps_per_core   =  avg_mpps_per_core)
321
322
323        report.show()
324
325        print("")
326        golden = scenario_cfg['mpps_per_core_golden']
327        print("golden Mpps per core (at 100% CPU):      min: {0}, max {1}".format(golden['min'], golden['max']))
328
329
330        return report
331
332