stl_basic_tests.py revision 2cf44419
1
2import outer_packages
3from platform_cmd_link import *
4import functional_general_test
5from nose.tools import assert_equal
6from nose.tools import assert_not_equal
7from nose.tools import nottest
8from nose.plugins.attrib import attr
9from trex import CTRexScenario
10from trex_stl_lib import trex_stl_sim
11from trex_stl_lib.trex_stl_streams import STLProfile
12from trex_stl_lib.trex_stl_packet_builder_scapy import RawPcapReader, RawPcapWriter, Ether
13from trex_stl_lib.utils.text_opts import *
14
15import sys
16
17if sys.version_info > (3,0):
18    from io import StringIO
19else:
20    from cStringIO import StringIO
21
22import os
23import subprocess
24import shlex
25from threading import Thread
26from collections import defaultdict
27
28@attr('run_on_trex')
29class CStlBasic_Test(functional_general_test.CGeneralFunctional_Test):
30    def setUp (self):
31        self.test_path = os.path.abspath(os.getcwd())
32        self.scripts_path = CTRexScenario.scripts_path
33
34        self.verify_exists(os.path.join(self.scripts_path, "bp-sim-64-debug"))
35
36        self.stl_sim = os.path.join(self.scripts_path, "stl-sim")
37
38        self.verify_exists(self.stl_sim)
39
40        self.profiles_path = os.path.join(self.scripts_path, "stl/yaml/")
41
42        self.profiles = {}
43        self.profiles['imix_3pkt'] = os.path.join(self.profiles_path, "imix_3pkt.yaml")
44        self.profiles['imix_3pkt_vm'] = os.path.join(self.profiles_path, "imix_3pkt_vm.yaml")
45        self.profiles['random_size_9k'] = os.path.join(self.profiles_path, "../udp_rand_len_9k.py")
46        self.profiles['imix_tuple_gen'] = os.path.join(self.profiles_path, "imix_1pkt_tuple_gen.yaml")
47
48        for k, v in self.profiles.items():
49            self.verify_exists(v)
50
51        self.valgrind_profiles = [ self.profiles['imix_3pkt_vm'],
52                                   self.profiles['random_size_9k'],
53                                   self.profiles['imix_tuple_gen'] ]
54
55        self.golden_path = os.path.join(self.test_path,"stl/golden/")
56
57        os.chdir(self.scripts_path)
58
59
60    def tearDown (self):
61        os.chdir(self.test_path)
62
63
64
65    def get_golden (self, name):
66        golden = os.path.join(self.golden_path, name)
67        self.verify_exists(golden)
68        return golden
69
70
71    def verify_exists (self, name):
72        if not os.path.exists(name):
73            raise Exception("cannot find '{0}'".format(name))
74
75
76    def scapy_pkt_show_to_str (self, scapy_pkt):
77        capture = StringIO()
78        save_stdout = sys.stdout
79        sys.stdout = capture
80        scapy_pkt.show()
81        sys.stdout = save_stdout
82        return capture.getvalue()
83
84
85    def compare_caps (self, output, golden, max_diff_sec = 0.01):
86        pkts1 = []
87        pkts2 = []
88        pkts_ts_buckets = defaultdict(list)
89
90        for pkt in RawPcapReader(output):
91            ts = pkt[1][0] * 1e6 + pkt[1][1]
92            pkts_ts_buckets[ts].append(pkt)
93        # don't take last ts bucket, it can be cut in middle and packets inside bucket might be different
94        #for ts in sorted(pkts_ts_buckets.keys())[:-1]:
95        for ts in sorted(pkts_ts_buckets.keys()):
96            pkts1.extend(sorted(pkts_ts_buckets[ts]))
97        pkts_ts_buckets.clear()
98
99        for pkt in RawPcapReader(golden):
100            ts = pkt[1][0] * 1e6 + pkt[1][1]
101            pkts_ts_buckets[ts].append(pkt)
102        # don't take last ts bucket, it can be cut in middle and packets inside bucket might be different
103        #for ts in sorted(pkts_ts_buckets.keys())[:-1]:
104        for ts in sorted(pkts_ts_buckets.keys()):
105            pkts2.extend(sorted(pkts_ts_buckets[ts]))
106
107        assert_equal(len(pkts1), len(pkts2), 'Lengths of generated pcap (%s) and golden (%s) are different' % (output, golden))
108
109        for pkt1, pkt2, i in zip(pkts1, pkts2, range(1, len(pkts1))):
110            ts1 = float(pkt1[1][0]) + (float(pkt1[1][1]) / 1e6)
111            ts2 = float(pkt2[1][0]) + (float(pkt2[1][1]) / 1e6)
112
113            if abs(ts1-ts2) > 0.000005: # 5 nsec
114                raise AssertionError("TS error: cap files '{0}', '{1}' differ in cap #{2} - '{3}' vs. '{4}'".format(output, golden, i, ts1, ts2))
115
116            if pkt1[0] != pkt2[0]:
117                errmsg = "RAW error: output file '{0}', differs from golden '{1}' in cap #{2}".format(output, golden, i)
118                print(errmsg)
119
120                print(format_text("\ndifferent fields for packet #{0}:".format(i), 'underline'))
121
122                scapy_pkt1_info = self.scapy_pkt_show_to_str(Ether(pkt1[0])).split('\n')
123                scapy_pkt2_info = self.scapy_pkt_show_to_str(Ether(pkt2[0])).split('\n')
124
125                print(format_text("\nGot:\n", 'bold', 'underline'))
126                for line, ref in zip(scapy_pkt1_info, scapy_pkt2_info):
127                    if line != ref:
128                        print(format_text(line, 'bold'))
129
130                print(format_text("\nExpected:\n", 'bold', 'underline'))
131                for line, ref in zip(scapy_pkt2_info, scapy_pkt1_info):
132                    if line != ref:
133                        print(format_text(line, 'bold'))
134
135                print("\n")
136                raise AssertionError(errmsg)
137
138
139    def run_sim (self, yaml, output, options = "", silent = False, obj = None, tunables = None):
140        if output:
141            user_cmd = "-f {0} -o {1} {2} -p {3}".format(yaml, output, options, self.scripts_path)
142        else:
143            user_cmd = "-f {0} {1} -p {2}".format(yaml, options, self.scripts_path)
144
145        if silent:
146            user_cmd += " --silent"
147
148        if tunables:
149            user_cmd += " -t"
150            for k, v in tunables.items():
151                user_cmd += " {0}={1}".format(k, v)
152
153        rc = trex_stl_sim.main(args = shlex.split(user_cmd))
154        if obj:
155            obj['rc'] = (rc == 0)
156
157        return (rc == 0)
158
159
160
161    def run_py_profile_path (self,
162                             profile,
163                             options,
164                             silent = False,
165                             do_no_remove = False,
166                             compare = True,
167                             test_generated = True,
168                             do_no_remove_generated = False,
169                             tunables = None):
170
171        print('Testing profile: %s' % profile)
172        output_cap = "a.pcap"
173        input_file =  os.path.join('stl/', profile)
174        golden_file = os.path.join('exp',os.path.basename(profile).split('.')[0]+'.pcap');
175        if os.path.exists(output_cap):
176            os.unlink(output_cap)
177        try:
178            rc = self.run_sim(yaml     = input_file,
179                              output   = output_cap,
180                              options  = options,
181                              silent   = silent,
182                              tunables = tunables)
183            assert_equal(rc, True, 'Simulation on profile %s failed.' % profile)
184            #s='cp  '+output_cap+' '+golden_file;
185            #print s
186            #os.system(s)
187
188            if compare:
189                self.compare_caps(output_cap, golden_file)
190        finally:
191            if not do_no_remove:
192                os.unlink(output_cap)
193
194        if test_generated:
195            try:
196                generated_filename = input_file.replace('.py', '_GENERATED.py').replace('.yaml', '_GENERATED.py')
197                if input_file.endswith('.py'):
198                    profile = STLProfile.load_py(input_file, **(tunables if tunables else {}))
199                elif input_file.endswith('.yaml'):
200                    profile = STLProfile.load_yaml(input_file)
201
202                profile.dump_to_code(generated_filename)
203
204                rc = self.run_sim(yaml     = generated_filename,
205                                  output   = output_cap,
206                                  options  = options,
207                                  silent   = silent)
208                assert_equal(rc, True, 'Simulation on profile %s (generated) failed.' % profile)
209
210                if compare:
211                    self.compare_caps(output_cap, golden_file)
212
213
214            finally:
215                if not do_no_remove_generated:
216                    os.unlink(generated_filename)
217                    # python 3 does not generate PYC under the same dir
218                    if os.path.exists(generated_filename + 'c'):
219                        os.unlink(generated_filename + 'c')
220                if not do_no_remove:
221                    os.unlink(output_cap)
222
223
224    def test_stl_profiles (self):
225        p = [
226             ["udp_1pkt_1mac_override.py","-m 1 -l 50",True],
227             ["syn_attack.py","-m 1 -l 50",True],
228             ["udp_1pkt_1mac.py","-m 1 -l 50",True],
229             ["udp_1pkt_mac.py","-m 1 -l 50",True],
230             ["udp_1pkt.py","-m 1 -l 50",True],
231             ["udp_1pkt_tuple_gen.py","-m 1 -l 50",True],
232             ["udp_rand_len_9k.py","-m 1 -l 50",True],
233             ["udp_1pkt_mpls.py","-m 1 -l 50",True],
234             ["udp_1pkt_mpls_vm.py","-m 1 ",True],
235             ["imix.py","-m 1 -l 100",True],
236             ["udp_inc_len_9k.py","-m 1 -l 100",True],
237             ["udp_1pkt_range_clients.py","-m 1 -l 100",True],
238             ["multi_burst_2st_1000pkt.py","-m 1 -l 100",True],
239             ["pcap.py", "-m 1", True, False],
240             ["pcap_with_vm.py", "-m 1", True, False],
241             ["flow_stats.py", "-m 1 -l 1", True],
242             ["flow_stats_latency.py", "-m 1 -l 1", True],
243
244            # YAML test
245             ["yaml/burst_1000_pkt.yaml","-m 1 -l 100",True],
246             ["yaml/burst_1pkt_1burst.yaml","-m 1 -l 100",True],
247             ["yaml/burst_1pkt_vm.yaml","-m 1 -l 100",True],
248             ["yaml/imix_1pkt.yaml","-m 1 -l 100",True],
249             ["yaml/imix_1pkt_2.yaml","-m 1 -l 100",True],
250             ["yaml/imix_1pkt_tuple_gen.yaml","-m 1 -l 100",True],
251             ["yaml/imix_1pkt_vm.yaml","-m 1 -l 100",True],
252             ["udp_1pkt_pcap.py","-m 1 -l 10",True, False],
253             ["udp_3pkt_pcap.py","-m 1 -l 10",True, False],
254             #["udp_1pkt_simple.py","-m 1 -l 3",True],
255             ["udp_1pkt_pcap_relative_path.py","-m 1 -l 3",True, False],
256             ["udp_1pkt_tuple_gen_split.py","-m 1 -l 100",True],
257             ["udp_1pkt_range_clients_split.py","-m 1 -l 100",True],
258             ["udp_1pkt_vxlan.py","-m 1 -l 17",True, False], # can't generate: no VXLAN in Scapy, only in profile
259             ["udp_1pkt_ipv6_in_ipv4.py","-m 1 -l 17",True],
260             ["yaml/imix_3pkt.yaml","-m 50kpps --limit 20",True],
261             ["yaml/imix_3pkt_vm.yaml","-m 50kpps --limit 20",True],
262             ["udp_1pkt_simple_mac_dst.py","-m 1 -l 1 ",True],
263             ["udp_1pkt_simple_mac_src.py","-m 1 -l 1 ",True],
264             ["udp_1pkt_simple_mac_dst_src.py","-m 1 -l 1 ",True],
265             ["burst_3st_loop_x_times.py","-m 1 -l 20 ",True],
266             ["udp_1pkt_mac_step.py","-m 1 -l 20 ",True],
267             ["udp_1pkt_mac_mask1.py","-m 1 -l 20 ",True] ,
268             ["udp_1pkt_mac_mask2.py","-m 1 -l 20 ",True],
269             ["udp_1pkt_mac_mask3.py","-m 1 -l 20 ",True],
270             ["udp_1pkt_simple_test2.py","-m 1 -l 10 ",True, False], # test split of packet with ip option
271             ["udp_1pkt_simple_test.py","-m 1 -l 10 ",True, False],
272             ["udp_1pkt_mac_mask5.py","-m 1 -l 30 ",True],
273             ["udp_1pkt_range_clients_split_garp.py","-m 1 -l 50",True],
274             ["udp_1pkt_src_ip_split.py","-m 1 -l 50",True],
275             ["udp_1pkt_repeat_random.py","-m 1 -l 50",True],
276          ];
277
278        p1 = [ ["udp_1pkt_repeat_random.py","-m 1 -l 50",True] ];
279
280        for obj in p:
281            try:
282                test_generated = obj[3]
283            except: # check generated if not said otherwise
284                test_generated = True
285            self.run_py_profile_path (obj[0],obj[1],compare =obj[2], test_generated = test_generated, do_no_remove=True, do_no_remove_generated = False)
286
287
288    def test_hlt_profiles (self):
289        p = (
290            ['hlt/hlt_udp_inc_dec_len_9k.py', '-m 1 -l 20', True, None],
291            ['hlt/hlt_imix_default.py', '-m 1 -l 20', True, None],
292            ['hlt/hlt_imix_4rates.py', '-m 1 -l 20', True, None],
293            ['hlt/hlt_david1.py', '-m 1 -l 20', True, None],
294            ['hlt/hlt_david2.py', '-m 1 -l 20', True, None],
295            ['hlt/hlt_david3.py', '-m 1 -l 20', True, None],
296            ['hlt/hlt_david4.py', '-m 1 -l 20', True, None],
297            ['hlt/hlt_wentong1.py', '-m 1 -l 20', True, None],
298            ['hlt/hlt_wentong2.py', '-m 1 -l 20', True, None],
299            ['hlt/hlt_tcp_ranges.py', '-m 1 -l 20', True, None],
300            ['hlt/hlt_udp_ports.py', '-m 1 -l 20', True, None],
301            ['hlt/hlt_udp_random_ports.py', '-m 1 -l 20', True, None],
302            ['hlt/hlt_ip_ranges.py', '-m 1 -l 20', True, None],
303            ['hlt/hlt_framesize_vm.py', '-m 1 -l 20', True, None],
304            ['hlt/hlt_l3_length_vm.py', '-m 1 -l 20', True, None],
305            ['hlt/hlt_vlan_default.py', '-m 1 -l 20', True, None],
306            ['hlt/hlt_4vlans.py', '-m 1 -l 20', True, None],
307            ['hlt/hlt_vlans_vm.py', '-m 1 -l 20', True, {'random_seed': 1}],
308            ['hlt/hlt_ipv6_default.py', '-m 1 -l 20', True, None],
309            ['hlt/hlt_ipv6_ranges.py', '-m 1 -l 20', True, None],
310            ['hlt/hlt_mac_ranges.py', '-m 1 -l 20', True, None],
311            )
312
313        for obj in p:
314            self.run_py_profile_path (obj[0], obj[1], compare =obj[2], do_no_remove=True, do_no_remove_generated = False, tunables = obj[3])
315
316    # valgrind tests - this runs in multi thread as it safe (no output)
317    def test_valgrind_various_profiles (self):
318        print("\n")
319        threads = []
320        for profile in self.valgrind_profiles:
321            print("\n*** VALGRIND: testing profile '{0}' ***\n".format(profile))
322            obj = {'t': None, 'rc': None}
323            t = Thread(target = self.run_sim,
324                       kwargs = {'obj': obj, 'yaml': profile, 'output':None, 'options': "--cores 8 --limit 20 --valgrind", 'silent': True})
325            obj['t'] = t
326
327            threads.append(obj)
328            t.start()
329
330        for obj in threads:
331            obj['t'].join()
332
333        for obj in threads:
334            assert_equal(obj['rc'], True)
335
336
337
338    def test_multicore_scheduling (self):
339
340        seed = time.time()
341
342        # test with simple vars
343        rc = self.run_sim('stl/tests/multi_core_test.py', output = None, options = '--test_multi_core --limit=840 -t test_type=plain#seed={0} -m 27kpps'.format(seed), silent = True)
344        assert_equal(rc, True)
345
346
347        # test with tuple
348        rc = self.run_sim('stl/tests/multi_core_test.py', output = None, options = '--test_multi_core --limit=840 -t test_type=tuple#seed={0} -m 27kpps'.format(seed), silent = True)
349        assert_equal(rc, True)
350
351        # some tests
352        mc_tests = [
353                    'stl/tests/single_cont.py',
354                    'stl/tests/single_burst.py',
355                    'stl/tests/multi_burst.py',
356                   ]
357
358        for mc_test in mc_tests:
359            rc = self.run_sim(mc_test, output = None, options = '--test_multi_core --limit=840 -m 27kpps', silent = True)
360            assert_equal(rc, True)
361
362        return
363
364
365