1420216e5SHanoh Haimimport os
2420216e5SHanoh Haimimport outer_packages
3420216e5SHanoh Haimimport json
4420216e5SHanoh Haimimport pprint
5420216e5SHanoh Haimfrom elasticsearch import Elasticsearch
6420216e5SHanoh Haimfrom pprint import pprint
7420216e5SHanoh Haimfrom elasticsearch import helpers
8420216e5SHanoh Haimimport random
9420216e5SHanoh Haimimport datetime
10420216e5SHanoh Haim
11420216e5SHanoh Haim# one object example for perf
12420216e5SHanoh Haimdef create_one_object (build_id):
13420216e5SHanoh Haim    d={};
14420216e5SHanoh Haim
15420216e5SHanoh Haim    sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30));
16420216e5SHanoh Haim    info = {};
17420216e5SHanoh Haim
18420216e5SHanoh Haim
19420216e5SHanoh Haim    img={}
20420216e5SHanoh Haim    img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
21420216e5SHanoh Haim    img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S")
22420216e5SHanoh Haim    img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
23420216e5SHanoh Haim    img['formal'] = False
24420216e5SHanoh Haim
25420216e5SHanoh Haim    setup={}
26420216e5SHanoh Haim
27420216e5SHanoh Haim    setup['distro']='Ubunto14.03'
28420216e5SHanoh Haim    setup['kernel']='2.6.12'
29420216e5SHanoh Haim    setup['baremetal']=True
30420216e5SHanoh Haim    setup['hypervisor']='None'
31420216e5SHanoh Haim    setup['name']='trex07'
32420216e5SHanoh Haim    setup['cpu-sockets']=2
33420216e5SHanoh Haim    setup['cores']=16
34420216e5SHanoh Haim    setup['cpu-speed']=3.5
35420216e5SHanoh Haim
36420216e5SHanoh Haim    setup['dut'] ='loopback'
37420216e5SHanoh Haim    setup['drv-name']='mlx5'
38420216e5SHanoh Haim    setup['nic-ports']=2
39420216e5SHanoh Haim    setup['total-nic-ports']=2
40420216e5SHanoh Haim    setup['nic-speed'] ="40GbE"
41420216e5SHanoh Haim
42420216e5SHanoh Haim
43420216e5SHanoh Haim
44420216e5SHanoh Haim    info['image'] = img
45420216e5SHanoh Haim    info['setup'] = setup
46420216e5SHanoh Haim
47420216e5SHanoh Haim    d['info'] =info;
48420216e5SHanoh Haim
49420216e5SHanoh Haim    d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S")
50420216e5SHanoh Haim    d['build_id']=str("build-%d" %(build_id))
51420216e5SHanoh Haim    d['test']={ "name" : "test1",
52420216e5SHanoh Haim                "type"  : "stateless",
53420216e5SHanoh Haim                "cores" : random.randint(1,10),
54420216e5SHanoh Haim                "cpu%" : random.randint(60,99),
55420216e5SHanoh Haim                "mpps" :  random.randint(9,32),
56420216e5SHanoh Haim                "mpps_pc" :  random.randint(9,32),
57420216e5SHanoh Haim                "gbps_pc" :  random.randint(9,32),
58420216e5SHanoh Haim                "gbps" :  random.randint(9,32),
59420216e5SHanoh Haim                "avg-pktsize" : random.randint(60,1500),
60420216e5SHanoh Haim                "latecny" : { "min" : random.randint(1,10),
61420216e5SHanoh Haim                              "max" : random.randint(100,120),
62420216e5SHanoh Haim                              "avr" : random.randint(1,60)
63420216e5SHanoh Haim                             }
64420216e5SHanoh Haim        };
65420216e5SHanoh Haim
66420216e5SHanoh Haim
67420216e5SHanoh Haim    return(d)
68420216e5SHanoh Haim
69420216e5SHanoh Haim
70420216e5SHanoh Haimclass EsHelper(object):
71420216e5SHanoh Haim
72420216e5SHanoh Haim    def __init__ (self, es,
73420216e5SHanoh Haim                        alias,
74420216e5SHanoh Haim                        index_name,
75420216e5SHanoh Haim                        mapping):
76420216e5SHanoh Haim        self.es      = es
77420216e5SHanoh Haim        self.alias   = alias
78420216e5SHanoh Haim        self.index_name = index_name
79420216e5SHanoh Haim        self.mapping = mapping
80420216e5SHanoh Haim        self.setting = { "index.mapper.dynamic":"false"};
81420216e5SHanoh Haim
82420216e5SHanoh Haim    def delete (self):
83420216e5SHanoh Haim        es=self.es;
84420216e5SHanoh Haim        es.indices.delete(index=self.alias, ignore=[400, 404]);
85420216e5SHanoh Haim
86420216e5SHanoh Haim    def is_exists (self):
87420216e5SHanoh Haim        es=self.es;
88420216e5SHanoh Haim        return es.indices.exists(index=self.alias, ignore=[400, 404])
89420216e5SHanoh Haim
90420216e5SHanoh Haim    def create_first_fime (self):
91420216e5SHanoh Haim        es=self.es;
92420216e5SHanoh Haim        index_name=self.index_name
93420216e5SHanoh Haim        es.indices.create(index=index_name, ignore=[],body = {
94420216e5SHanoh Haim           "aliases": { self.alias : {} },
95420216e5SHanoh Haim           "mappings" : { "data": self.mapping },
96420216e5SHanoh Haim           "settings" : self.setting
97420216e5SHanoh Haim           });
98420216e5SHanoh Haim
99420216e5SHanoh Haim    def update(self):
100420216e5SHanoh Haim        es=self.es;
101420216e5SHanoh Haim        es.indices.put_mapping(index=self.alias, doc_type="data",body=self.mapping);
102420216e5SHanoh Haim        es.indices.rollover(alias=self.alias,body={
103420216e5SHanoh Haim              "conditions": {
104420216e5SHanoh Haim                            "max_age":   "30d",
105420216e5SHanoh Haim                            "max_docs":  100000
106420216e5SHanoh Haim                            },
107420216e5SHanoh Haim               "mappings" : { "data": self.mapping },
108420216e5SHanoh Haim               "settings" : self.setting
109420216e5SHanoh Haim               }
110420216e5SHanoh Haim              );
111420216e5SHanoh Haim
112420216e5SHanoh Haim    def open(self):
113420216e5SHanoh Haim        if not self.is_exists():
114420216e5SHanoh Haim            self.create_first_fime ()
115420216e5SHanoh Haim        else:
116420216e5SHanoh Haim            self.update()
117420216e5SHanoh Haim
118420216e5SHanoh Haim    def close(self):
119420216e5SHanoh Haim        pass;
120420216e5SHanoh Haim
121420216e5SHanoh Haim    def push_data(self,data):
122420216e5SHanoh Haim        es=self.es;
123420216e5SHanoh Haim        es.index(index=self.alias,doc_type="data", body=data);
124420216e5SHanoh Haim
125420216e5SHanoh Haim
126420216e5SHanoh Haim
127420216e5SHanoh Haim
128420216e5SHanoh Haimdef create_reg_object (build_id):
129420216e5SHanoh Haim    d={};
130420216e5SHanoh Haim
131420216e5SHanoh Haim    sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30));
132420216e5SHanoh Haim    info = {};
133420216e5SHanoh Haim
134420216e5SHanoh Haim
135420216e5SHanoh Haim    img={}
136420216e5SHanoh Haim    img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
137420216e5SHanoh Haim    img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S")
138420216e5SHanoh Haim    img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
139420216e5SHanoh Haim    img['formal'] = False
140420216e5SHanoh Haim
141420216e5SHanoh Haim    setup={}
142420216e5SHanoh Haim
143420216e5SHanoh Haim    setup['distro']='Ubunto14.03'
144420216e5SHanoh Haim    setup['kernel']='2.6.12'
145420216e5SHanoh Haim    setup['baremetal']=True
146420216e5SHanoh Haim    setup['hypervisor']='None'
147420216e5SHanoh Haim    setup['name']='trex07'
148420216e5SHanoh Haim    setup['cpu-sockets']=2
149420216e5SHanoh Haim    setup['cores']=16
150420216e5SHanoh Haim    setup['cpu-speed']=3.5
151420216e5SHanoh Haim
152420216e5SHanoh Haim    setup['dut'] ='loopback'
153420216e5SHanoh Haim    setup['drv-name']='mlx5'
154420216e5SHanoh Haim    setup['nic-ports']=2
155420216e5SHanoh Haim    setup['total-nic-ports']=2
156420216e5SHanoh Haim    setup['nic-speed'] ="40GbE"
157420216e5SHanoh Haim
158420216e5SHanoh Haim
159420216e5SHanoh Haim
160420216e5SHanoh Haim    info['image'] = img
161420216e5SHanoh Haim    info['setup'] = setup
162420216e5SHanoh Haim
163420216e5SHanoh Haim    d['info'] =info;
164420216e5SHanoh Haim
165420216e5SHanoh Haim    d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S")
166420216e5SHanoh Haim    d['build_id']=str("build-%d" %(build_id))
167420216e5SHanoh Haim    d['test']= { "name" : "stateful_tests.trex_imix_test.CTRexIMIX_Test.test_routing_imix" ,
168420216e5SHanoh Haim                "type"  : "stateless",
169420216e5SHanoh Haim                "duration_sec" : random.uniform(1,10),
170420216e5SHanoh Haim                "result" : random.choice(["PASS","SKIP","FAIL"]),
171420216e5SHanoh Haim                "stdout" : """
172420216e5SHanoh Haim                            LATEST RESULT OBJECT:
173420216e5SHanoh Haim                            Total ARP received : 16 pkts
174420216e5SHanoh Haim                            maximum-latency : 300 usec
175420216e5SHanoh Haim                            average-latency : 277 usec
176420216e5SHanoh Haim                            latency-any-error : ERROR
177420216e5SHanoh Haim                """
178420216e5SHanoh Haim        };
179420216e5SHanoh Haim
180420216e5SHanoh Haim    return(d)
181420216e5SHanoh Haim
182420216e5SHanoh Haim
183420216e5SHanoh Haim
184420216e5SHanoh Haim# how to add new keyword
185420216e5SHanoh Haim# you can add a new field but you can't remove old field
186420216e5SHanoh Haimclass TRexEs(object):
187420216e5SHanoh Haim
188420216e5SHanoh Haim    def __init__ (self, host,
189420216e5SHanoh Haim                        port,
190420216e5SHanoh Haim                        ):
191420216e5SHanoh Haim        self.es = Elasticsearch([{"host": host, "port": port}])
192420216e5SHanoh Haim        es=self.es;
193420216e5SHanoh Haim        res=es.info()
194420216e5SHanoh Haim        es_version=res["version"]["number"];
195420216e5SHanoh Haim        l=es_version.split('.');
196420216e5SHanoh Haim        if not(len(l)==3 and int(l[0])>=5):
197420216e5SHanoh Haim            print("NOT valid ES version should be at least 5.0.x",es_version);
198420216e5SHanoh Haim            raise RuntimeError
199420216e5SHanoh Haim
200420216e5SHanoh Haim        setup_info = { # constant per setup
201420216e5SHanoh Haim                         "properties": {
202420216e5SHanoh Haim
203420216e5SHanoh Haim                          "image" : {
204420216e5SHanoh Haim                              "properties": {
205420216e5SHanoh Haim                                "sha"          : { "type": "keyword" },   # git sha
206420216e5SHanoh Haim                                "build_time"   : { "type": "date",        # build time
207420216e5SHanoh Haim                                                  "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
208420216e5SHanoh Haim                                "version"      : { "type": "keyword" },   # version name like 'v2.12'
209420216e5SHanoh Haim                                "formal"       : { "type": "boolean" },   # true for formal release
210420216e5SHanoh Haim                               }
211420216e5SHanoh Haim                          },
212420216e5SHanoh Haim
213420216e5SHanoh Haim                          "setup" : {
214420216e5SHanoh Haim                              "properties": {
215420216e5SHanoh Haim                               "distro"         : { "type": "keyword" },   # 'ubuntu'
216420216e5SHanoh Haim                               "kernel"         : { "type": "keyword" },   # 2.3.19
217420216e5SHanoh Haim                               "baremetal"      : { "type": "boolean" },   # true or false for
218420216e5SHanoh Haim                               "hypervisor"     : { "type": "keyword" },   # kvm,esxi , none
219420216e5SHanoh Haim                               "name"           : { "type": "keyword" },   # setup name , e.g. kiwi02
220420216e5SHanoh Haim                               "cpu-sockets"    : { "type": "long" },      # number of socket
221420216e5SHanoh Haim                               "cores"          : { "type": "long" },      # total cores
222420216e5SHanoh Haim                               "cpu-speed"      : { "type": "double" },    # 3.5 in ghz
223420216e5SHanoh Haim                               "dut"            : { "type": "keyword" },   # asr1k, loopback
224420216e5SHanoh Haim                               "drv-name"       : { "type": "keyword" },   # vic, mlx5,599,xl710,x710
225420216e5SHanoh Haim                               "nic-ports"      : { "type": "long" },      #2,1,4
226420216e5SHanoh Haim                               "total-nic-ports"  : { "type": "long" },    #8
227420216e5SHanoh Haim                               "nic-speed"      : { "type": "keyword" },   #40Gb
228420216e5SHanoh Haim                           }
229420216e5SHanoh Haim                         }
230420216e5SHanoh Haim                       }
231420216e5SHanoh Haim                   }
232420216e5SHanoh Haim
233420216e5SHanoh Haim
234420216e5SHanoh Haim        perf_mapping = {
235420216e5SHanoh Haim              "dynamic": "strict",
236420216e5SHanoh Haim              "properties": {
237420216e5SHanoh Haim
238420216e5SHanoh Haim                   "scenario"  : { "type": "keyword" },
239420216e5SHanoh Haim                   "build_id"  : { "type": "keyword" },
240420216e5SHanoh Haim                   "timestamp" : { "type": "date",
241420216e5SHanoh Haim                             "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
242420216e5SHanoh Haim
243420216e5SHanoh Haim                  "info"     : setup_info,
244420216e5SHanoh Haim
245420216e5SHanoh Haim                  "test"     : {
246420216e5SHanoh Haim                                "properties": {
247420216e5SHanoh Haim                                    "name"        : { "type": "keyword" }, # name of the test
248420216e5SHanoh Haim                                    "type"        : { "type": "keyword" }, # stateless,stateful, other
249420216e5SHanoh Haim                                    "cores"       : { "type": "long" },
250420216e5SHanoh Haim                                    "cpu%"        : { "type": "double" },
251420216e5SHanoh Haim                                    "mpps"        : { "type": "double" },
252420216e5SHanoh Haim                                    "streams_count" : { "type": "long" },
253420216e5SHanoh Haim                                    "mpps_pc"     : { "type": "double" },
254420216e5SHanoh Haim                                    "gbps_pc"     : { "type": "double" },
255420216e5SHanoh Haim                                    "gbps"        : { "type": "double" },
256420216e5SHanoh Haim                                    "avg-pktsize" : { "type": "long" },
257420216e5SHanoh Haim                                    "kcps"        : { "type": "double" },
258420216e5SHanoh Haim                                    "latecny"     : {
259420216e5SHanoh Haim                                                     "properties": {
260420216e5SHanoh Haim                                                       "min"        : { "type": "double" },
261420216e5SHanoh Haim                                                       "max"        : { "type": "double" },
262420216e5SHanoh Haim                                                       "avr"        : { "type": "double" },
263420216e5SHanoh Haim                                                       "max-win"    : { "type": "double" },
264420216e5SHanoh Haim                                                       "drop-rate"  : { "type": "double" },
265420216e5SHanoh Haim                                                       "jitter"     : { "type": "double" },
266420216e5SHanoh Haim                                                      }
267420216e5SHanoh Haim                                                    }
268420216e5SHanoh Haim
269420216e5SHanoh Haim                                }
270420216e5SHanoh Haim                           }
271420216e5SHanoh Haim            }
272420216e5SHanoh Haim        }
273420216e5SHanoh Haim
274420216e5SHanoh Haim        self.perf = EsHelper(es=es,
275420216e5SHanoh Haim                             alias="perf",
276420216e5SHanoh Haim                             index_name='trex_perf-000001',
277420216e5SHanoh Haim                             mapping=perf_mapping)
278420216e5SHanoh Haim
279420216e5SHanoh Haim
280420216e5SHanoh Haim
281420216e5SHanoh Haim        reg_mapping = {
282420216e5SHanoh Haim              "dynamic": "strict",
283420216e5SHanoh Haim              "properties": {
284420216e5SHanoh Haim
285420216e5SHanoh Haim                   "scenario"  : { "type": "keyword" },
286420216e5SHanoh Haim                   "build_id"  : { "type": "keyword" },
287420216e5SHanoh Haim                   "timestamp" : { "type": "date",
288420216e5SHanoh Haim                             "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
289420216e5SHanoh Haim
290420216e5SHanoh Haim                  "info"     : setup_info,
291420216e5SHanoh Haim
292420216e5SHanoh Haim                  "test"     : {
293420216e5SHanoh Haim                                "properties": {
294f22fba6eSHanoh Haim                                    "name"            : { "type" : "text" }, # name of the test
295f22fba6eSHanoh Haim                                    "name_key"        : { "type" : "keyword" }, # name of the test
296f22fba6eSHanoh Haim                                    "name_full"       : { "type" : "keyword" }, # full name of the test
297f22fba6eSHanoh Haim                                    "type"            : { "type" : "keyword" }, # stateless,stateful, other
298f22fba6eSHanoh Haim                                    "duration_sec"    : { "type": "double" }, # sec
299f22fba6eSHanoh Haim                                    "result"          : { "type" : "keyword" }, # PASS,FAIL,SKIP
300f22fba6eSHanoh Haim                                    "stdout"          : { "type" : "text" },  # output in case of faliue
301420216e5SHanoh Haim                                }
302420216e5SHanoh Haim                           }
303420216e5SHanoh Haim            }
304420216e5SHanoh Haim        }
305420216e5SHanoh Haim
306420216e5SHanoh Haim
307420216e5SHanoh Haim        self.reg = EsHelper(es=es,
308420216e5SHanoh Haim                             alias="reg",
309420216e5SHanoh Haim                             index_name='trex_reg-000001',
310420216e5SHanoh Haim                             mapping=reg_mapping)
311420216e5SHanoh Haim
312420216e5SHanoh Haim
313420216e5SHanoh Haim        self.perf.open();
314420216e5SHanoh Haim        self.reg.open();
315420216e5SHanoh Haim
316420216e5SHanoh Haim
317420216e5SHanoh Haim
318420216e5SHanoh Haim
319420216e5SHanoh Haim
320420216e5SHanoh Haim
321420216e5SHanoh Haim
322420216e5SHanoh Haim
323