1import os
2import outer_packages
3import json
4import pprint
5from elasticsearch import Elasticsearch
6from pprint import pprint
7from elasticsearch import helpers
8import random
9import datetime
10
11# one object example for perf
12def create_one_object (build_id):
13    d={};
14
15    sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30));
16    info = {};
17
18
19    img={}
20    img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
21    img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S")
22    img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
23    img['formal'] = False
24
25    setup={}
26
27    setup['distro']='Ubunto14.03'
28    setup['kernel']='2.6.12'
29    setup['baremetal']=True
30    setup['hypervisor']='None'
31    setup['name']='trex07'
32    setup['cpu-sockets']=2
33    setup['cores']=16
34    setup['cpu-speed']=3.5
35
36    setup['dut'] ='loopback'
37    setup['drv-name']='mlx5'
38    setup['nic-ports']=2
39    setup['total-nic-ports']=2
40    setup['nic-speed'] ="40GbE"
41
42
43
44    info['image'] = img
45    info['setup'] = setup
46
47    d['info'] =info;
48
49    d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S")
50    d['build_id']=str("build-%d" %(build_id))
51    d['test']={ "name" : "test1",
52                "type"  : "stateless",
53                "cores" : random.randint(1,10),
54                "cpu%" : random.randint(60,99),
55                "mpps" :  random.randint(9,32),
56                "mpps_pc" :  random.randint(9,32),
57                "gbps_pc" :  random.randint(9,32),
58                "gbps" :  random.randint(9,32),
59                "avg-pktsize" : random.randint(60,1500),
60                "latecny" : { "min" : random.randint(1,10),
61                              "max" : random.randint(100,120),
62                              "avr" : random.randint(1,60)
63                             }
64        };
65
66
67    return(d)
68
69
70class EsHelper(object):
71
72    def __init__ (self, es,
73                        alias,
74                        index_name,
75                        mapping):
76        self.es      = es
77        self.alias   = alias
78        self.index_name = index_name
79        self.mapping = mapping
80        self.setting = { "index.mapper.dynamic":"false"};
81
82    def delete (self):
83        es=self.es;
84        es.indices.delete(index=self.alias, ignore=[400, 404]);
85
86    def is_exists (self):
87        es=self.es;
88        return es.indices.exists(index=self.alias, ignore=[400, 404])
89
90    def create_first_fime (self):
91        es=self.es;
92        index_name=self.index_name
93        es.indices.create(index=index_name, ignore=[],body = {
94           "aliases": { self.alias : {} },
95           "mappings" : { "data": self.mapping },
96           "settings" : self.setting
97           });
98
99    def update(self):
100        es=self.es;
101        es.indices.put_mapping(index=self.alias, doc_type="data",body=self.mapping);
102        es.indices.rollover(alias=self.alias,body={
103              "conditions": {
104                            "max_age":   "30d",
105                            "max_docs":  100000
106                            },
107               "mappings" : { "data": self.mapping },
108               "settings" : self.setting
109               }
110              );
111
112    def open(self):
113        if not self.is_exists():
114            self.create_first_fime ()
115        else:
116            self.update()
117
118    def close(self):
119        pass;
120
121    def push_data(self,data):
122        es=self.es;
123        es.index(index=self.alias,doc_type="data", body=data);
124
125
126
127
128def create_reg_object (build_id):
129    d={};
130
131    sim_date=datetime.datetime.now()-datetime.timedelta(hours=random.randint(0,24*30));
132    info = {};
133
134
135    img={}
136    img['sha'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
137    img['build_time'] = sim_date.strftime("%Y-%m-%d %H:%M:%S")
138    img['version'] = random.choice(["v2.11","v2.10","v2.12","v2.13","v2.14"])
139    img['formal'] = False
140
141    setup={}
142
143    setup['distro']='Ubunto14.03'
144    setup['kernel']='2.6.12'
145    setup['baremetal']=True
146    setup['hypervisor']='None'
147    setup['name']='trex07'
148    setup['cpu-sockets']=2
149    setup['cores']=16
150    setup['cpu-speed']=3.5
151
152    setup['dut'] ='loopback'
153    setup['drv-name']='mlx5'
154    setup['nic-ports']=2
155    setup['total-nic-ports']=2
156    setup['nic-speed'] ="40GbE"
157
158
159
160    info['image'] = img
161    info['setup'] = setup
162
163    d['info'] =info;
164
165    d['timestamp']=sim_date.strftime("%Y-%m-%d %H:%M:%S")
166    d['build_id']=str("build-%d" %(build_id))
167    d['test']= { "name" : "stateful_tests.trex_imix_test.CTRexIMIX_Test.test_routing_imix" ,
168                "type"  : "stateless",
169                "duration_sec" : random.uniform(1,10),
170                "result" : random.choice(["PASS","SKIP","FAIL"]),
171                "stdout" : """
172                            LATEST RESULT OBJECT:
173                            Total ARP received : 16 pkts
174                            maximum-latency : 300 usec
175                            average-latency : 277 usec
176                            latency-any-error : ERROR
177                """
178        };
179
180    return(d)
181
182
183
184# how to add new keyword
185# you can add a new field but you can't remove old field
186class TRexEs(object):
187
188    def __init__ (self, host,
189                        port,
190                        ):
191        self.es = Elasticsearch([{"host": host, "port": port}])
192        es=self.es;
193        res=es.info()
194        es_version=res["version"]["number"];
195        l=es_version.split('.');
196        if not(len(l)==3 and int(l[0])>=5):
197            print("NOT valid ES version should be at least 5.0.x",es_version);
198            raise RuntimeError
199
200        setup_info = { # constant per setup
201                         "properties": {
202
203                          "image" : {
204                              "properties": {
205                                "sha"          : { "type": "keyword" },   # git sha
206                                "build_time"   : { "type": "date",        # build time
207                                                  "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
208                                "version"      : { "type": "keyword" },   # version name like 'v2.12'
209                                "formal"       : { "type": "boolean" },   # true for formal release
210                               }
211                          },
212
213                          "setup" : {
214                              "properties": {
215                               "distro"         : { "type": "keyword" },   # 'ubuntu'
216                               "kernel"         : { "type": "keyword" },   # 2.3.19
217                               "baremetal"      : { "type": "boolean" },   # true or false for
218                               "hypervisor"     : { "type": "keyword" },   # kvm,esxi , none
219                               "name"           : { "type": "keyword" },   # setup name , e.g. kiwi02
220                               "cpu-sockets"    : { "type": "long" },      # number of socket
221                               "cores"          : { "type": "long" },      # total cores
222                               "cpu-speed"      : { "type": "double" },    # 3.5 in ghz
223                               "dut"            : { "type": "keyword" },   # asr1k, loopback
224                               "drv-name"       : { "type": "keyword" },   # vic, mlx5,599,xl710,x710
225                               "nic-ports"      : { "type": "long" },      #2,1,4
226                               "total-nic-ports"  : { "type": "long" },    #8
227                               "nic-speed"      : { "type": "keyword" },   #40Gb
228                           }
229                         }
230                       }
231                   }
232
233
234        perf_mapping = {
235              "dynamic": "strict",
236              "properties": {
237
238                   "scenario"  : { "type": "keyword" },
239                   "build_id"  : { "type": "keyword" },
240                   "timestamp" : { "type": "date",
241                             "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
242
243                  "info"     : setup_info,
244
245                  "test"     : {
246                                "properties": {
247                                    "name"        : { "type": "keyword" }, # name of the test
248                                    "type"        : { "type": "keyword" }, # stateless,stateful, other
249                                    "cores"       : { "type": "long" },
250                                    "cpu%"        : { "type": "double" },
251                                    "mpps"        : { "type": "double" },
252                                    "streams_count" : { "type": "long" },
253                                    "mpps_pc"     : { "type": "double" },
254                                    "gbps_pc"     : { "type": "double" },
255                                    "gbps"        : { "type": "double" },
256                                    "avg-pktsize" : { "type": "long" },
257                                    "kcps"        : { "type": "double" },
258                                    "latecny"     : {
259                                                     "properties": {
260                                                       "min"        : { "type": "double" },
261                                                       "max"        : { "type": "double" },
262                                                       "avr"        : { "type": "double" },
263                                                       "max-win"    : { "type": "double" },
264                                                       "drop-rate"  : { "type": "double" },
265                                                       "jitter"     : { "type": "double" },
266                                                      }
267                                                    }
268
269                                }
270                           }
271            }
272        }
273
274        self.perf = EsHelper(es=es,
275                             alias="perf",
276                             index_name='trex_perf-000001',
277                             mapping=perf_mapping)
278
279
280
281        reg_mapping = {
282              "dynamic": "strict",
283              "properties": {
284
285                   "scenario"  : { "type": "keyword" },
286                   "build_id"  : { "type": "keyword" },
287                   "timestamp" : { "type": "date",
288                             "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"},
289
290                  "info"     : setup_info,
291
292                  "test"     : {
293                                "properties": {
294                                    "name"            : { "type" : "text" }, # name of the test
295                                    "name_key"        : { "type" : "keyword" }, # name of the test
296                                    "name_full"       : { "type" : "keyword" }, # full name of the test
297                                    "type"            : { "type" : "keyword" }, # stateless,stateful, other
298                                    "duration_sec"    : { "type": "double" }, # sec
299                                    "result"          : { "type" : "keyword" }, # PASS,FAIL,SKIP
300                                    "stdout"          : { "type" : "text" },  # output in case of faliue
301                                }
302                           }
303            }
304        }
305
306
307        self.reg = EsHelper(es=es,
308                             alias="reg",
309                             index_name='trex_reg-000001',
310                             mapping=reg_mapping)
311
312
313        self.perf.open();
314        self.reg.open();
315
316
317
318
319
320
321
322
323