generator_alerts.py revision cbfa26dc
1# Copyright (c) 2019 Cisco and/or its affiliates.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at:
5#
6#     http://www.apache.org/licenses/LICENSE-2.0
7#
8# Unless required by applicable law or agreed to in writing, software
9# distributed under the License is distributed on an "AS IS" BASIS,
10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11# See the License for the specific language governing permissions and
12# limitations under the License.
13
14"""Generator of alerts:
15- failed tests
16- regressions
17- progressions
18"""
19
20
21import smtplib
22import logging
23
24from email.mime.text import MIMEText
25from email.mime.multipart import MIMEMultipart
26from os.path import isdir
27from collections import OrderedDict
28
29from pal_utils import get_last_completed_build_number
30from pal_errors import PresentationError
31
32
33class AlertingError(PresentationError):
34    """Exception(s) raised by the alerting module.
35
36    When raising this exception, put this information to the message in this
37    order:
38     - short description of the encountered problem (parameter msg),
39     - relevant messages if there are any collected, e.g., from caught
40       exception (optional parameter details),
41     - relevant data if there are any collected (optional parameter details).
42    """
43
44    def __init__(self, msg, details=u'', level=u"CRITICAL"):
45        """Sets the exception message and the level.
46
47        :param msg: Short description of the encountered problem.
48        :param details: Relevant messages if there are any collected, e.g.,
49            from caught exception (optional parameter details), or relevant data
50            if there are any collected (optional parameter details).
51        :param level: Level of the error, possible choices are: "DEBUG", "INFO",
52            "WARNING", "ERROR" and "CRITICAL".
53        :type msg: str
54        :type details: str
55        :type level: str
56        """
57
58        super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
59
60    def __repr__(self):
61        return (
62            f"AlertingError(msg={self._msg!r},details={self._details!r},"
63            f"level={self._level!r})"
64        )
65
66
67class Alerting:
68    """Class implementing the alerting mechanism.
69    """
70
71    def __init__(self, spec):
72        """Initialization.
73
74        :param spec: The CPTA specification.
75        :type spec: Specification
76        """
77
78        # Implemented alerts:
79        self._implemented_alerts = (u"failed-tests", )
80
81        self._spec = spec
82
83        try:
84            self._spec_alert = spec.alerting
85        except KeyError as err:
86            raise AlertingError(u"Alerting is not configured, skipped.",
87                                repr(err),
88                                u"WARNING")
89
90        self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
91
92        # Verify and validate input specification:
93        self.configs = self._spec_alert.get(u"configurations", None)
94        if not self.configs:
95            raise AlertingError(u"No alert configuration is specified.")
96        for config_type, config_data in self.configs.items():
97            if config_type == u"email":
98                if not config_data.get(u"server", None):
99                    raise AlertingError(u"Parameter 'server' is missing.")
100                if not config_data.get(u"address-to", None):
101                    raise AlertingError(u"Parameter 'address-to' (recipient) "
102                                        u"is missing.")
103                if not config_data.get(u"address-from", None):
104                    raise AlertingError(u"Parameter 'address-from' (sender) is "
105                                        u"missing.")
106            elif config_type == u"jenkins":
107                if not isdir(config_data.get(u"output-dir", u"")):
108                    raise AlertingError(u"Parameter 'output-dir' is "
109                                        u"missing or it is not a directory.")
110                if not config_data.get(u"output-file", None):
111                    raise AlertingError(u"Parameter 'output-file' is missing.")
112            else:
113                raise AlertingError(
114                    f"Alert of type {config_type} is not implemented."
115                )
116
117        self.alerts = self._spec_alert.get(u"alerts", None)
118        if not self.alerts:
119            raise AlertingError(u"No alert is specified.")
120        for alert_data in self.alerts.values():
121            if not alert_data.get(u"title", None):
122                raise AlertingError(u"Parameter 'title' is missing.")
123            if not alert_data.get(u"type", None) in self._implemented_alerts:
124                raise AlertingError(u"Parameter 'failed-tests' is missing or "
125                                    u"incorrect.")
126            if not alert_data.get(u"way", None) in self.configs.keys():
127                raise AlertingError(u"Parameter 'way' is missing or incorrect.")
128            if not alert_data.get(u"include", None):
129                raise AlertingError(u"Parameter 'include' is missing or the "
130                                    u"list is empty.")
131
132    def __str__(self):
133        """Return string with human readable description of the alert.
134
135        :returns: Readable description.
136        :rtype: str
137        """
138        return f"configs={self.configs}, alerts={self.alerts}"
139
140    def __repr__(self):
141        """Return string executable as Python constructor call.
142
143        :returns: Executable constructor call.
144        :rtype: str
145        """
146        return f"Alerting(spec={self._spec})"
147
148    def generate_alerts(self):
149        """Generate alert(s) using specified way(s).
150        """
151
152        for alert_data in self.alerts.values():
153            if alert_data[u"way"] == u"jenkins":
154                self._generate_email_body(alert_data)
155            else:
156                raise AlertingError(
157                    f"Alert with way {alert_data[u'way']} is not implemented."
158                )
159
160    @staticmethod
161    def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
162        """Send an email using predefined configuration.
163
164        :param server: SMTP server used to send email.
165        :param addr_from: Sender address.
166        :param addr_to: Recipient address(es).
167        :param subject: Subject of the email.
168        :param text: Message in the ASCII text format.
169        :param html: Message in the HTML format.
170        :type server: str
171        :type addr_from: str
172        :type addr_to: list
173        :type subject: str
174        :type text: str
175        :type html: str
176        """
177
178        if not text and not html:
179            raise AlertingError(u"No text/data to send.")
180
181        msg = MIMEMultipart(u'alternative')
182        msg[u'Subject'] = subject
183        msg[u'From'] = addr_from
184        msg[u'To'] = u", ".join(addr_to)
185
186        if text:
187            msg.attach(MIMEText(text, u'plain'))
188        if html:
189            msg.attach(MIMEText(html, u'html'))
190
191        smtp_server = None
192        try:
193            logging.info(f"Trying to send alert {subject} ...")
194            logging.debug(f"SMTP Server: {server}")
195            logging.debug(f"From: {addr_from}")
196            logging.debug(f"To: {u', '.join(addr_to)}")
197            logging.debug(f"Message: {msg.as_string()}")
198            smtp_server = smtplib.SMTP(server)
199            smtp_server.sendmail(addr_from, addr_to, msg.as_string())
200        except smtplib.SMTPException as err:
201            raise AlertingError(u"Not possible to send the alert via email.",
202                                str(err))
203        finally:
204            if smtp_server:
205                smtp_server.quit()
206
207    def _get_compressed_failed_tests(self, alert, test_set, sort=True):
208        """Return the dictionary with compressed faild tests. The compression is
209        done by grouping the tests from the same area but with different NICs,
210        frame sizes and number of processor cores.
211
212        For example, the failed tests:
213          10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
214          10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
215          10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
216          10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
217          10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
218          10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
219
220        will be represented as:
221          ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
222          (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
223
224        Structure of returned data:
225
226        {
227            "trimmed_TC_name_1": {
228                "nics": [],
229                "framesizes": [],
230                "cores": []
231            }
232            ...
233            "trimmed_TC_name_N": {
234                "nics": [],
235                "framesizes": [],
236                "cores": []
237            }
238        }
239
240        :param alert: Files are created for this alert.
241        :param test_set: Specifies which set of tests will be included in the
242            result. Its name is the same as the name of file with failed tests.
243        :param sort: If True, the failed tests are sorted alphabetically.
244        :type alert: dict
245        :type test_set: str
246        :type sort: bool
247        :returns: CSIT build number, VPP version, Number of passed tests,
248            Number of failed tests, Compressed failed tests.
249        :rtype: tuple(str, str, int, int, OrderedDict)
250        """
251
252        directory = self.configs[alert[u"way"]][u"output-dir"]
253        failed_tests = OrderedDict()
254        file_path = f"{directory}/{test_set}.txt"
255        version = u""
256        try:
257            with open(file_path, u'r') as f_txt:
258                for idx, line in enumerate(f_txt):
259                    if idx == 0:
260                        build = line[:-1]
261                        continue
262                    if idx == 1:
263                        version = line[:-1]
264                        continue
265                    if idx == 2:
266                        passed = line[:-1]
267                        continue
268                    if idx == 3:
269                        failed = line[:-1]
270                        continue
271                    try:
272                        test = line[:-1].split(u'-')
273                        name = u'-'.join(test[3:-1])
274                    except IndexError:
275                        continue
276                    if failed_tests.get(name, None) is None:
277                        failed_tests[name] = dict(nics=list(),
278                                                  framesizes=list(),
279                                                  cores=list())
280                    if test[0] not in failed_tests[name][u"nics"]:
281                        failed_tests[name][u"nics"].append(test[0])
282                    if test[1] not in failed_tests[name][u"framesizes"]:
283                        failed_tests[name][u"framesizes"].append(test[1])
284                    if test[2] not in failed_tests[name][u"cores"]:
285                        failed_tests[name][u"cores"].append(test[2])
286        except IOError:
287            logging.error(f"No such file or directory: {file_path}")
288            return None, None, None, None, None
289        if sort:
290            sorted_failed_tests = OrderedDict()
291            for key in sorted(failed_tests.keys()):
292                sorted_failed_tests[key] = failed_tests[key]
293            return build, version, passed, failed, sorted_failed_tests
294
295        return build, version, passed, failed, failed_tests
296
297    def _list_gressions(self, alert, idx, header, re_pro):
298        """Create a file with regressions or progressions for the test set
299        specified by idx.
300
301        :param alert: Files are created for this alert.
302        :param idx: Index of the test set as it is specified in the
303            specification file.
304        :param header: The header of the list of [re|pro]gressions.
305        :param re_pro: 'regression' or 'progression'.
306        :type alert: dict
307        :type idx: int
308        :type header: str
309        :type re_pro: str
310        """
311
312        if re_pro not in (u"regressions", u"progressions"):
313            return
314
315        in_file = (
316            f"{self.configs[alert[u'way']][u'output-dir']}/"
317            f"cpta-{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
318        )
319        out_file = (
320            f"{self.configs[alert[u'way']][u'output-dir']}/"
321            f"trending-{re_pro}.txt"
322        )
323
324        try:
325            with open(in_file, u'r') as txt_file:
326                file_content = txt_file.read()
327                with open(out_file, u'a+') as reg_file:
328                    reg_file.write(header)
329                    if file_content:
330                        reg_file.write(file_content)
331                    else:
332                        reg_file.write(f"No {re_pro}")
333        except IOError as err:
334            logging.warning(repr(err))
335
336    def _generate_email_body(self, alert):
337        """Create the file which is used in the generated alert.
338
339        :param alert: Files are created for this alert.
340        :type alert: dict
341        """
342
343        if alert[u"type"] != u"failed-tests":
344            raise AlertingError(
345                f"Alert of type {alert[u'type']} is not implemented."
346            )
347
348        text = u""
349        for idx, test_set in enumerate(alert.get(u"include", [])):
350            build, version, passed, failed, failed_tests = \
351                self._get_compressed_failed_tests(alert, test_set)
352            if build is None:
353                ret_code, build_nr, _ = get_last_completed_build_number(
354                    self._spec.environment[u"urls"][u"URL[JENKINS,CSIT]"],
355                    alert[u"urls"][idx].split(u'/')[-1])
356                if ret_code != 0:
357                    build_nr = u''
358                text += (
359                    f"\n\nNo input data available for "
360                    f"{u'-'.join(test_set.split('-')[-2:])}. See CSIT build "
361                    f"{alert[u'urls'][idx]}/{build_nr} for more information.\n"
362                )
363                continue
364            text += (
365                f"\n\n{test_set.split('-')[-2]}-{test_set.split('-')[-1]}, "
366                f"{failed} tests failed, "
367                f"{passed} tests passed, CSIT build: "
368                f"{alert[u'urls'][idx]}/{build}, VPP version: {version}\n\n"
369            )
370
371            class MaxLens():
372                """Class to store the max lengths of strings displayed in
373                failed tests list.
374                """
375                def __init__(self, tst_name, nics, framesizes, cores):
376                    """Initialisation.
377
378                    :param tst_name: Name of the test.
379                    :param nics: NICs used in the test.
380                    :param framesizes: Frame sizes used in the tests
381                    :param cores: Cores used in th test.
382                    """
383                    self.name = tst_name
384                    self.nics = nics
385                    self.frmsizes = framesizes
386                    self.cores = cores
387
388            max_len = MaxLens(0, 0, 0, 0)
389
390            for name, params in failed_tests.items():
391                failed_tests[name][u"nics"] = u",".join(sorted(params[u"nics"]))
392                failed_tests[name][u"framesizes"] = \
393                    u",".join(sorted(params[u"framesizes"]))
394                failed_tests[name][u"cores"] = \
395                    u",".join(sorted(params[u"cores"]))
396                if len(name) > max_len.name:
397                    max_len.name = len(name)
398                if len(failed_tests[name][u"nics"]) > max_len.nics:
399                    max_len.nics = len(failed_tests[name][u"nics"])
400                if len(failed_tests[name][u"framesizes"]) > max_len.frmsizes:
401                    max_len.frmsizes = len(failed_tests[name][u"framesizes"])
402                if len(failed_tests[name][u"cores"]) > max_len.cores:
403                    max_len.cores = len(failed_tests[name][u"cores"])
404
405            for name, params in failed_tests.items():
406                text += (
407                    f"{name + u' ' * (max_len.name - len(name))}  "
408                    f"{params[u'nics']}"
409                    f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
410                    f"{params[u'framesizes']}"
411                    f"{u' ' * (max_len.frmsizes-len(params[u'framesizes']))}  "
412                    f"{params[u'cores']}"
413                    f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
414                )
415
416            gression_hdr = (
417                f"\n\n{test_set.split(u'-')[-2]}-{test_set.split(u'-')[-1]}, "
418                f"CSIT build: {alert[u'urls'][idx]}/{build}, "
419                f"VPP version: {version}\n\n"
420            )
421            # Add list of regressions:
422            self._list_gressions(alert, idx, gression_hdr, u"regressions")
423
424            # Add list of progressions:
425            self._list_gressions(alert, idx, gression_hdr, u"progressions")
426
427        text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
428        file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
429                    f"{self.configs[alert[u'way']][u'output-file']}"
430        logging.info(f"Writing the file {file_name}.txt ...")
431
432        try:
433            with open(f"{file_name}.txt", u'w') as txt_file:
434                txt_file.write(text)
435        except IOError:
436            logging.error(f"Not possible to write the file {file_name}.txt.")
437