csit-perf.yaml revision 6aad83f2
1# Copyright (c) 2019 Cisco and/or its affiliates.
2# Copyright (c) 2019 PANTHEON.tech s.r.o. and/or its affiliates.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at:
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15---
16- project:
17    name: csit-perf
18    description: 'CSIT Jenkins jobs.'
19
20    jobs:
21      - 'csit-vpp-perf-verify-{stream}-{node-arch}'
22      - 'csit-vpp-perf-mrr-daily-master{node-arch}':
23          node-arch:
24            - ''
25            - -2n-skx
26            - -2n-dnv
27            - -3n-skx
28            - -3n-dnv
29            - -3n-tsh:
30                executor: 'ubuntu1804arm-us'
31                vpp-mrr-daily-periodicity: '0 16 * * 6'
32      - 'csit-vpp-perf-mrr-weekly-master-2n-skx'
33      - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}':
34          node-arch:
35            - 2n-skx
36            - 3n-skx
37            - 3n-hsw
38      - 'csit-dpdk-perf-verify-{stream}-{node-arch}'
39      - 'csit-dpdk-perf-mrr-weekly-master{node-arch}':
40          node-arch:
41            - ''
42            - -2n-skx
43            - -3n-skx
44      - 'csit-ligato-perf-verify-{stream}-{node-arch}'
45
46    project: 'csit'
47    executor: 'ubuntu1804-us'
48    ligato-executor: 'ubuntu1804-basebuild-4c-4g'
49    branch: 'master'
50    branch-id: 'oper'
51    vpp-mrr-daily-periodicity: '0 14 * * 1,4'
52    stream:
53      - master:
54          branch: 'master'
55          branch-id: 'oper'
56          executor: 'ubuntu1804-us'
57      - '1901':
58          branch: 'rls1901'
59          branch-id: 'oper-rls1901'
60          executor: 'ubuntu1604-basebuild-8c-32g'
61      - '1904':
62          branch: 'rls1904'
63          branch-id: 'oper-rls1904'
64          executor: 'ubuntu1804-us'
65      - '1908':
66          branch: 'rls1908'
67          branch-id: 'oper-rls1908'
68          executor: 'ubuntu1804-us'
69
70    build-artifact-num-to-keep: 10
71    node-arch:
72      - 2n-skx
73      - 2n-dnv
74      - 3n-skx
75      - 3n-hsw
76      - 3n-dnv
77      - 3n-tsh:
78          executor: 'ubuntu1804arm-us'
79          ligato-executor: 'ubuntu1804arm-us'
80
81- job-template:
82    name: 'csit-vpp-perf-verify-{stream}-{node-arch}'
83
84    description: |
85      <ul>
86          <li>objective
87              <ul>
88                  <li>perf verify - per csit gerrit patch
89              </ul>
90          <li>job
91              <ul>
92                  <li>name: <b>csit-vpp-perf-verify-{stream}-{node-arch}</b>
93                  <li>trigger: keyword in comment to csit patch:
94                      <b>csit-{node-arch}-perftest tag_list</b>
95              </ul>
96          <li>perf test-cases selection
97              <ul>
98                  <li>test-cases selected using trigger keyword followed by tags
99                  <li>all test-cases are labelled with CSIT RF topic tags
100                      defined in <i>csit/docs/tag_documentation.rst</i>
101              </ul>
102          <li>proposed keywords
103              <ul>
104                  <li>focus on testing specified vpp packet processing path
105                  <li>format: csit-{node-arch}-perftest tag_list
106              </ul>
107          <li>default tags for vpp startup config and perf test type
108              <ul>
109                  <li>IP4BASE, IP6BASE, L2BDBASE
110                  <li>64B/78B: 64B/78B packet size
111                  <li>1C: one core
112                  <li>X710: x710 Fortville NIC
113                  <li>MRR: MRR throughput
114              </ul>
115          <li>results
116              <ul>
117                  <li>jjb verify: PASS/FAIL
118                  <li>present: no
119                  <li>analyse: no
120              </ul>
121      </ul>
122
123    project-type: freestyle
124    node: '{executor}'
125    concurrent: true
126    archive-artifacts: 'archive/*.*'
127    latest-only: false
128
129    build-discarder:
130      daysToKeep: '{build-days-to-keep}'
131      numToKeep: '{build-num-to-keep}'
132      artifactDaysToKeep: '{build-artifact-days-to-keep}'
133      artifactNumToKeep: '{build-artifact-num-to-keep}'
134
135    parameters:
136      - project-parameter:
137          project: '{project}'
138      - gerrit-parameter:
139          branch: '{branch}'
140      - gerrit-refspec-parameter
141      - gerrit-event-type-parameter
142      - gerrit-event-comment-text-parameter
143      - string:
144          name: ARCHIVE_ARTIFACTS
145          default: '{archive-artifacts}'
146          description: Artifacts to archive to the logs server.
147      # This is a manually-triggered verify job, part of API coverage.
148      # CRC checking is needed to ensure vpp crc job breaks only when intended.
149      # Can be removed when this job is no longer needed for full API coverage.
150      - csit-fail-on-crc-mismatch-parameter:
151          fail-on-crc-mismatch: 'True'
152
153    scm:
154      - gerrit-trigger-scm:
155          credentials-id: 'jenkins-gerrit-credentials'
156          refspec: '$GERRIT_REFSPEC'
157          choosing-strategy: 'gerrit'
158
159    wrappers:
160      - fdio-infra-wrappers-non-activity-timeout:
161          build-timeout: '{build-timeout}'
162
163    triggers:
164      - gerrit:
165          server-name: 'Primary'
166          trigger-on:
167            - comment-added-contains-event:
168                comment-contains-value: 'csit-{node-arch}-perftest'
169          projects:
170            - project-compare-type: 'ANT'
171              project-pattern: '{project}'
172              branches:
173                - branch-compare-type: 'ANT'
174                  branch-pattern: '**/{branch}'
175          skip-vote:
176            successful: false
177            failed: false
178            unstable: false
179            notbuilt: false
180
181    builders:
182      - shell:
183          !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
184
185    publishers:
186      - robot:
187          output-path: archive
188          other-files:
189            - '*.*'
190
191      - fdio-infra-shiplogs:
192          maven-version: 'mvn33-new'
193
194- job-template:
195    name: 'csit-vpp-perf-mrr-daily-master{node-arch}'
196
197    description: |
198      <ul>
199          <li>objective
200              <ul>
201                  <li>check vpp perf mrr daily
202              </ul>
203          <li>perf test-cases selection
204              <ul>
205                  <li>all CSIT VPP MRR perf suites
206              </ul>
207          <li>results
208              <ul>
209                  <li>jjb verify: PASS/FAIL
210                  <li>present: no
211                  <li>analyse: csit project (catch malfunctions/degradations)
212              </ul>
213      </ul>
214
215    project-type: freestyle
216    node: '{executor}'
217    concurrent: false
218    archive-artifacts: 'csit/archive/*.*'
219    latest-only: false
220
221    build-discarder:
222      daysToKeep: '{build-days-to-keep}'
223      numToKeep: '{build-num-to-keep}'
224      artifactDaysToKeep: '{build-artifact-days-to-keep}'
225      artifactNumToKeep: '{build-artifact-num-to-keep}'
226
227    parameters:
228      - project-parameter:
229          project: '{project}'
230      - gerrit-parameter:
231          branch: '{branch}'
232      - string:
233          name: ARCHIVE_ARTIFACTS
234          default: '{archive-artifacts}'
235          description: Artifacts to archive to the logs server.
236
237    wrappers:
238      - fdio-infra-wrappers-non-activity-timeout:
239          build-timeout: '{build-timeout}'
240
241    triggers:
242      - timed: '{vpp-mrr-daily-periodicity}'
243
244    builders:
245      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
246      - inject:
247          properties-file: variables.prop
248      - shell:
249          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
250
251    publishers:
252      - robot:
253          output-path: 'csit/archive'
254          other-files:
255            - '*.*'
256
257      - fdio-infra-shiplogs:
258          maven-version: 'mvn33-new'
259
260- job-template:
261    name: 'csit-vpp-perf-mrr-weekly-master-2n-skx'
262
263    description: |
264      <ul>
265          <li>objective
266              <ul>
267                  <li>check vpp perf mrr weekly on 2n-skx
268              </ul>
269          <li>perf test-cases selection
270              <ul>
271                  <li>NF Density CSIT VPP MRR perf suites
272              </ul>
273          <li>results
274              <ul>
275                  <li>jjb verify: PASS/FAIL
276                  <li>present: no
277                  <li>analyse: csit project (catch malfunctions/degradations)
278              </ul>
279      </ul>
280
281    project-type: freestyle
282    node: '{executor}'
283    concurrent: false
284    archive-artifacts: 'csit/archive/*.*'
285    latest-only: false
286
287    build-discarder:
288      daysToKeep: '{build-days-to-keep}'
289      numToKeep: '{build-num-to-keep}'
290      artifactDaysToKeep: '{build-artifact-days-to-keep}'
291      artifactNumToKeep: '{build-artifact-num-to-keep}'
292
293    parameters:
294      - project-parameter:
295          project: '{project}'
296      - gerrit-parameter:
297          branch: '{branch}'
298      - string:
299          name: ARCHIVE_ARTIFACTS
300          default: '{archive-artifacts}'
301          description: Artifacts to archive to the logs server.
302
303    wrappers:
304      - fdio-infra-wrappers-non-activity-timeout:
305          build-timeout: '{build-timeout}'
306
307    triggers:
308      - timed: '0 0 * * 6'
309
310    builders:
311      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
312      - inject:
313          properties-file: variables.prop
314      - shell:
315          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
316
317    publishers:
318      - robot:
319          output-path: 'csit/archive'
320          other-files:
321            - '*.*'
322
323      - fdio-infra-shiplogs:
324          maven-version: 'mvn33-new'
325
326- job-template:
327    name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}'
328
329    description: |
330      <ul>
331          <li>objective
332              <ul>
333                  <li>check vpp perf ndrpdr weekly on {node-arch}
334              </ul>
335          <li>perf test-cases selection
336              <ul>
337                  <li>pre-selected CSIT VPP NDRPDR perf suites
338              </ul>
339          <li>results
340              <ul>
341                  <li>jjb verify: PASS/FAIL
342                  <li>present: no
343                  <li>analyse: csit project (catch malfunctions/degradations)
344              </ul>
345      </ul>
346    project-type: freestyle
347    node: '{executor}'
348    concurrent: false
349    archive-artifacts: 'csit/archive/*.*'
350    latest-only: false
351
352    build-discarder:
353      daysToKeep: '{build-days-to-keep}'
354      numToKeep: '{build-num-to-keep}'
355      artifactDaysToKeep: '{build-artifact-days-to-keep}'
356      artifactNumToKeep: '{build-artifact-num-to-keep}'
357
358    parameters:
359      - project-parameter:
360          project: '{project}'
361      - gerrit-parameter:
362          branch: '{branch}'
363      - string:
364          name: ARCHIVE_ARTIFACTS
365          default: '{archive-artifacts}'
366          description: Artifacts to archive to the logs server.
367
368    wrappers:
369      - fdio-infra-wrappers-non-activity-timeout:
370          build-timeout: '{build-timeout}'
371
372    triggers:
373      - timed: '0 0 * * 6'
374
375    builders:
376      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
377      - inject:
378          properties-file: variables.prop
379      - shell:
380          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
381
382    publishers:
383      - robot:
384          output-path: 'csit/archive'
385          other-files:
386            - '*.*'
387
388      - fdio-infra-shiplogs:
389          maven-version: 'mvn33-new'
390
391- job-template:
392    name: 'csit-dpdk-perf-verify-{stream}-{node-arch}'
393
394    description: |
395      <ul>
396          <li>objective
397              <ul>
398                  <li>perf verify - per csit gerrit patch
399              </ul>
400          <li>job
401              <ul>
402                  <li>name: <b>csit-dpdk-perf-verify-{stream}-{node-arch}</b>
403                  <li>trigger: keyword in comment to csit patch:
404                      <b>csit-dpdk-{node-arch}-perftest tag_list</b>
405              </ul>
406          <li>perf test-cases selection
407              <ul>
408                  <li>test-cases selected using trigger keyword followed by tags
409                  <li>all test-cases are labelled with CSIT RF topic tags
410                      defined in <i>csit/docs/tag_documentation.rst</i>
411              </ul>
412          <li>default tags for dpdk perf test type
413              <ul>
414                  <li>IP4BASE, L2XCBASE
415                  <li>64B: 64B packet size
416                  <li>1C: one core
417                  <li>X710: x710 Fortville NIC
418                  <li>MRR: MRR throughput
419              </ul>
420          <li>results
421              <ul>
422                  <li>jjb verify: PASS/FAIL
423                  <li>present: no
424                  <li>analyse: no
425              </ul>
426      </ul>
427
428    project-type: freestyle
429    node: '{executor}'
430    concurrent: true
431    archive-artifacts: 'archive/*.*'
432    latest-only: false
433
434    build-discarder:
435      daysToKeep: '{build-days-to-keep}'
436      numToKeep: '{build-num-to-keep}'
437      artifactDaysToKeep: '{build-artifact-days-to-keep}'
438      artifactNumToKeep: '{build-artifact-num-to-keep}'
439
440    parameters:
441      - project-parameter:
442          project: '{project}'
443      - gerrit-parameter:
444          branch: '{branch}'
445      - string:
446          name: ARCHIVE_ARTIFACTS
447          default: '{archive-artifacts}'
448          description: Artifacts to archive to the logs server.
449
450    scm:
451      - gerrit-trigger-scm:
452          credentials-id: 'jenkins-gerrit-credentials'
453          refspec: '$GERRIT_REFSPEC'
454          choosing-strategy: 'gerrit'
455
456    wrappers:
457      - fdio-infra-wrappers-non-activity-timeout:
458          build-timeout: '{build-timeout}'
459
460    triggers:
461      - gerrit:
462          server-name: 'Primary'
463          trigger-on:
464            - comment-added-contains-event:
465                comment-contains-value: 'csit-dpdk-{node-arch}-perftest'
466          projects:
467            - project-compare-type: 'ANT'
468              project-pattern: '{project}'
469              branches:
470                - branch-compare-type: 'ANT'
471                  branch-pattern: '**/{branch}'
472          skip-vote:
473            successful: false
474            failed: false
475            unstable: false
476            notbuilt: false
477
478    builders:
479      - shell:
480          !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
481
482    publishers:
483      - robot:
484          output-path: archive
485          other-files:
486            - '*.*'
487
488      - fdio-infra-shiplogs:
489          maven-version: 'mvn33-new'
490
491- job-template:
492    name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}'
493
494    description: |
495      <ul>
496          <li>objective
497              <ul>
498                  <li>check dpdk perf mrr weekly
499              </ul>
500          <li>perf test-cases selection
501              <ul>
502                  <li>all CSIT DPDK MRR perf suites
503              </ul>
504          <li>results
505              <ul>
506                  <li>jjb verify: PASS/FAIL
507                  <li>present: no
508                  <li>analyse: csit project (catch malfunctions/degradations)
509              </ul>
510      </ul>
511
512    project-type: freestyle
513    node: '{executor}'
514    concurrent: false
515    archive-artifacts: 'csit/archive/*.*'
516    latest-only: false
517
518    build-discarder:
519      daysToKeep: '{build-days-to-keep}'
520      numToKeep: '{build-num-to-keep}'
521      artifactDaysToKeep: '{build-artifact-days-to-keep}'
522      artifactNumToKeep: '{build-artifact-num-to-keep}'
523
524    parameters:
525      - project-parameter:
526          project: '{project}'
527      - gerrit-parameter:
528          branch: '{branch}'
529      - string:
530          name: ARCHIVE_ARTIFACTS
531          default: '{archive-artifacts}'
532          description: Artifacts to archive to the logs server.
533
534    wrappers:
535      - fdio-infra-wrappers-non-activity-timeout:
536          build-timeout: '{build-timeout}'
537
538    triggers:
539      - timed: '0 0 * * 6'
540
541    builders:
542      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
543      - inject:
544          properties-file: variables.prop
545      - shell:
546          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
547
548    publishers:
549      - robot:
550          output-path: 'csit/archive'
551          other-files:
552            - '*.*'
553
554      - fdio-infra-shiplogs:
555          maven-version: 'mvn33-new'
556
557- job-template:
558    name: 'csit-ligato-perf-verify-{stream}-{node-arch}'
559
560    # yamllint disable rule:line-length
561    description: |
562      <ul>
563          <li>objective
564              <ul>
565                  <li>perf verify - per csit gerrit patch
566              </ul>
567          <li>job
568              <ul>
569                  <li>name: <b>csit-ligato-perf-verify-{stream}-{node-arch}</b>
570                  <li>trigger: keyword in comment to csit patch:
571                      <b>csit--ligato-{node-arch}-perftest tag_list</b>
572              </ul>
573          <li>perf test-cases selection
574              <ul>
575                  <li>test-cases selected using trigger keyword followed by tags
576                  <li>all test-cases are labelled with CSIT RF topic tags
577                      defined in <i>csit/docs/tag_documentation.rst</i>
578              </ul>
579          <li>proposed keywords
580              <ul>
581                  <li>focus on testing specified Kubernetes orchestrated topologies
582                  <li>format: csit-ligato-{node-arch}-perftest tag_list
583              </ul>
584          <li>default tags for vpp startup config and perf test type
585              <ul>
586                  <li>IP4BASE, IP6BASE, L2BDBASE
587                  <li>64B/78B: 64B/78B packet size
588                  <li>1C: one core
589                  <li>X710: x710 Fortville NIC
590                  <li>MRR: MRR throughput
591              </ul>
592          <li>results
593              <ul>
594                  <li>jjb verify: PASS/FAIL
595                  <li>present: no
596                  <li>analyse: no
597              </ul>
598      </ul>
599      # yamllint enable rule:line-length
600
601    project-type: freestyle
602    # As the nomad executors are running docker containers we cannot run
603    # ligato-perf job due to dependency on using docker inside.
604    # TODO: Find the mechanics how to offload job logic from compiling docker
605    #   image so we can move this executor to ubuntu1804-us
606    node: '{ligato-executor}'
607    concurrent: true
608    archive-artifacts: 'archive/*.*'
609    latest-only: false
610
611    build-discarder:
612      daysToKeep: '{build-days-to-keep}'
613      numToKeep: '{build-num-to-keep}'
614      artifactDaysToKeep: '{build-artifact-days-to-keep}'
615      artifactNumToKeep: '{build-artifact-num-to-keep}'
616
617    parameters:
618      - project-parameter:
619          project: '{project}'
620      - gerrit-parameter:
621          branch: '{branch}'
622      - gerrit-refspec-parameter
623      - gerrit-event-type-parameter
624      - gerrit-event-comment-text-parameter
625      - string:
626          name: ARCHIVE_ARTIFACTS
627          default: '{archive-artifacts}'
628          description: Artifacts to archive to the logs server.
629
630    scm:
631      - gerrit-trigger-scm:
632          credentials-id: 'jenkins-gerrit-credentials'
633          refspec: '$GERRIT_REFSPEC'
634          choosing-strategy: 'gerrit'
635
636    wrappers:
637      - fdio-infra-wrappers-non-activity-timeout:
638          build-timeout: '{build-timeout}'
639
640    triggers:
641      - gerrit:
642          server-name: 'Primary'
643          trigger-on:
644            - comment-added-contains-event:
645                comment-contains-value: 'csit-ligato-{node-arch}-perftest'
646          projects:
647            - project-compare-type: 'ANT'
648              project-pattern: '{project}'
649              branches:
650                - branch-compare-type: 'ANT'
651                  branch-pattern: '**/{branch}'
652          skip-vote:
653            successful: false
654            failed: false
655            unstable: false
656            notbuilt: false
657
658    builders:
659      - shell:
660          !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
661
662    publishers:
663      - robot:
664          output-path: archive
665          other-files:
666            - '*.*'
667
668      - fdio-infra-shiplogs:
669          maven-version: 'mvn33-new'
670