csit-perf.yaml revision 0276dee0
1# Copyright (c) 2019 Cisco and/or its affiliates.
2# Copyright (c) 2019 PANTHEON.tech s.r.o. and/or its affiliates.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at:
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15---
16- project:
17    name: csit-perf
18    description: 'CSIT Jenkins jobs.'
19
20    jobs:
21      - 'csit-vpp-perf-verify-{stream}-{node-arch}'
22      - 'csit-vpp-perf-mrr-daily-master{node-arch}':
23          node-arch:
24            - ''
25            - -2n-clx
26            - -2n-skx
27            - -2n-dnv
28            - -3n-skx
29            - -3n-dnv
30            - -3n-tsh:
31                executor: 'ubuntu1804arm-us'
32                vpp-mrr-daily-periodicity: '0 14 * * 0'
33      - 'csit-vpp-perf-mrr-weekly-master-{node-arch}':
34          node-arch:
35            - 2n-clx
36            - 2n-skx
37      - 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}':
38          node-arch:
39            - 2n-clx
40            - 2n-skx
41            - 3n-skx
42            - 3n-hsw
43      - 'csit-dpdk-perf-verify-{stream}-{node-arch}'
44      - 'csit-dpdk-perf-mrr-weekly-master{node-arch}':
45          node-arch:
46            - ''
47            - -2n-clx
48            - -2n-skx
49            - -3n-skx
50
51    project: 'csit'
52    executor: 'ubuntu1804-us'
53    ligato-executor: 'ubuntu1804-basebuild-4c-4g'
54    branch: 'master'
55    branch-id: 'oper'
56    vpp-mrr-daily-periodicity: '0 2,14 * * *'
57    csit-perf-trial-duration: "1.0"
58    csit-perf-trial-multiplicity: "10"
59    stream:
60      - master:
61          branch: 'master'
62          branch-id: 'oper'
63          executor: 'ubuntu1804-us'
64      - '1904':
65          branch: 'rls1904'
66          branch-id: 'oper-rls1904'
67          executor: 'ubuntu1804-us'
68      - '1908_1':
69          branch: 'rls1908_1'
70          branch-id: 'oper-rls1908_1'
71          executor: 'ubuntu1804-us'
72
73    build-artifact-num-to-keep: 10
74    node-arch:
75      - 2n-clx
76      - 2n-skx
77      - 2n-dnv
78      - 3n-skx
79      - 3n-hsw
80      - 3n-dnv
81      - 3n-tsh:
82          executor: 'ubuntu1804arm-us'
83          ligato-executor: 'ubuntu1804arm-us'
84
85- job-template:
86    name: 'csit-vpp-perf-verify-{stream}-{node-arch}'
87
88    description: |
89      <ul>
90          <li>objective
91              <ul>
92                  <li>perf verify - per csit gerrit patch
93              </ul>
94          <li>job
95              <ul>
96                  <li>name: <b>csit-vpp-perf-verify-{stream}-{node-arch}</b>
97                  <li>trigger: keyword in comment to csit patch:
98                      <b>csit-{node-arch}-perftest tag_list</b>
99              </ul>
100          <li>executor
101              <ul>
102                  <li>{executor}
103              </ul>
104          <li>perf test-cases selection
105              <ul>
106                  <li>test-cases selected using trigger keyword followed by tags
107                  <li>all test-cases are labelled with CSIT RF topic tags
108                      defined in <i>csit/docs/tag_documentation.rst</i>
109              </ul>
110          <li>proposed keywords
111              <ul>
112                  <li>focus on testing specified vpp packet processing path
113                  <li>format: csit-{node-arch}-perftest tag_list
114              </ul>
115          <li>default tags for vpp startup config and perf test type
116              <ul>
117                  <li>IP4BASE, IP6BASE, L2BDBASE
118                  <li>64B/78B: 64B/78B packet size
119                  <li>1C: one core
120                  <li>X710: x710 Fortville NIC
121                  <li>MRR: MRR throughput
122              </ul>
123          <li>results
124              <ul>
125                  <li>jjb verify: PASS/FAIL
126                  <li>present: no
127                  <li>analyse: no
128              </ul>
129      </ul>
130
131    project-type: freestyle
132    node: '{executor}'
133    concurrent: true
134    archive-artifacts: 'archive/*.*'
135    latest-only: false
136
137    build-discarder:
138      daysToKeep: '{build-days-to-keep}'
139      numToKeep: '{build-num-to-keep}'
140      artifactDaysToKeep: '{build-artifact-days-to-keep}'
141      artifactNumToKeep: '{build-artifact-num-to-keep}'
142
143    parameters:
144      - project-parameter:
145          project: '{project}'
146      - gerrit-parameter:
147          branch: '{branch}'
148      - gerrit-refspec-parameter
149      - gerrit-event-type-parameter
150      - gerrit-event-comment-text-parameter
151      - csit-perf-trial-duration-parameter:
152          csit-perf-trial-duration: "{csit-perf-trial-duration}"
153      - csit-perf-trial-multiplicity-parameter:
154          csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
155      - string:
156          name: ARCHIVE_ARTIFACTS
157          default: '{archive-artifacts}'
158          description: Artifacts to archive to the logs server.
159      # This is a manually-triggered verify job, part of API coverage.
160      # CRC checking is needed to ensure vpp crc job breaks only when intended.
161      # Can be removed when this job is no longer needed for full API coverage.
162      - csit-fail-on-crc-mismatch-parameter:
163          fail-on-crc-mismatch: 'True'
164
165    scm:
166      - gerrit-trigger-scm:
167          credentials-id: 'jenkins-gerrit-credentials'
168          refspec: '$GERRIT_REFSPEC'
169          choosing-strategy: 'gerrit'
170
171    wrappers:
172      - fdio-infra-wrappers-non-activity-timeout:
173          build-timeout: '{build-timeout}'
174
175    triggers:
176      - gerrit:
177          server-name: 'Primary'
178          trigger-on:
179            - comment-added-contains-event:
180                comment-contains-value: 'csit-{node-arch}-perftest'
181          projects:
182            - project-compare-type: 'ANT'
183              project-pattern: '{project}'
184              branches:
185                - branch-compare-type: 'ANT'
186                  branch-pattern: '**/{branch}'
187          skip-vote:
188            successful: false
189            failed: false
190            unstable: false
191            notbuilt: false
192
193    builders:
194      - shell:
195          !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
196
197    publishers:
198      - robot:
199          output-path: archive
200          other-files:
201            - '*.*'
202
203      - fdio-infra-shiplogs:
204          maven-version: 'mvn33-new'
205
206- job-template:
207    name: 'csit-vpp-perf-mrr-daily-master{node-arch}'
208
209    description: |
210      <ul>
211          <li>objective
212              <ul>
213                  <li>check vpp perf mrr daily
214              </ul>
215          <li>executor
216              <ul>
217                  <li>{executor}
218              </ul>
219          <li>perf test-cases selection
220              <ul>
221                  <li>all CSIT VPP MRR perf suites
222              </ul>
223          <li>results
224              <ul>
225                  <li>jjb verify: PASS/FAIL
226                  <li>present: no
227                  <li>analyse: csit project (catch malfunctions/degradations)
228              </ul>
229      </ul>
230
231    project-type: freestyle
232    node: '{executor}'
233    concurrent: false
234    archive-artifacts: 'csit/archive/*.*'
235    latest-only: false
236
237    build-discarder:
238      daysToKeep: '{build-days-to-keep}'
239      numToKeep: '{build-num-to-keep}'
240      artifactDaysToKeep: '{build-artifact-days-to-keep}'
241      artifactNumToKeep: '{build-artifact-num-to-keep}'
242
243    parameters:
244      - project-parameter:
245          project: '{project}'
246      - gerrit-parameter:
247          branch: '{branch}'
248      - csit-perf-trial-duration-parameter:
249          csit-perf-trial-duration: "{csit-perf-trial-duration}"
250      - csit-perf-trial-multiplicity-parameter:
251          csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
252      - string:
253          name: ARCHIVE_ARTIFACTS
254          default: '{archive-artifacts}'
255          description: Artifacts to archive to the logs server.
256
257    wrappers:
258      - fdio-infra-wrappers-non-activity-timeout:
259          build-timeout: '{build-timeout}'
260
261    triggers:
262      - timed: '{vpp-mrr-daily-periodicity}'
263
264    builders:
265      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
266      - inject:
267          properties-file: variables.prop
268      - shell:
269          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
270
271    publishers:
272      - robot:
273          output-path: 'csit/archive'
274          other-files:
275            - '*.*'
276
277      - fdio-infra-shiplogs:
278          maven-version: 'mvn33-new'
279
280- job-template:
281    name: 'csit-vpp-perf-mrr-weekly-master-{node-arch}'
282
283    description: |
284      <ul>
285          <li>objective
286              <ul>
287                  <li>check vpp perf mrr weekly on {node-arch}
288              </ul>
289          <li>executor
290              <ul>
291                  <li>{executor}
292              </ul>
293          <li>perf test-cases selection
294              <ul>
295                  <li>NF Density CSIT VPP MRR perf suites
296              </ul>
297          <li>results
298              <ul>
299                  <li>jjb verify: PASS/FAIL
300                  <li>present: no
301                  <li>analyse: csit project (catch malfunctions/degradations)
302              </ul>
303      </ul>
304
305    project-type: freestyle
306    node: '{executor}'
307    concurrent: false
308    archive-artifacts: 'csit/archive/*.*'
309    latest-only: false
310
311    build-discarder:
312      daysToKeep: '{build-days-to-keep}'
313      numToKeep: '{build-num-to-keep}'
314      artifactDaysToKeep: '{build-artifact-days-to-keep}'
315      artifactNumToKeep: '{build-artifact-num-to-keep}'
316
317    parameters:
318      - project-parameter:
319          project: '{project}'
320      - gerrit-parameter:
321          branch: '{branch}'
322      - csit-perf-trial-duration-parameter:
323          csit-perf-trial-duration: "{csit-perf-trial-duration}"
324      - csit-perf-trial-multiplicity-parameter:
325          csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
326      - string:
327          name: ARCHIVE_ARTIFACTS
328          default: '{archive-artifacts}'
329          description: Artifacts to archive to the logs server.
330
331    wrappers:
332      - fdio-infra-wrappers-non-activity-timeout:
333          build-timeout: '{build-timeout}'
334
335    triggers:
336      - timed: '0 0 * * 6'
337
338    builders:
339      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
340      - inject:
341          properties-file: variables.prop
342      - shell:
343          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
344
345    publishers:
346      - robot:
347          output-path: 'csit/archive'
348          other-files:
349            - '*.*'
350
351      - fdio-infra-shiplogs:
352          maven-version: 'mvn33-new'
353
354- job-template:
355    name: 'csit-vpp-perf-ndrpdr-weekly-master-{node-arch}'
356
357    description: |
358      <ul>
359          <li>objective
360              <ul>
361                  <li>check vpp perf ndrpdr weekly on {node-arch}
362              </ul>
363          <li>executor
364              <ul>
365                  <li>{executor}
366              </ul>
367          <li>perf test-cases selection
368              <ul>
369                  <li>pre-selected CSIT VPP NDRPDR perf suites
370              </ul>
371          <li>results
372              <ul>
373                  <li>jjb verify: PASS/FAIL
374                  <li>present: no
375                  <li>analyse: csit project (catch malfunctions/degradations)
376              </ul>
377      </ul>
378    project-type: freestyle
379    node: '{executor}'
380    concurrent: false
381    archive-artifacts: 'csit/archive/*.*'
382    latest-only: false
383
384    build-discarder:
385      daysToKeep: '{build-days-to-keep}'
386      numToKeep: '{build-num-to-keep}'
387      artifactDaysToKeep: '{build-artifact-days-to-keep}'
388      artifactNumToKeep: '{build-artifact-num-to-keep}'
389
390    parameters:
391      - project-parameter:
392          project: '{project}'
393      - gerrit-parameter:
394          branch: '{branch}'
395      - string:
396          name: ARCHIVE_ARTIFACTS
397          default: '{archive-artifacts}'
398          description: Artifacts to archive to the logs server.
399
400    wrappers:
401      - fdio-infra-wrappers-non-activity-timeout:
402          build-timeout: '{build-timeout}'
403
404    triggers:
405      - timed: '0 0 * * 6'
406
407    builders:
408      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
409      - inject:
410          properties-file: variables.prop
411      - shell:
412          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
413
414    publishers:
415      - robot:
416          output-path: 'csit/archive'
417          other-files:
418            - '*.*'
419
420      - fdio-infra-shiplogs:
421          maven-version: 'mvn33-new'
422
423- job-template:
424    name: 'csit-dpdk-perf-verify-{stream}-{node-arch}'
425
426    description: |
427      <ul>
428          <li>objective
429              <ul>
430                  <li>perf verify - per csit gerrit patch
431              </ul>
432          <li>job
433              <ul>
434                  <li>name: <b>csit-dpdk-perf-verify-{stream}-{node-arch}</b>
435                  <li>trigger: keyword in comment to csit patch:
436                      <b>csit-dpdk-{node-arch}-perftest tag_list</b>
437              </ul>
438          <li>executor
439              <ul>
440                  <li>{executor}
441              </ul>
442          <li>perf test-cases selection
443              <ul>
444                  <li>test-cases selected using trigger keyword followed by tags
445                  <li>all test-cases are labelled with CSIT RF topic tags
446                      defined in <i>csit/docs/tag_documentation.rst</i>
447              </ul>
448          <li>default tags for dpdk perf test type
449              <ul>
450                  <li>IP4BASE, L2XCBASE
451                  <li>64B: 64B packet size
452                  <li>1C: one core
453                  <li>X710: x710 Fortville NIC
454                  <li>MRR: MRR throughput
455              </ul>
456          <li>results
457              <ul>
458                  <li>jjb verify: PASS/FAIL
459                  <li>present: no
460                  <li>analyse: no
461              </ul>
462      </ul>
463
464    project-type: freestyle
465    node: '{executor}'
466    concurrent: true
467    archive-artifacts: 'archive/*.*'
468    latest-only: false
469
470    build-discarder:
471      daysToKeep: '{build-days-to-keep}'
472      numToKeep: '{build-num-to-keep}'
473      artifactDaysToKeep: '{build-artifact-days-to-keep}'
474      artifactNumToKeep: '{build-artifact-num-to-keep}'
475
476    parameters:
477      - project-parameter:
478          project: '{project}'
479      - gerrit-parameter:
480          branch: '{branch}'
481      - csit-perf-trial-duration-parameter:
482          csit-perf-trial-duration: "{csit-perf-trial-duration}"
483      - csit-perf-trial-multiplicity-parameter:
484          csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
485      - string:
486          name: ARCHIVE_ARTIFACTS
487          default: '{archive-artifacts}'
488          description: Artifacts to archive to the logs server.
489
490    scm:
491      - gerrit-trigger-scm:
492          credentials-id: 'jenkins-gerrit-credentials'
493          refspec: '$GERRIT_REFSPEC'
494          choosing-strategy: 'gerrit'
495
496    wrappers:
497      - fdio-infra-wrappers-non-activity-timeout:
498          build-timeout: '{build-timeout}'
499
500    triggers:
501      - gerrit:
502          server-name: 'Primary'
503          trigger-on:
504            - comment-added-contains-event:
505                comment-contains-value: 'csit-dpdk-{node-arch}-perftest'
506          projects:
507            - project-compare-type: 'ANT'
508              project-pattern: '{project}'
509              branches:
510                - branch-compare-type: 'ANT'
511                  branch-pattern: '**/{branch}'
512          skip-vote:
513            successful: false
514            failed: false
515            unstable: false
516            notbuilt: false
517
518    builders:
519      - shell:
520          !include-raw-escape: jjb/csit/include-raw-csit-perf-verify.sh
521
522    publishers:
523      - robot:
524          output-path: archive
525          other-files:
526            - '*.*'
527
528      - fdio-infra-shiplogs:
529          maven-version: 'mvn33-new'
530
531- job-template:
532    name: 'csit-dpdk-perf-mrr-weekly-master{node-arch}'
533
534    description: |
535      <ul>
536          <li>objective
537              <ul>
538                  <li>check dpdk perf mrr weekly
539              </ul>
540          <li>executor
541              <ul>
542                  <li>{executor}
543              </ul>
544          <li>perf test-cases selection
545              <ul>
546                  <li>all CSIT DPDK MRR perf suites
547              </ul>
548          <li>results
549              <ul>
550                  <li>jjb verify: PASS/FAIL
551                  <li>present: no
552                  <li>analyse: csit project (catch malfunctions/degradations)
553              </ul>
554      </ul>
555
556    project-type: freestyle
557    node: '{executor}'
558    concurrent: false
559    archive-artifacts: 'csit/archive/*.*'
560    latest-only: false
561
562    build-discarder:
563      daysToKeep: '{build-days-to-keep}'
564      numToKeep: '{build-num-to-keep}'
565      artifactDaysToKeep: '{build-artifact-days-to-keep}'
566      artifactNumToKeep: '{build-artifact-num-to-keep}'
567
568    parameters:
569      - project-parameter:
570          project: '{project}'
571      - gerrit-parameter:
572          branch: '{branch}'
573      - csit-perf-trial-duration-parameter:
574          csit-perf-trial-duration: "{csit-perf-trial-duration}"
575      - csit-perf-trial-multiplicity-parameter:
576          csit-perf-trial-multiplicity: "{csit-perf-trial-multiplicity}"
577      - string:
578          name: ARCHIVE_ARTIFACTS
579          default: '{archive-artifacts}'
580          description: Artifacts to archive to the logs server.
581
582    wrappers:
583      - fdio-infra-wrappers-non-activity-timeout:
584          build-timeout: '{build-timeout}'
585
586    triggers:
587      - timed: '0 0 * * 6'
588
589    builders:
590      - shell: 'echo "BRANCH_ID={branch-id}" > $WORKSPACE/variables.prop'
591      - inject:
592          properties-file: variables.prop
593      - shell:
594          !include-raw-escape: jjb/csit/include-raw-csit-perf-timed.sh
595
596    publishers:
597      - robot:
598          output-path: 'csit/archive'
599          other-files:
600            - '*.*'
601
602      - fdio-infra-shiplogs:
603          maven-version: 'mvn33-new'
604