remove --user now that pip is 20.0.x
[releng/global-jjb.git] / jjb / lf-macros.yaml
1 ---
2 ############
3 # BUILDERS #
4 ############
5
6 # To take advantage of this macro, have your build write
7 # out the file 'gerrit_comment.txt' with information to post
8 # back to gerrit and include this macro in the list of builders.
9 - builder:
10     name: comment-to-gerrit
11     builders:
12       - shell: !include-raw: ../shell/comment-to-gerrit.sh
13
14 - builder:
15     name: lf-fetch-dependent-patches
16     builders:
17       - shell: !include-raw: ../shell/gerrit-fetch-dependencies.sh
18       - inject:
19           properties-file: $WORKSPACE/.dependency.properties
20
21 - builder:
22     name: lf-license-check
23     builders:
24       - inject:
25           properties-content: |
26             FILE_PATTERNS={file-patterns}
27             SPDX_DISABLE={spdx-disable}
28             LHC_VERSION={lhc-version}
29             LICENSE_EXCLUDE_PATHS={license-exclude-paths}
30             LICENSES_ALLOWED={licenses-allowed}
31       - shell: !include-raw-escape:
32           - ../shell/license-check.sh
33
34 - builder:
35     name: lf-infra-create-netrc
36     builders:
37       - inject:
38           properties-content: "SERVER_ID={server-id}"
39       - shell: !include-raw-escape: ../shell/create-netrc.sh
40
41 - builder:
42     name: lf-infra-docker-login
43     builders:
44       - lf-provide-maven-settings:
45           global-settings-file: "{global-settings-file}"
46           settings-file: "{settings-file}"
47       - shell: !include-raw-escape: ../shell/docker-login.sh
48       - lf-provide-maven-settings-cleanup
49
50 - builder:
51     name: lf-infra-gpg-verify-git-signature
52     # TODO: Verify signature after downloading users public key from a locally
53     # created repository instead of the public keymesh. This requires a process
54     # in place to get ODL developers public keys into a local repository without
55     # increasing the job thoughput.
56     builders:
57       - shell: !include-raw: ../shell/gpg-verify-git-signature.sh
58
59 - builder:
60     name: lf-infra-pre-build
61     builders:
62       - shell: !include-raw-escape: ../shell/python-tools-install.sh
63
64 - builder:
65     name: lf-infra-ship-logs
66     builders:
67       # Ensure no pre-existing .netrc files are overriding logs config
68       - lf-provide-maven-settings-cleanup
69       - config-file-provider:
70           files:
71             - file-id: "jenkins-log-archives-settings"
72               variable: "SETTINGS_FILE"
73       - lf-infra-create-netrc:
74           server-id: logs
75       - shell: !include-raw:
76           # Ensure python-tools are installed in case job template does not
77           # call the lf-infra-pre-build macro.
78           - ../shell/python-tools-install.sh
79       - shell: !include-raw:
80           - ../shell/sudo-logs.sh
81       - shell: !include-raw:
82           - ../shell/job-cost.sh
83       - shell: !include-raw:
84           - ../shell/logs-deploy.sh
85       - shell: !include-raw:
86           - ../shell/logs-clear-credentials.sh
87       - description-setter:
88           regexp: "^Build logs: .*"
89
90 - builder:
91     name: lf-infra-package-listing
92     builders:
93       - shell: !include-raw:
94           - ../shell/package-listing.sh
95
96 - builder:
97     name: lf-infra-packer-build
98     builders:
99       - conditional-step:
100           condition-kind: boolean-expression
101           condition-expression: "{openstack}"
102           steps:
103             - config-file-provider:
104                 files:
105                   - file-id: clouds-yaml
106                     target: "$HOME/.config/openstack/clouds.yaml"
107             - inject:
108                 properties-content: OS_CLOUD={openstack-cloud}
109       - config-file-provider:
110           files:
111             - file-id: "{packer-cloud-settings}"
112               variable: CLOUDENV
113       - inject:
114           properties-content: |
115             PACKER_PLATFORM={platform}
116             PACKER_TEMPLATE={template}
117             PACKER_VERSION={packer-version}
118             UPDATE_CLOUD_IMAGE={update-cloud-image}
119       - shell: !include-raw-escape:
120           - ../shell/packer-install.sh
121           - ../shell/packer-build.sh
122       - shell: !include-raw:
123           - ../shell/packer-clear-credentials.sh
124
125 - builder:
126     name: lf-infra-packer-validate
127     builders:
128       - conditional-step:
129           condition-kind: boolean-expression
130           condition-expression: "{openstack}"
131           steps:
132             - config-file-provider:
133                 files:
134                   - file-id: clouds-yaml
135                     target: "$HOME/.config/openstack/clouds.yaml"
136             - inject:
137                 properties-content: OS_CLOUD={openstack-cloud}
138       - config-file-provider:
139           files:
140             - file-id: "{packer-cloud-settings}"
141               variable: CLOUDENV
142       - inject:
143           properties-content: |
144             PACKER_VERSION={packer-version}
145       - shell: !include-raw-escape:
146           - ../shell/packer-install.sh
147           - ../shell/packer-validate.sh
148       - shell: !include-raw:
149           - ../shell/packer-clear-credentials.sh
150
151 - builder:
152     name: lf-infra-update-packer-images
153     builders:
154       - shell: !include-raw: ../shell/update-cloud-images.sh
155
156 - builder:
157     name: lf-infra-push-gerrit-patch
158     builders:
159       - inject:
160           properties-content: |
161             PROJECT={project}
162             GERRIT_COMMIT_MESSAGE={gerrit-commit-message}
163             GERRIT_HOST={gerrit-host}
164             GERRIT_TOPIC={gerrit-topic}
165             GERRIT_USER={gerrit-user}
166             REVIEWERS_EMAIL={reviewers-email}
167       - shell: !include-raw-escape: ../shell/gerrit-push-patch.sh
168
169 - builder:
170     name: lf-infra-sysstat
171     builders:
172       - shell: !include-raw:
173           - ../shell/sysstat.sh
174
175 - builder:
176     name: lf-jacoco-nojava-workaround
177     builders:
178       - shell: "mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes"
179
180 - builder:
181     name: lf-infra-deploy-maven-file
182     builders:
183       - lf-maven-install:
184           mvn-version: "{mvn-version}"
185       - lf-provide-maven-settings:
186           global-settings-file: "{global-settings-file}"
187           settings-file: "{settings-file}"
188       - inject:
189           properties-content: |
190             MAVEN_REPO_URL={maven-repo-url}
191             REPO_ID={repo-id}
192             GROUP_ID={group-id}
193             UPLOAD_FILES_PATH={upload-files-dir}
194       - shell: !include-raw-escape:
195           - ../shell/common-variables.sh
196           - ../shell/deploy-maven-file.sh
197       - lf-provide-maven-settings-cleanup
198
199 - builder:
200     name: lf-maven-central
201     builders:
202       - conditional-step:
203           condition-kind: boolean-expression
204           condition-expression: "{mvn-central}"
205           steps:
206             - lf-provide-maven-settings:
207                 global-settings-file: "{mvn-global-settings}"
208                 settings-file: "{mvn-settings}"
209             - lf-infra-create-netrc:
210                 server-id: "ossrh"
211             - inject:
212                 properties-content: "OSSRH_PROFILE_ID={ossrh-profile-id}"
213             - shell: !include-raw-escape: ../shell/maven-central.sh
214             - lf-provide-maven-settings-cleanup
215
216 - builder:
217     name: lf-maven-install
218     builders:
219       # Create a $HOME/.wgetrc to make the Maven download quiet.
220       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
221       - maven-target:
222           maven-version: "{mvn-version}"
223           goals: "--version"
224       - shell: 'rm "$HOME/.wgetrc"'
225
226 - builder:
227     name: lf-provide-maven-settings
228     builders:
229       - config-file-provider:
230           files:
231             - file-id: "{global-settings-file}"
232               variable: "GLOBAL_SETTINGS_FILE"
233             - file-id: "{settings-file}"
234               variable: "SETTINGS_FILE"
235
236 - builder:
237     name: lf-provide-maven-settings-cleanup
238     builders:
239       - shell: |
240           #!/bin/bash
241           set +e  # DO NOT cause build failure if any of the rm calls fail.
242
243           rm -f "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE"
244
245           # In some cases we use the lf-provide-maven-settings macro to produce
246           # a "$HOME/.netrc" file containing credentials. Remove that file here
247           # too if it exists.
248           rm -f "$HOME/.netrc"
249
250           # DO NOT fail build if any of the above lines fail.
251           exit 0
252
253 - builder:
254     name: lf-rtd-trigger-build
255     builders:
256       - inject:
257           properties-content: |
258             RTD_BUILD_URL={rtd-build-url}
259             RTD_TOKEN={rtd-token}
260       - shell: !include-raw-escape: ../shell/rtd-trigger-build.sh
261
262 - builder:
263     name: lf-rtd-verify
264     builders:
265       - inject:
266           properties-content: |
267             DOC_DIR={doc-dir}
268             PYTHON={python-version}
269       - shell: !include-raw-escape: ../shell/tox-install.sh
270       - shell: !include-raw-escape: ../shell/rtd-verify.sh
271
272 - builder:
273     name: lf-rtdv3-build
274     builders:
275       - inject:
276           properties-content: |
277             DEFAULT_VERSION={default-version}
278       - shell: !include-raw-escape: ../shell/rtdv3.sh
279
280 - builder:
281     name: check-info-votes
282     builders:
283       - inject:
284           properties-content: JENKINS_SSH_CREDENTIAL={jenkins-ssh-credential}
285       - shell: !include-raw-escape: ../shell/check-info-votes.sh
286
287 - builder:
288     name: lf-release
289     builders:
290       - config-file-provider:
291           files:
292             - file-id: lftoolsini
293               target: "$HOME/.config/lftools/lftools.ini"
294       - shell: !include-raw: ../shell/release-job.sh
295
296 - builder:
297     name: lf-sigul-sign-dir
298     # Requires that Jenkins be configured with SIGUL_BRIDGE_IP as a global
299     # Environment variable
300     builders:
301       - conditional-step:
302           condition-kind: boolean-expression
303           condition-expression: "{sign-artifacts}"
304           steps:
305             - config-file-provider:
306                 files:
307                   - file-id: sigul-config
308                     variable: SIGUL_CONFIG
309                   - file-id: sigul-password
310                     variable: SIGUL_PASSWORD
311                   - file-id: sigul-pki
312                     variable: SIGUL_PKI
313             - shell: !include-raw-escape: ../shell/sigul-configuration.sh
314             - shell: !include-raw-escape: ../shell/sigul-install.sh
315             - inject:
316                 properties-content: |
317                   SIGN_DIR={sign-dir}
318                   SIGN_MODE={sign-mode}
319             - shell: !include-raw-escape:
320                 - ../shell/sigul-sign-dir.sh
321             - shell: !include-raw-escape: ../shell/sigul-configuration-cleanup.sh
322
323 - builder:
324     name: lf-infra-provide-docker-cleanup
325     builders:
326       - shell: |
327           #!/bin/bash
328           set +e  # DO NOT cause build failure if docker rmi fails
329           docker rmi -f $(docker images -a -q)
330           exit 0
331
332 - builder:
333     name: lf-infra-sonar
334     # Run a Sonar Jenkins Plugin
335     builders:
336       - sonar:
337           sonar-name: Sonar
338           scanner-name: SonarQubeScanner
339           task: "{sonar-task}"
340           project: "{sonar-project-file}"
341           properties: "{sonar-properties}"
342           java-opts: "{sonar-java-opts}"
343           additional-arguments: "{sonar-additional-args}"
344
345 - builder:
346     name: lf-infra-sonar-with-prescan
347     # Run a Sonar Jenkins Plugin
348     builders:
349       - lf-sonar-prescan # Must be defined by caller
350       - sonar:
351           sonar-name: Sonar
352           scanner-name: SonarQubeScanner
353           task: "{sonar-task}"
354           project: "{sonar-project-file}"
355           properties: "{sonar-properties}"
356           java-opts: "{sonar-java-opts}"
357           additional-arguments: "{sonar-additional-args}"
358
359 ##############
360 # PARAMETERS #
361 ##############
362
363 - parameter:
364     name: lf-clm-parameters
365     parameters:
366       - string:
367           name: NEXUS_IQ_STAGE
368           default: "{nexus-iq-stage}"
369           description: |
370             Stage the policy evaluation will be run against on the Nexus IQ Server.
371
372 - parameter:
373     name: lf-cmake-parameters
374     # Parameters useful in CMake builds.
375     parameters:
376       - string:
377           name: BUILD_DIR
378           default: "{build-dir}"
379           description: "Directory to build the project in."
380       - string:
381           name: CMAKE_OPTS
382           default: "{cmake-opts}"
383           description: |
384             Options to pass to CMAKE.
385             Example: -DCMAKE_INSTALL_PREFIX=$WORKSPACE/target/output
386       - string:
387           name: INSTALL_PREFIX
388           default: "{install-prefix}"
389           description: "Install prefix for C/C++ builds. Example: /usr/local"
390       - string:
391           name: MAKE_OPTS
392           default: "{make-opts}"
393           description: "Options to pass to make. Example: -j8"
394
395 - parameter:
396     name: lf-infra-maven-parameters
397     parameters:
398       - string:
399           name: M2_HOME
400           # Sets an env var for shell scripts to be able to call the dynamically
401           # installed maven without having to calculate the M2_HOME themselves.
402           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}"
403           description: "Maven selector to be used by shell scripts"
404       - string:
405           name: MAVEN_OPTS
406           default: "{mvn-opts}"
407           description: |
408             Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
409       - string:
410           name: MAVEN_PARAMS
411           default: "{mvn-params}"
412           description: |
413             Maven parameters to pass to the mvn command.
414       - string:
415           name: MVN
416           # Sets an env var for shell scripts to be able to call the dynamically
417           # installed maven without having to calculate the path themselves.
418           # yamllint disable-line rule:line-length
419           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn"
420           description: "Maven selector to be used by shell scripts"
421
422 - parameter:
423     name: lf-infra-openstack-parameters
424     parameters:
425       - string:
426           name: OS_CLOUD
427           default: "{os-cloud}"
428           description: |
429             The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
430             variable name that is significant to openstack client as a
431             environment variable. Please refer to the documentation for
432             further details.
433             https://docs.openstack.org/developer/python-openstackclient/
434
435 - parameter:
436     name: lf-infra-parameters
437     # Standard parameters used in the LF CI environments. Gerrit variables are
438     # not used by GitHub projects, but defining them isn't harmful.
439     parameters:
440       - string:
441           name: PROJECT
442           default: "{project}"
443           description: |
444             Parameter to identify a SCM project to build. This is typically
445             the project repo path. For example: ofextensions/circuitsw
446       - string:
447           name: STREAM
448           default: "{stream}"
449           description: |
450             Stream is often set to the same name as 'branch' but can
451             sometimes be used as a name representing a project's release code
452             name.
453       - string:
454           name: GERRIT_PROJECT
455           default: "{project}"
456           description: |
457             Gerrit Trigger provided parameter to identify Gerrit project that
458             triggered the build. This is typically the project repo path as
459             exists in Gerrit. For example: ofextensions/circuitsw
460
461             If using Gerrit, in a manual build this should match the PROJECT
462             parameter above.
463       - string:
464           name: GERRIT_BRANCH
465           default: "{branch}"
466           description: |
467             Gerrit Trigger provided parameter to identify a Gerrit branch.
468
469             If using Gerrit, in a manual build override with the branch to
470             build against.
471       - string:
472           name: GERRIT_REFSPEC
473           default: "refs/heads/{branch}"
474           description: |
475             Gerrit Trigger provided parameter to identify a refspec to fetch
476             from Gerrit.
477
478             If using Gerrit, in a manual build override with a refspec.
479             https://git-scm.com/book/en/v2/Git-Internals-The-Refspec
480             For example: 'refs/heads/master'
481       - string:
482           name: sha1
483           default: "origin/{branch}"
484           description: |
485             GitHub PR Trigger provided parameter for specifying the commit
486             to checkout.
487
488             If using GitHub, in a manual build override with a branch path or
489             sha1 hash to a specific commit. For example: 'origin/master'
490
491 - parameter:
492     name: lf-infra-node-parameters
493     parameters:
494       - string:
495           name: NODE_DIR
496           default: "{node-dir}"
497           description: Path to a Node project directory.
498       - string:
499           name: NODE_VERSION
500           default: "{node-version}"
501           description: Version of NodeJS to install.
502
503 - parameter:
504     name: lf-infra-tox-parameters
505     parameters:
506       - string:
507           name: TOX_DIR
508           default: "{tox-dir}"
509           description: |
510             Path to directory containing tox.ini file.
511       - string:
512           name: TOX_ENVS
513           default: "{tox-envs}"
514           description: |
515             Tox environments to run build against.
516             Example: docs,py2,py3
517
518 - parameter:
519     name: lf-build-with-parameters-maven-release
520     parameters:
521       - string:
522           name: VERSION
523           default: ""
524           description: "Version to release, for example: 1.0.0"
525       - string:
526           name: GIT_TAG
527           default: ""
528           description: "Tag to push to git repo; optional, defaults to VERSION"
529       - string:
530           name: LOG_DIR
531           default: ""
532           description: "Log dir, for example: project-maven-stage-master/17/"
533       - choice:
534           name: DISTRIBUTION_TYPE
535           choices:
536             - None
537             - container
538             - maven
539           description: "Set to maven for build with parameters"
540       - bool:
541           name: USE_RELEASE_FILE
542           default: "{use-release-file}"
543           description: "Set to False for job built with parameters"
544
545 ##############
546 # PROPERTIES #
547 ##############
548
549 - property:
550     name: lf-infra-properties
551     properties:
552       - build-discarder:
553           # Allow build data to be stored at a length configured by the
554           # downstream project.
555           days-to-keep: "{build-days-to-keep}"
556           # Do not allow artifacts to be stored in Jenkins.
557           artifact-num-to-keep: 0
558
559 ##############
560 # PUBLISHERS #
561 ##############
562
563 - publisher:
564     name: lf-jacoco-report
565     publishers:
566       - jacoco:
567           exec-pattern: "**/**.exec"
568           class-pattern: "**/classes"
569           source-pattern: "**/src/main/java"
570           # yamllint disable-line rule:line-length
571           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
572           status-update: true
573           targets:
574             - branch:
575                 healthy: 10
576                 unhealthy: 20
577             - method:
578                 healthy: 50
579                 unhealthy: 40
580
581 - publisher:
582     name: lf-infra-publish
583     # lf-infra macro to finish up a build.
584     #
585     # Handles the following:
586     #   - Shipping logs to Nexus logs site repository
587     #   - Cleanup workspace
588     publishers:
589       - postbuildscript:
590           builders:
591             - role: BOTH
592               build-on:
593                 - ABORTED
594                 - FAILURE
595                 - NOT_BUILT
596                 - SUCCESS
597                 - UNSTABLE
598               build-steps:
599                 - lf-infra-sysstat
600                 - lf-infra-package-listing
601                 - lf-infra-ship-logs
602           mark-unstable-if-failed: true
603       - workspace-cleanup:
604           exclude:
605             # Do not clean up *.jenkins-trigger files for jobs that use a
606             # properties file as input for triggering another build.
607             - "**/*.jenkins-trigger"
608           fail-build: false
609
610 - publisher:
611     name: lf-infra-publish-windows
612     # lf-infra macro to finish up a build.
613     #
614     # Handles the following:
615     #   - Shipping logs to Nexus logs site repository
616     #   - Cleanup workspace
617     publishers:
618       # TODO: RELENG-1228 Develop log shipping script for Windows systems
619       # - postbuildscript:
620       #     builders:
621       #       - role: BOTH
622       #         build-on:
623       #           - ABORTED
624       #           - FAILURE
625       #           - NOT_BUILT
626       #           - SUCCESS
627       #           - UNSTABLE
628       #         build-steps:
629       #           - lf-infra-ship-logs-windows
630       #     mark-unstable-if-failed: true
631       - workspace-cleanup:
632           exclude:
633             # Do not clean up *.jenkins-trigger files for jobs that use a
634             # properties file as input for triggering another build.
635             - "**/*.jenkins-trigger"
636           fail-build: false
637
638 #######
639 # SCM #
640 #######
641
642 - scm:
643     name: lf-infra-gerrit-scm
644     scm:
645       - git:
646           credentials-id: "{jenkins-ssh-credential}"
647           url: "{git-url}"
648           refspec: "{refspec}"
649           branches:
650             - "refs/heads/{branch}"
651           skip-tag: true
652           wipe-workspace: true
653           submodule:
654             disable: "{submodule-disable}"
655             recursive: "{submodule-recursive}"
656             timeout: "{submodule-timeout}"
657           choosing-strategy: "{choosing-strategy}"
658
659 - scm:
660     name: lf-infra-github-scm
661     scm:
662       - git:
663           credentials-id: "{jenkins-ssh-credential}"
664           url: "{url}"
665           refspec: "{refspec}"
666           branches:
667             - "{branch}"
668           skip-tag: true
669           wipe-workspace: true
670           submodule:
671             disable: "{submodule-disable}"
672             recursive: "{submodule-recursive}"
673             timeout: "{submodule-timeout}"
674           choosing-strategy: "{choosing-strategy}"
675
676 ############
677 # WRAPPERS #
678 ############
679
680 - wrapper:
681     name: lf-infra-wrappers-common
682     wrappers:
683       - mask-passwords
684       - timeout:
685           type: absolute
686           timeout: "{build-timeout}"
687           timeout-var: "BUILD_TIMEOUT"
688           fail: true
689       - timestamps
690       - openstack:
691           single-use: true
692
693 - wrapper:
694     name: lf-infra-wrappers
695     wrappers:
696       - lf-infra-wrappers-common:
697           build-timeout: "{build-timeout}"
698       - config-file-provider:
699           files:
700             - file-id: npmrc
701               target: "$HOME/.npmrc"
702             - file-id: pipconf
703               target: "$HOME/.config/pip/pip.conf"
704       - ssh-agent-credentials:
705           users:
706             - "{jenkins-ssh-credential}"
707
708 - wrapper:
709     name: lf-infra-wrappers-windows
710     wrappers:
711       - lf-infra-wrappers-common:
712           build-timeout: "{build-timeout}"