Fix OS_CLOUD export for image validation
[releng/global-jjb.git] / jjb / lf-macros.yaml
1 ---
2 ############
3 # BUILDERS #
4 ############
5
6 # To take advantage of this macro, have your build write
7 # out the file 'gerrit_comment.txt' with information to post
8 # back to gerrit and include this macro in the list of builders.
9 - builder:
10     name: comment-to-gerrit
11     builders:
12       - shell: !include-raw: ../shell/comment-to-gerrit.sh
13
14 - builder:
15     name: lf-fetch-dependent-patches
16     builders:
17       - shell: !include-raw: ../shell/gerrit-fetch-dependencies.sh
18       - inject:
19           properties-file: $WORKSPACE/.dependency.properties
20
21 - builder:
22     name: lf-license-check
23     builders:
24       - inject:
25           properties-content: |
26             FILE_PATTERNS={file-patterns}
27             SPDX_DISABLE={spdx-disable}
28             LHC_VERSION={lhc-version}
29             LICENSE_EXCLUDE_PATHS={license-exclude-paths}
30             LICENSES_ALLOWED={licenses-allowed}
31       - shell: !include-raw-escape:
32           - ../shell/license-check.sh
33
34 - builder:
35     name: lf-infra-create-netrc
36     builders:
37       - inject:
38           properties-content: "SERVER_ID={server-id}"
39       - shell: !include-raw-escape: ../shell/create-netrc.sh
40
41 - builder:
42     name: lf-infra-docker-login
43     builders:
44       - lf-provide-maven-settings:
45           global-settings-file: "{global-settings-file}"
46           settings-file: "{settings-file}"
47       - shell: !include-raw-escape: ../shell/docker-login.sh
48       - lf-provide-maven-settings-cleanup
49
50 - builder:
51     name: lf-infra-gpg-verify-git-signature
52     # TODO: Verify signature after downloading users public key from a locally
53     # created repository instead of the public keymesh. This requires a process
54     # in place to get ODL developers public keys into a local repository without
55     # increasing the job thoughput.
56     builders:
57       - shell: !include-raw: ../shell/gpg-verify-git-signature.sh
58
59 - builder:
60     name: lf-infra-pre-build
61     builders:
62       - shell: !include-raw-escape: ../shell/python-tools-install.sh
63
64 - builder:
65     name: lf-infra-ship-logs
66     builders:
67       # Ensure no pre-existing .netrc files are overriding logs config
68       - lf-provide-maven-settings-cleanup
69       - config-file-provider:
70           files:
71             - file-id: "jenkins-log-archives-settings"
72               variable: "SETTINGS_FILE"
73       - lf-infra-create-netrc:
74           server-id: logs
75       - shell: !include-raw:
76           # Ensure python-tools are installed in case job template does not
77           # call the lf-infra-pre-build macro.
78           - ../shell/python-tools-install.sh
79       - shell: !include-raw:
80           - ../shell/sudo-logs.sh
81       - shell: !include-raw:
82           - ../shell/logs-deploy.sh
83       - shell: !include-raw:
84           - ../shell/logs-clear-credentials.sh
85       - description-setter:
86           regexp: "^Build logs: .*"
87
88 - builder:
89     name: lf-infra-package-listing
90     builders:
91       - shell: !include-raw:
92           - ../shell/package-listing.sh
93
94 - builder:
95     name: lf-infra-packer-build
96     builders:
97       - conditional-step:
98           condition-kind: boolean-expression
99           condition-expression: "{openstack}"
100           steps:
101             - config-file-provider:
102                 files:
103                   - file-id: clouds-yaml
104                     target: "$HOME/.config/openstack/clouds.yaml"
105             - inject:
106                 properties-content: OS_CLOUD={openstack-cloud}
107       - config-file-provider:
108           files:
109             - file-id: "{packer-cloud-settings}"
110               variable: CLOUDENV
111       - inject:
112           properties-content: |
113             PACKER_PLATFORM={platform}
114             PACKER_TEMPLATE={template}
115             PACKER_VERSION={packer-version}
116             UPDATE_CLOUD_IMAGE={update-cloud-image}
117       - shell: !include-raw-escape:
118           - ../shell/packer-install.sh
119           - ../shell/packer-build.sh
120       - shell: !include-raw:
121           - ../shell/packer-clear-credentials.sh
122
123 - builder:
124     name: lf-infra-packer-validate
125     builders:
126       - conditional-step:
127           condition-kind: boolean-expression
128           condition-expression: "{openstack}"
129           steps:
130             - config-file-provider:
131                 files:
132                   - file-id: clouds-yaml
133                     target: "$HOME/.config/openstack/clouds.yaml"
134             - inject:
135                 properties-content: OS_CLOUD={openstack-cloud}
136       - config-file-provider:
137           files:
138             - file-id: "{packer-cloud-settings}"
139               variable: CLOUDENV
140       - inject:
141           properties-content: |
142             PACKER_VERSION={packer-version}
143       - shell: !include-raw-escape:
144           - ../shell/packer-install.sh
145           - ../shell/packer-validate.sh
146       - shell: !include-raw:
147           - ../shell/packer-clear-credentials.sh
148
149 - builder:
150     name: lf-infra-update-packer-images
151     builders:
152       - shell: !include-raw: ../shell/update-cloud-images.sh
153
154 - builder:
155     name: lf-infra-push-gerrit-patch
156     builders:
157       - inject:
158           properties-content: |
159             PROJECT={project}
160             GERRIT_COMMIT_MESSAGE={gerrit-commit-message}
161             GERRIT_HOST={gerrit-host}
162             GERRIT_TOPIC={gerrit-topic}
163             GERRIT_USER={gerrit-user}
164             REVIEWERS_EMAIL={reviewers-email}
165       - shell: !include-raw-escape: ../shell/gerrit-push-patch.sh
166
167 - builder:
168     name: lf-infra-sysstat
169     builders:
170       - shell: !include-raw:
171           - ../shell/sysstat.sh
172
173 - builder:
174     name: lf-jacoco-nojava-workaround
175     builders:
176       - shell: "mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes"
177
178 - builder:
179     name: lf-infra-deploy-maven-file
180     builders:
181       - lf-maven-install:
182           mvn-version: "{mvn-version}"
183       - lf-provide-maven-settings:
184           global-settings-file: "{global-settings-file}"
185           settings-file: "{settings-file}"
186       - inject:
187           properties-content: |
188             MAVEN_REPO_URL={maven-repo-url}
189             REPO_ID={repo-id}
190             GROUP_ID={group-id}
191             UPLOAD_FILES_PATH={upload-files-dir}
192       - shell: !include-raw-escape:
193           - ../shell/common-variables.sh
194           - ../shell/deploy-maven-file.sh
195       - lf-provide-maven-settings-cleanup
196
197 - builder:
198     name: lf-maven-central
199     builders:
200       - conditional-step:
201           condition-kind: boolean-expression
202           condition-expression: "{mvn-central}"
203           steps:
204             - lf-provide-maven-settings:
205                 global-settings-file: "{mvn-global-settings}"
206                 settings-file: "{mvn-settings}"
207             - lf-infra-create-netrc:
208                 server-id: "ossrh"
209             - inject:
210                 properties-content: "OSSRH_PROFILE_ID={ossrh-profile-id}"
211             - shell: !include-raw-escape: ../shell/maven-central.sh
212             - lf-provide-maven-settings-cleanup
213
214 - builder:
215     name: lf-maven-install
216     builders:
217       # Create a $HOME/.wgetrc to make the Maven download quiet.
218       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
219       - maven-target:
220           maven-version: "{mvn-version}"
221           goals: "--version"
222       - shell: 'rm "$HOME/.wgetrc"'
223
224 - builder:
225     name: lf-provide-maven-settings
226     builders:
227       - config-file-provider:
228           files:
229             - file-id: "{global-settings-file}"
230               variable: "GLOBAL_SETTINGS_FILE"
231             - file-id: "{settings-file}"
232               variable: "SETTINGS_FILE"
233
234 - builder:
235     name: lf-pip-install
236     builders:
237       - inject:
238           properties-content: PIP_PACKAGES={pip-packages}
239       - shell: !include-raw-escape: ../shell/pip-install.sh
240
241 - builder:
242     name: lf-provide-maven-settings-cleanup
243     builders:
244       - shell: |
245           #!/bin/bash
246           set +e  # DO NOT cause build failure if any of the rm calls fail.
247
248           rm -f "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE"
249
250           # In some cases we use the lf-provide-maven-settings macro to produce
251           # a "$HOME/.netrc" file containing credentials. Remove that file here
252           # too if it exists.
253           rm -f "$HOME/.netrc"
254
255           # DO NOT fail build if any of the above lines fail.
256           exit 0
257
258 - builder:
259     name: lf-rtd-trigger-build
260     builders:
261       - inject:
262           properties-content: |
263             RTD_BUILD_URL={rtd-build-url}
264             RTD_TOKEN={rtd-token}
265       - shell: !include-raw-escape: ../shell/rtd-trigger-build.sh
266
267 - builder:
268     name: lf-rtd-verify
269     builders:
270       - inject:
271           properties-content: |
272             DOC_DIR={doc-dir}
273             PYTHON={python-version}
274       - shell: !include-raw-escape: ../shell/tox-install.sh
275       - shell: !include-raw-escape: ../shell/rtd-verify.sh
276
277 - builder:
278     name: lf-rtdv3-build
279
280     builders:
281       - conditional-step:
282           condition-kind: file-exists
283           condition-filename: .readthedocs.yaml
284           on-evaluation-failure: dont-run
285           steps:
286             - inject:
287                 properties-content: |
288                   TOX_ENVS=docs,docs-linkcheck
289             - lf-infra-pre-build
290             - lf-infra-tox-install:
291                 python-version: "python3"
292             - shell: !include-raw: ../shell/tox-run.sh
293             - shell: !include-raw: ../shell/rtdv3.sh
294
295 - builder:
296     name: check-info-votes
297     builders:
298       - inject:
299           properties-content: JENKINS_SSH_CREDENTIAL={jenkins-ssh-credential}
300       - shell: !include-raw-escape: ../shell/check-info-votes.sh
301
302 - builder:
303     name: lf-release
304     builders:
305       - config-file-provider:
306           files:
307             - file-id: lftoolsini
308               target: "$HOME/.config/lftools/lftools.ini"
309       - shell: !include-raw: ../shell/release-job.sh
310
311 - builder:
312     name: lf-sigul-sign-dir
313     # Requires that Jenkins be configured with SIGUL_BRIDGE_IP as a global
314     # Environment variable
315     builders:
316       - conditional-step:
317           condition-kind: boolean-expression
318           condition-expression: "{sign-artifacts}"
319           steps:
320             - config-file-provider:
321                 files:
322                   - file-id: sigul-config
323                     variable: SIGUL_CONFIG
324                   - file-id: sigul-password
325                     variable: SIGUL_PASSWORD
326                   - file-id: sigul-pki
327                     variable: SIGUL_PKI
328             - shell: !include-raw-escape: ../shell/sigul-configuration.sh
329             - shell: !include-raw-escape: ../shell/sigul-install.sh
330             - inject:
331                 properties-content: |
332                   SIGN_DIR={sign-dir}
333                   SIGN_MODE={sign-mode}
334             - shell: !include-raw-escape:
335                 - ../shell/sigul-sign-dir.sh
336             - shell: !include-raw-escape: ../shell/sigul-configuration-cleanup.sh
337
338 - builder:
339     name: lf-infra-provide-docker-cleanup
340     builders:
341       - shell: |
342           #!/bin/bash
343           set +e  # DO NOT cause build failure if docker rmi fails
344           docker rmi -f $(docker images -a -q)
345           exit 0
346
347 - builder:
348     name: lf-infra-sonar
349     # Run a Sonar Jenkins Plugin
350     builders:
351       - sonar:
352           sonar-name: Sonar
353           scanner-name: SonarQubeScanner
354           task: "{sonar-task}"
355           project: "{sonar-project-file}"
356           properties: "{sonar-properties}"
357           java-opts: "{sonar-java-opts}"
358           additional-arguments: "{sonar-additional-args}"
359
360 - builder:
361     name: lf-infra-sonar-with-prescan
362     # Run a Sonar Jenkins Plugin
363     builders:
364       - lf-sonar-prescan # Must be defined by caller
365       - sonar:
366           sonar-name: Sonar
367           scanner-name: SonarQubeScanner
368           task: "{sonar-task}"
369           project: "{sonar-project-file}"
370           properties: "{sonar-properties}"
371           java-opts: "{sonar-java-opts}"
372           additional-arguments: "{sonar-additional-args}"
373
374 ##############
375 # PARAMETERS #
376 ##############
377
378 - parameter:
379     name: lf-clm-parameters
380     parameters:
381       - string:
382           name: NEXUS_IQ_STAGE
383           default: "{nexus-iq-stage}"
384           description: |
385             Stage the policy evaluation will be run against on the Nexus IQ Server.
386
387 - parameter:
388     name: lf-cmake-parameters
389     # Parameters useful in CMake builds.
390     parameters:
391       - string:
392           name: BUILD_DIR
393           default: "{build-dir}"
394           description: "Directory to build the project in."
395       - string:
396           name: CMAKE_OPTS
397           default: "{cmake-opts}"
398           description: |
399             Options to pass to CMAKE.
400             Example: -DCMAKE_INSTALL_PREFIX=$WORKSPACE/target/output
401       - string:
402           name: INSTALL_PREFIX
403           default: "{install-prefix}"
404           description: "Install prefix for C/C++ builds. Example: /usr/local"
405       - string:
406           name: MAKE_OPTS
407           default: "{make-opts}"
408           description: "Options to pass to make. Example: -j8"
409
410 - parameter:
411     name: lf-infra-maven-parameters
412     parameters:
413       - string:
414           name: M2_HOME
415           # Sets an env var for shell scripts to be able to call the dynamically
416           # installed maven without having to calculate the M2_HOME themselves.
417           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}"
418           description: "Maven selector to be used by shell scripts"
419       - string:
420           name: MAVEN_OPTS
421           default: "{mvn-opts}"
422           description: |
423             Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
424       - string:
425           name: MAVEN_PARAMS
426           default: "{mvn-params}"
427           description: |
428             Maven parameters to pass to the mvn command.
429       - string:
430           name: MVN
431           # Sets an env var for shell scripts to be able to call the dynamically
432           # installed maven without having to calculate the path themselves.
433           # yamllint disable-line rule:line-length
434           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn"
435           description: "Maven selector to be used by shell scripts"
436
437 - parameter:
438     name: lf-infra-openstack-parameters
439     parameters:
440       - string:
441           name: OS_CLOUD
442           default: "{os-cloud}"
443           description: |
444             The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
445             variable name that is significant to openstack client as a
446             environment variable. Please refer to the documentation for
447             further details.
448             https://docs.openstack.org/developer/python-openstackclient/
449
450 - parameter:
451     name: lf-infra-parameters
452     # Standard parameters used in the LF CI environments. Gerrit variables are
453     # not used by GitHub projects, but defining them isn't harmful.
454     parameters:
455       - string:
456           name: PROJECT
457           default: "{project}"
458           description: |
459             Parameter to identify a SCM project to build. This is typically
460             the project repo path. For example: ofextensions/circuitsw
461       - string:
462           name: STREAM
463           default: "{stream}"
464           description: |
465             Stream is often set to the same name as 'branch' but can
466             sometimes be used as a name representing a project's release code
467             name.
468       - string:
469           name: GERRIT_PROJECT
470           default: "{project}"
471           description: |
472             Gerrit Trigger provided parameter to identify Gerrit project that
473             triggered the build. This is typically the project repo path as
474             exists in Gerrit. For example: ofextensions/circuitsw
475
476             If using Gerrit, in a manual build this should match the PROJECT
477             parameter above.
478       - string:
479           name: GERRIT_BRANCH
480           default: "{branch}"
481           description: |
482             Gerrit Trigger provided parameter to identify a Gerrit branch.
483
484             If using Gerrit, in a manual build override with the branch to
485             build against.
486       - string:
487           name: GERRIT_REFSPEC
488           default: "refs/heads/{branch}"
489           description: |
490             Gerrit Trigger provided parameter to identify a refspec to fetch
491             from Gerrit.
492
493             If using Gerrit, in a manual build override with a refspec.
494             https://git-scm.com/book/en/v2/Git-Internals-The-Refspec
495             For example: 'refs/heads/master'
496       - string:
497           name: sha1
498           default: "origin/{branch}"
499           description: |
500             GitHub PR Trigger provided parameter for specifying the commit
501             to checkout.
502
503             If using GitHub, in a manual build override with a branch path or
504             sha1 hash to a specific commit. For example: 'origin/master'
505
506 - parameter:
507     name: lf-infra-node-parameters
508     parameters:
509       - string:
510           name: NODE_DIR
511           default: "{node-dir}"
512           description: Path to a Node project directory.
513       - string:
514           name: NODE_VERSION
515           default: "{node-version}"
516           description: Version of NodeJS to install.
517
518 - parameter:
519     name: lf-infra-tox-parameters
520     parameters:
521       - string:
522           name: TOX_DIR
523           default: "{tox-dir}"
524           description: |
525             Path to directory containing tox.ini file.
526       - string:
527           name: TOX_ENVS
528           default: "{tox-envs}"
529           description: |
530             Tox environments to run build against.
531             Example: docs,py2,py3
532
533 - parameter:
534     name: lf-build-with-parameters-maven-release
535     parameters:
536       - string:
537           name: VERSION
538           default: ""
539           description: "This is the tag, Example: 1.0.0"
540       - string:
541           name: LOG_DIR
542           default: ""
543           description: "Log dir, example: project-maven-stage-master/17/"
544       - choice:
545           name: DISTRIBUTION_TYPE
546           choices:
547             - None
548             - container
549             - maven
550           description: "Set to maven for build with parameters"
551       - bool:
552           name: USE_RELEASE_FILE
553           default: "{use-release-file}"
554           description: "Set to False for job built with parameters"
555
556 ##############
557 # PROPERTIES #
558 ##############
559
560 - property:
561     name: lf-infra-properties
562     properties:
563       - build-discarder:
564           # Allow build data to be stored at a length configured by the
565           # downstream project.
566           days-to-keep: "{build-days-to-keep}"
567           # Do not allow artifacts to be stored in Jenkins.
568           artifact-num-to-keep: 0
569
570 ##############
571 # PUBLISHERS #
572 ##############
573
574 - publisher:
575     name: lf-jacoco-report
576     publishers:
577       - jacoco:
578           exec-pattern: "**/**.exec"
579           class-pattern: "**/classes"
580           source-pattern: "**/src/main/java"
581           # yamllint disable-line rule:line-length
582           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
583           status-update: true
584           targets:
585             - branch:
586                 healthy: 10
587                 unhealthy: 20
588             - method:
589                 healthy: 50
590                 unhealthy: 40
591
592 - publisher:
593     name: lf-infra-publish
594     # lf-infra macro to finish up a build.
595     #
596     # Handles the following:
597     #   - Shipping logs to Nexus logs site repository
598     #   - Cleanup workspace
599     publishers:
600       - postbuildscript:
601           builders:
602             - role: BOTH
603               build-on:
604                 - ABORTED
605                 - FAILURE
606                 - NOT_BUILT
607                 - SUCCESS
608                 - UNSTABLE
609               build-steps:
610                 - lf-infra-sysstat
611                 - lf-infra-package-listing
612                 - lf-infra-ship-logs
613           mark-unstable-if-failed: true
614       - workspace-cleanup:
615           exclude:
616             # Do not clean up *.jenkins-trigger files for jobs that use a
617             # properties file as input for triggering another build.
618             - "**/*.jenkins-trigger"
619           fail-build: false
620
621 - publisher:
622     name: lf-infra-publish-windows
623     # lf-infra macro to finish up a build.
624     #
625     # Handles the following:
626     #   - Shipping logs to Nexus logs site repository
627     #   - Cleanup workspace
628     publishers:
629       # TODO: RELENG-1228 Develop log shipping script for Windows systems
630       # - postbuildscript:
631       #     builders:
632       #       - role: BOTH
633       #         build-on:
634       #           - ABORTED
635       #           - FAILURE
636       #           - NOT_BUILT
637       #           - SUCCESS
638       #           - UNSTABLE
639       #         build-steps:
640       #           - lf-infra-ship-logs-windows
641       #     mark-unstable-if-failed: true
642       - workspace-cleanup:
643           exclude:
644             # Do not clean up *.jenkins-trigger files for jobs that use a
645             # properties file as input for triggering another build.
646             - "**/*.jenkins-trigger"
647           fail-build: false
648
649 #######
650 # SCM #
651 #######
652
653 - scm:
654     name: lf-infra-gerrit-scm
655     scm:
656       - git:
657           credentials-id: "{jenkins-ssh-credential}"
658           url: "{git-url}"
659           refspec: "{refspec}"
660           branches:
661             - "refs/heads/{branch}"
662           skip-tag: true
663           wipe-workspace: true
664           submodule:
665             disable: "{submodule-disable}"
666             recursive: "{submodule-recursive}"
667             timeout: "{submodule-timeout}"
668           choosing-strategy: "{choosing-strategy}"
669
670 - scm:
671     name: lf-infra-github-scm
672     scm:
673       - git:
674           credentials-id: "{jenkins-ssh-credential}"
675           url: "{url}"
676           refspec: "{refspec}"
677           branches:
678             - "{branch}"
679           skip-tag: true
680           wipe-workspace: true
681           submodule:
682             disable: "{submodule-disable}"
683             recursive: "{submodule-recursive}"
684             timeout: "{submodule-timeout}"
685           choosing-strategy: "{choosing-strategy}"
686
687 ############
688 # WRAPPERS #
689 ############
690
691 - wrapper:
692     name: lf-infra-wrappers-common
693     wrappers:
694       - mask-passwords
695       - timeout:
696           type: absolute
697           timeout: "{build-timeout}"
698           timeout-var: "BUILD_TIMEOUT"
699           fail: true
700       - timestamps
701       - openstack:
702           single-use: true
703
704 - wrapper:
705     name: lf-infra-wrappers
706     wrappers:
707       - lf-infra-wrappers-common:
708           build-timeout: "{build-timeout}"
709       - config-file-provider:
710           files:
711             - file-id: npmrc
712               target: "$HOME/.npmrc"
713             - file-id: pipconf
714               target: "$HOME/.config/pip/pip.conf"
715       - ssh-agent-credentials:
716           users:
717             - "{jenkins-ssh-credential}"
718
719 - wrapper:
720     name: lf-infra-wrappers-windows
721     wrappers:
722       - lf-infra-wrappers-common:
723           build-timeout: "{build-timeout}"