Add CMake templates to build/publish DEB/RPM pkgs
[releng/global-jjb.git] / jjb / lf-macros.yaml
1 ---
2 ############
3 # BUILDERS #
4 ############
5
6 # To take advantage of this macro, have your build write
7 # out the file 'gerrit_comment.txt' with information to post
8 # back to gerrit and include this macro in the list of builders.
9 - builder:
10     name: comment-to-gerrit
11     builders:
12       - shell: !include-raw: ../shell/comment-to-gerrit.sh
13
14 - builder:
15     name: lf-fetch-dependent-patches
16     builders:
17       - shell: !include-raw: ../shell/gerrit-fetch-dependencies.sh
18       - inject:
19           properties-file: $WORKSPACE/.dependency.properties
20
21 - builder:
22     name: lf-license-check
23     builders:
24       - inject:
25           properties-content: |
26             FILE_PATTERNS={file-patterns}
27             SPDX_DISABLE={spdx-disable}
28             LHC_VERSION={lhc-version}
29             LICENSE_EXCLUDE_PATHS={license-exclude-paths}
30             LICENSES_ALLOWED={licenses-allowed}
31       - shell: !include-raw-escape:
32           - ../shell/license-check.sh
33
34 - builder:
35     name: lf-infra-create-netrc
36     builders:
37       - inject:
38           properties-content: "SERVER_ID={server-id}"
39       - shell: !include-raw-escape: ../shell/create-netrc.sh
40
41 - builder:
42     name: lf-infra-docker-login
43     builders:
44       - lf-provide-maven-settings:
45           global-settings-file: "{global-settings-file}"
46           settings-file: "{settings-file}"
47       - shell: !include-raw-escape: ../shell/docker-login.sh
48       - lf-provide-maven-settings-cleanup
49
50 - builder:
51     name: lf-infra-gpg-verify-git-signature
52     # TODO: Verify signature after downloading users public key from a locally
53     # created repository instead of the public keymesh. This requires a process
54     # in place to get ODL developers public keys into a local repository without
55     # increasing the job thoughput.
56     builders:
57       - shell: !include-raw: ../shell/gpg-verify-git-signature.sh
58
59 - builder:
60     name: lf-infra-pre-build
61     builders:
62       - shell: !include-raw-escape: ../shell/python-tools-install.sh
63
64 - builder:
65     name: lf-infra-ship-logs
66     builders:
67       # Ensure no pre-existing .netrc files are overriding logs config
68       - lf-provide-maven-settings-cleanup
69       - config-file-provider:
70           files:
71             - file-id: "jenkins-log-archives-settings"
72               variable: "SETTINGS_FILE"
73       - conditional-step:
74           condition-kind: regex-match
75           regex: "^.*logs-s3.*"
76           label: $S3_BUCKET
77           steps:
78             - config-file-provider:
79                 files:
80                   - file-id: "jenkins-s3-log-ship"
81                     target: $HOME/.aws/credentials
82       - lf-infra-create-netrc:
83           server-id: logs
84       - shell: !include-raw:
85           # Ensure python-tools are installed in case job template does not
86           # call the lf-infra-pre-build macro.
87           - ../shell/python-tools-install.sh
88       - shell: !include-raw:
89           - ../shell/sudo-logs.sh
90       - shell: !include-raw:
91           - ../shell/job-cost.sh
92       - shell: !include-raw:
93           - ../shell/logs-deploy.sh
94       - shell: !include-raw:
95           - ../shell/logs-clear-credentials.sh
96       - description-setter:
97           regexp: "^Build logs: .*"
98
99 - builder:
100     name: lf-infra-package-listing
101     builders:
102       - shell: !include-raw:
103           - ../shell/package-listing.sh
104
105 - builder:
106     name: lf-infra-packer-build
107     builders:
108       - conditional-step:
109           condition-kind: boolean-expression
110           condition-expression: "{openstack}"
111           steps:
112             - config-file-provider:
113                 files:
114                   - file-id: clouds-yaml
115                     target: "$HOME/.config/openstack/clouds.yaml"
116             - inject:
117                 properties-content: OS_CLOUD={openstack-cloud}
118       - config-file-provider:
119           files:
120             - file-id: "{packer-cloud-settings}"
121               variable: CLOUDENV
122       - inject:
123           properties-content: |
124             PACKER_PLATFORM={platform}
125             PACKER_TEMPLATE={template}
126             PACKER_VERSION={packer-version}
127             UPDATE_CLOUD_IMAGE={update-cloud-image}
128       - shell: !include-raw-escape:
129           - ../shell/packer-install.sh
130           - ../shell/packer-build.sh
131       - shell: !include-raw:
132           - ../shell/packer-clear-credentials.sh
133
134 - builder:
135     name: lf-infra-packer-validate
136     builders:
137       - conditional-step:
138           condition-kind: boolean-expression
139           condition-expression: "{openstack}"
140           steps:
141             - config-file-provider:
142                 files:
143                   - file-id: clouds-yaml
144                     target: "$HOME/.config/openstack/clouds.yaml"
145             - inject:
146                 properties-content: OS_CLOUD={openstack-cloud}
147       - config-file-provider:
148           files:
149             - file-id: "{packer-cloud-settings}"
150               variable: CLOUDENV
151       - inject:
152           properties-content: |
153             PACKER_VERSION={packer-version}
154       - shell: !include-raw-escape:
155           - ../shell/packer-install.sh
156           - ../shell/packer-validate.sh
157       - shell: !include-raw:
158           - ../shell/packer-clear-credentials.sh
159
160 - builder:
161     name: lf-infra-update-packer-images
162     builders:
163       - shell: !include-raw: ../shell/update-cloud-images.sh
164
165 - builder:
166     name: lf-infra-push-gerrit-patch
167     builders:
168       - inject:
169           properties-content: |
170             PROJECT={project}
171             GERRIT_COMMIT_MESSAGE={gerrit-commit-message}
172             GERRIT_HOST={gerrit-host}
173             GERRIT_TOPIC={gerrit-topic}
174             GERRIT_USER={gerrit-user}
175             REVIEWERS_EMAIL={reviewers-email}
176       - shell: !include-raw-escape: ../shell/gerrit-push-patch.sh
177
178 - builder:
179     name: lf-infra-sysstat
180     builders:
181       - shell: !include-raw:
182           - ../shell/sysstat.sh
183
184 - builder:
185     name: lf-jacoco-nojava-workaround
186     builders:
187       - shell: "mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes"
188
189 - builder:
190     name: lf-infra-deploy-maven-file
191     builders:
192       - lf-maven-install:
193           mvn-version: "{mvn-version}"
194       - lf-provide-maven-settings:
195           global-settings-file: "{global-settings-file}"
196           settings-file: "{settings-file}"
197       - inject:
198           properties-content: |
199             MAVEN_REPO_URL={maven-repo-url}
200             REPO_ID={repo-id}
201             GROUP_ID={group-id}
202             UPLOAD_FILES_PATH={upload-files-dir}
203       - shell: !include-raw-escape:
204           - ../shell/common-variables.sh
205           - ../shell/deploy-maven-file.sh
206       - lf-provide-maven-settings-cleanup
207
208 - builder:
209     name: lf-maven-central
210     builders:
211       - conditional-step:
212           condition-kind: boolean-expression
213           condition-expression: "{mvn-central}"
214           steps:
215             - lf-provide-maven-settings:
216                 global-settings-file: "{mvn-global-settings}"
217                 settings-file: "{mvn-settings}"
218             - lf-infra-create-netrc:
219                 server-id: "ossrh"
220             - inject:
221                 properties-content: "OSSRH_PROFILE_ID={ossrh-profile-id}"
222             - shell: !include-raw-escape: ../shell/maven-central.sh
223             - lf-provide-maven-settings-cleanup
224
225 - builder:
226     name: lf-packagecloud-file-provider
227     builders:
228       - config-file-provider:
229           files:
230             - file-id: ".packagecloud"
231               target: "$HOME/.packagecloud"
232       - config-file-provider:
233           files:
234             - file-id: "packagecloud_api"
235               target: "$HOME/packagecloud_api"
236
237 - builder:
238     name: lf-packagecloud-push
239     builders:
240       - inject:
241           properties-content: |
242             BUILD_DIR={build-dir}
243             PACKAGECLOUD_ACCOUNT={packagecloud-account}
244             PACKAGECLOUD_REPO={packagecloud-repo}
245             DEBIAN_DISTRIBUTION_VERSIONS={debian-distribution-versions}
246             RPM_DISTRIBUTION_VERSIONS={rpm-distribution-versions}
247       - shell: !include-raw-escape: ../shell/packagecloud-push.sh
248
249 - builder:
250     name: lf-maven-install
251     builders:
252       # Create a $HOME/.wgetrc to make the Maven download quiet.
253       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
254       - maven-target:
255           maven-version: "{mvn-version}"
256           goals: "--version"
257       - shell: 'rm "$HOME/.wgetrc"'
258
259 - builder:
260     name: lf-provide-maven-settings
261     builders:
262       - config-file-provider:
263           files:
264             - file-id: "{global-settings-file}"
265               variable: "GLOBAL_SETTINGS_FILE"
266             - file-id: "{settings-file}"
267               variable: "SETTINGS_FILE"
268
269 - builder:
270     name: lf-provide-maven-settings-cleanup
271     builders:
272       - shell: |
273           #!/bin/bash
274           set +e  # DO NOT cause build failure if any of the rm calls fail.
275
276           rm -f "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE"
277
278           # In some cases we use the lf-provide-maven-settings macro to produce
279           # a "$HOME/.netrc" file containing credentials. Remove that file here
280           # too if it exists.
281           rm -f "$HOME/.netrc"
282
283           # DO NOT fail build if any of the above lines fail.
284           exit 0
285
286 - builder:
287     name: lf-rtd-trigger-build
288     builders:
289       - inject:
290           properties-content: |
291             RTD_BUILD_URL={rtd-build-url}
292             RTD_TOKEN={rtd-token}
293       - shell: !include-raw-escape: ../shell/rtd-trigger-build.sh
294
295 - builder:
296     name: lf-rtd-verify
297     builders:
298       - inject:
299           properties-content: |
300             DOC_DIR={doc-dir}
301             PYTHON={python-version}
302       - shell: !include-raw-escape: ../shell/tox-install.sh
303       - shell: !include-raw-escape: ../shell/rtd-verify.sh
304
305 - builder:
306     name: lf-rtdv3-build
307     builders:
308       - inject:
309           properties-content: |
310             DEFAULT_VERSION={default-version}
311       - shell: !include-raw-escape: ../shell/rtdv3.sh
312
313 - builder:
314     name: check-info-votes
315     builders:
316       - inject:
317           properties-content: JENKINS_SSH_CREDENTIAL={jenkins-ssh-credential}
318       - shell: !include-raw-escape: ../shell/check-info-votes.sh
319
320 - builder:
321     name: lf-release
322     builders:
323       - config-file-provider:
324           files:
325             - file-id: lftoolsini
326               target: "$HOME/.config/lftools/lftools.ini"
327       - shell: !include-raw: ../shell/release-job.sh
328
329 - builder:
330     name: lf-sigul-sign-dir
331     # Requires that Jenkins be configured with SIGUL_BRIDGE_IP as a global
332     # Environment variable
333     builders:
334       - conditional-step:
335           condition-kind: boolean-expression
336           condition-expression: "{sign-artifacts}"
337           steps:
338             - config-file-provider:
339                 files:
340                   - file-id: sigul-config
341                     variable: SIGUL_CONFIG
342                   - file-id: sigul-password
343                     variable: SIGUL_PASSWORD
344                   - file-id: sigul-pki
345                     variable: SIGUL_PKI
346             - shell: !include-raw-escape: ../shell/sigul-configuration.sh
347             - shell: !include-raw-escape: ../shell/sigul-install.sh
348             - inject:
349                 properties-content: |
350                   SIGN_DIR={sign-dir}
351                   SIGN_MODE={sign-mode}
352             - shell: !include-raw-escape:
353                 - ../shell/sigul-sign-dir.sh
354             - shell: !include-raw-escape: ../shell/sigul-configuration-cleanup.sh
355
356 - builder:
357     name: lf-infra-provide-docker-cleanup
358     builders:
359       - shell: |
360           #!/bin/bash
361           set +e  # DO NOT cause build failure if docker rmi fails
362           docker rmi -f $(docker images -a -q)
363           exit 0
364
365 - builder:
366     name: lf-infra-sonar
367     # Run a Sonar Jenkins Plugin
368     builders:
369       - sonar:
370           sonar-name: Sonar
371           scanner-name: SonarQubeScanner
372           task: "{sonar-task}"
373           project: "{sonar-project-file}"
374           properties: "{sonar-properties}"
375           java-opts: "{sonar-java-opts}"
376           additional-arguments: "{sonar-additional-args}"
377
378 - builder:
379     name: lf-infra-sonar-with-prescan
380     # Run a Sonar Jenkins Plugin
381     builders:
382       - lf-sonar-prescan # Must be defined by caller
383       - sonar:
384           sonar-name: Sonar
385           scanner-name: SonarQubeScanner
386           task: "{sonar-task}"
387           project: "{sonar-project-file}"
388           properties: "{sonar-properties}"
389           java-opts: "{sonar-java-opts}"
390           additional-arguments: "{sonar-additional-args}"
391
392 ##############
393 # PARAMETERS #
394 ##############
395
396 - parameter:
397     name: lf-clm-parameters
398     parameters:
399       - string:
400           name: NEXUS_IQ_STAGE
401           default: "{nexus-iq-stage}"
402           description: |
403             Stage the policy evaluation will be run against on the Nexus IQ Server.
404
405 - parameter:
406     name: lf-cmake-parameters
407     # Parameters useful in CMake builds.
408     parameters:
409       - string:
410           name: BUILD_DIR
411           default: "{build-dir}"
412           description: "Directory to build the project in."
413       - string:
414           name: CMAKE_OPTS
415           default: "{cmake-opts}"
416           description: |
417             Options to pass to CMAKE.
418             Example: -DCMAKE_INSTALL_PREFIX=$WORKSPACE/target/output
419       - string:
420           name: INSTALL_PREFIX
421           default: "{install-prefix}"
422           description: "Install prefix for C/C++ builds. Example: /usr/local"
423       - string:
424           name: MAKE_OPTS
425           default: "{make-opts}"
426           description: "Options to pass to make. Example: -j8"
427
428 - parameter:
429     name: lf-infra-maven-parameters
430     parameters:
431       - string:
432           name: M2_HOME
433           # Sets an env var for shell scripts to be able to call the dynamically
434           # installed maven without having to calculate the M2_HOME themselves.
435           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}"
436           description: "Maven selector to be used by shell scripts"
437       - string:
438           name: MAVEN_OPTS
439           default: "{mvn-opts}"
440           description: |
441             Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
442       - string:
443           name: MAVEN_PARAMS
444           default: "{mvn-params}"
445           description: |
446             Maven parameters to pass to the mvn command.
447       - string:
448           name: MVN
449           # Sets an env var for shell scripts to be able to call the dynamically
450           # installed maven without having to calculate the path themselves.
451           # yamllint disable-line rule:line-length
452           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn"
453           description: "Maven selector to be used by shell scripts"
454
455 - parameter:
456     name: lf-infra-openstack-parameters
457     parameters:
458       - string:
459           name: OS_CLOUD
460           default: "{os-cloud}"
461           description: |
462             The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
463             variable name that is significant to openstack client as a
464             environment variable. Please refer to the documentation for
465             further details.
466             https://docs.openstack.org/developer/python-openstackclient/
467
468 - parameter:
469     name: lf-infra-parameters
470     # Standard parameters used in the LF CI environments. Gerrit variables are
471     # not used by GitHub projects, but defining them isn't harmful.
472     parameters:
473       - string:
474           name: PROJECT
475           default: "{project}"
476           description: |
477             Parameter to identify a SCM project to build. This is typically
478             the project repo path. For example: ofextensions/circuitsw
479       - string:
480           name: STREAM
481           default: "{stream}"
482           description: |
483             Stream is often set to the same name as 'branch' but can
484             sometimes be used as a name representing a project's release code
485             name.
486       - string:
487           name: GERRIT_PROJECT
488           default: "{project}"
489           description: |
490             Gerrit Trigger provided parameter to identify Gerrit project that
491             triggered the build. This is typically the project repo path as
492             exists in Gerrit. For example: ofextensions/circuitsw
493
494             If using Gerrit, in a manual build this should match the PROJECT
495             parameter above.
496       - string:
497           name: GERRIT_BRANCH
498           default: "{branch}"
499           description: |
500             Gerrit Trigger provided parameter to identify a Gerrit branch.
501
502             If using Gerrit, in a manual build override with the branch to
503             build against.
504       - string:
505           name: GERRIT_REFSPEC
506           default: "refs/heads/{branch}"
507           description: |
508             Gerrit Trigger provided parameter to identify a refspec to fetch
509             from Gerrit.
510
511             If using Gerrit, in a manual build override with a refspec.
512             https://git-scm.com/book/en/v2/Git-Internals-The-Refspec
513             For example: 'refs/heads/master'
514       - string:
515           name: sha1
516           default: "origin/{branch}"
517           description: |
518             GitHub PR Trigger provided parameter for specifying the commit
519             to checkout.
520
521             If using GitHub, in a manual build override with a branch path or
522             sha1 hash to a specific commit. For example: 'origin/master'
523
524 - parameter:
525     name: lf-infra-node-parameters
526     parameters:
527       - string:
528           name: NODE_DIR
529           default: "{node-dir}"
530           description: Path to a Node project directory.
531       - string:
532           name: NODE_VERSION
533           default: "{node-version}"
534           description: Version of NodeJS to install.
535
536 - parameter:
537     name: lf-infra-tox-parameters
538     parameters:
539       - string:
540           name: TOX_DIR
541           default: "{tox-dir}"
542           description: |
543             Path to directory containing tox.ini file.
544       - string:
545           name: TOX_ENVS
546           default: "{tox-envs}"
547           description: |
548             Tox environments to run build against.
549             Example: docs,py2,py3
550
551 - parameter:
552     name: lf-build-with-parameters-maven-release
553     parameters:
554       - string:
555           name: VERSION
556           default: ""
557           description: "Version to release, for example: 1.0.0"
558       - string:
559           name: GIT_TAG
560           default: ""
561           description: "Tag to push to git repo; optional, defaults to VERSION"
562       - string:
563           name: LOG_DIR
564           default: ""
565           description: "Log dir, for example: project-maven-stage-master/17/"
566       - choice:
567           name: DISTRIBUTION_TYPE
568           choices:
569             - None
570             - container
571             - maven
572           description: "Set to maven for build with parameters"
573       - bool:
574           name: USE_RELEASE_FILE
575           default: "{use-release-file}"
576           description: "Set to False for job built with parameters"
577
578 ##############
579 # PROPERTIES #
580 ##############
581
582 - property:
583     name: lf-infra-properties
584     properties:
585       - build-discarder:
586           # Allow build data to be stored at a length configured by the
587           # downstream project.
588           days-to-keep: "{build-days-to-keep}"
589           # Do not allow artifacts to be stored in Jenkins.
590           artifact-num-to-keep: 0
591
592 ##############
593 # PUBLISHERS #
594 ##############
595
596 - publisher:
597     name: lf-jacoco-report
598     publishers:
599       - jacoco:
600           exec-pattern: "**/**.exec"
601           class-pattern: "**/classes"
602           source-pattern: "**/src/main/java"
603           # yamllint disable-line rule:line-length
604           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
605           status-update: true
606           targets:
607             - branch:
608                 healthy: 10
609                 unhealthy: 20
610             - method:
611                 healthy: 50
612                 unhealthy: 40
613
614 - publisher:
615     name: lf-infra-publish
616     # lf-infra macro to finish up a build.
617     #
618     # Handles the following:
619     #   - Shipping logs to Nexus logs site repository
620     #   - Cleanup workspace
621     publishers:
622       - postbuildscript:
623           builders:
624             - role: BOTH
625               build-on:
626                 - ABORTED
627                 - FAILURE
628                 - NOT_BUILT
629                 - SUCCESS
630                 - UNSTABLE
631               build-steps:
632                 - lf-infra-sysstat
633                 - lf-infra-package-listing
634                 - lf-infra-ship-logs
635           mark-unstable-if-failed: true
636       - workspace-cleanup:
637           exclude:
638             # Do not clean up *.jenkins-trigger files for jobs that use a
639             # properties file as input for triggering another build.
640             - "**/*.jenkins-trigger"
641           fail-build: false
642
643 - publisher:
644     name: lf-infra-publish-windows
645     # lf-infra macro to finish up a build.
646     #
647     # Handles the following:
648     #   - Shipping logs to Nexus logs site repository
649     #   - Cleanup workspace
650     publishers:
651       # TODO: RELENG-1228 Develop log shipping script for Windows systems
652       # - postbuildscript:
653       #     builders:
654       #       - role: BOTH
655       #         build-on:
656       #           - ABORTED
657       #           - FAILURE
658       #           - NOT_BUILT
659       #           - SUCCESS
660       #           - UNSTABLE
661       #         build-steps:
662       #           - lf-infra-ship-logs-windows
663       #     mark-unstable-if-failed: true
664       - workspace-cleanup:
665           exclude:
666             # Do not clean up *.jenkins-trigger files for jobs that use a
667             # properties file as input for triggering another build.
668             - "**/*.jenkins-trigger"
669           fail-build: false
670
671 #######
672 # SCM #
673 #######
674
675 - scm:
676     name: lf-infra-gerrit-scm
677     scm:
678       - git:
679           credentials-id: "{jenkins-ssh-credential}"
680           url: "{git-url}"
681           refspec: "{refspec}"
682           branches:
683             - "refs/heads/{branch}"
684           skip-tag: true
685           wipe-workspace: true
686           submodule:
687             disable: "{submodule-disable}"
688             recursive: "{submodule-recursive}"
689             timeout: "{submodule-timeout}"
690           choosing-strategy: "{choosing-strategy}"
691
692 - scm:
693     name: lf-infra-github-scm
694     scm:
695       - git:
696           credentials-id: "{jenkins-ssh-credential}"
697           url: "{url}"
698           refspec: "{refspec}"
699           branches:
700             - "{branch}"
701           skip-tag: true
702           wipe-workspace: true
703           submodule:
704             disable: "{submodule-disable}"
705             recursive: "{submodule-recursive}"
706             timeout: "{submodule-timeout}"
707           choosing-strategy: "{choosing-strategy}"
708
709 ############
710 # WRAPPERS #
711 ############
712
713 - wrapper:
714     name: lf-infra-wrappers-common
715     wrappers:
716       - mask-passwords
717       - timeout:
718           type: absolute
719           timeout: "{build-timeout}"
720           timeout-var: "BUILD_TIMEOUT"
721           fail: true
722       - timestamps
723       - openstack:
724           single-use: true
725
726 - wrapper:
727     name: lf-infra-wrappers
728     wrappers:
729       - lf-infra-wrappers-common:
730           build-timeout: "{build-timeout}"
731       - config-file-provider:
732           files:
733             - file-id: npmrc
734               target: "$HOME/.npmrc"
735             - file-id: pipconf
736               target: "$HOME/.config/pip/pip.conf"
737       - ssh-agent-credentials:
738           users:
739             - "{jenkins-ssh-credential}"
740
741 - wrapper:
742     name: lf-infra-wrappers-windows
743     wrappers:
744       - lf-infra-wrappers-common:
745           build-timeout: "{build-timeout}"