jjb/lf-macros.yaml

   1 ---
   2 ############
   3 # BUILDERS #
   4 ############
   5
   6 # To take advantage of this macro, have your build write
   7 # out the file 'gerrit_comment.txt' with information to post
   8 # back to gerrit and include this macro in the list of builders.
   9 - builder:
  10     name: comment-to-gerrit
  11     builders:
  12       - shell: !include-raw: ../shell/comment-to-gerrit.sh
  13
  14 - builder:
  15     name: lf-fetch-dependent-patches
  16     builders:
  17       - shell: !include-raw: ../shell/gerrit-fetch-dependencies.sh
  18       - inject:
  19           properties-file: $WORKSPACE/.dependency.properties
  20
  21 - builder:
  22     name: lf-license-check
  23     builders:
  24       - inject:
  25           properties-content: |
  26             FILE_PATTERNS={file-patterns}
  27             SPDX_DISABLE={spdx-disable}
  28             LHC_VERSION={lhc-version}
  29             LICENSE_EXCLUDE_PATHS={license-exclude-paths}
  30             LICENSES_ALLOWED={licenses-allowed}
  31       - shell: !include-raw-escape:
  32           - ../shell/license-check.sh
  33
  34 - builder:
  35     name: lf-infra-create-netrc
  36     builders:
  37       - inject:
  38           properties-content: "SERVER_ID={server-id}"
  39       - shell: !include-raw-escape: ../shell/create-netrc.sh
  40
  41 - builder:
  42     name: lf-infra-docker-login
  43     builders:
  44       - lf-provide-maven-settings:
  45           global-settings-file: "{global-settings-file}"
  46           settings-file: "{settings-file}"
  47       - shell: !include-raw-escape: ../shell/docker-login.sh
  48       - lf-provide-maven-settings-cleanup
  49
  50 - builder:
  51     name: lf-infra-gpg-verify-git-signature
  52     # TODO: Verify signature after downloading users public key from a locally
  53     # created repository instead of the public keymesh. This requires a process
  54     # in place to get ODL developers public keys into a local repository without
  55     # increasing the job thoughput.
  56     builders:
  57       - shell: !include-raw: ../shell/gpg-verify-git-signature.sh
  58
  59 - builder:
  60     name: lf-infra-pre-build
  61     builders:
  62       - shell: !include-raw-escape: ../shell/python-tools-install.sh
  63
  64 - builder:
  65     name: lf-infra-ship-logs
  66     builders:
  67       # Ensure no pre-existing .netrc files are overriding logs config
  68       - lf-provide-maven-settings-cleanup
  69       - config-file-provider:
  70           files:
  71             - file-id: "jenkins-log-archives-settings"
  72               variable: "SETTINGS_FILE"
  73       - lf-infra-create-netrc:
  74           server-id: logs
  75       - shell: !include-raw:
  76           # Ensure python-tools are installed in case job template does not
  77           # call the lf-infra-pre-build macro.
  78           - ../shell/python-tools-install.sh
  79       - shell: !include-raw:
  80           - ../shell/sudo-logs.sh
  81       - shell: !include-raw:
  82           - ../shell/job-cost.sh
  83       - shell: !include-raw:
  84           - ../shell/logs-deploy.sh
  85       - shell: !include-raw:
  86           - ../shell/logs-clear-credentials.sh
  87       - description-setter:
  88           regexp: "^Build logs: .*"
  89
  90 - builder:
  91     name: lf-infra-package-listing
  92     builders:
  93       - shell: !include-raw:
  94           - ../shell/package-listing.sh
  95
  96 - builder:
  97     name: lf-infra-packer-build
  98     builders:
  99       - conditional-step:
 100           condition-kind: boolean-expression
 101           condition-expression: "{openstack}"
 102           steps:
 103             - config-file-provider:
 104                 files:
 105                   - file-id: clouds-yaml
 106                     target: "$HOME/.config/openstack/clouds.yaml"
 107             - inject:
 108                 properties-content: OS_CLOUD={openstack-cloud}
 109       - config-file-provider:
 110           files:
 111             - file-id: "{packer-cloud-settings}"
 112               variable: CLOUDENV
 113       - inject:
 114           properties-content: |
 115             PACKER_PLATFORM={platform}
 116             PACKER_TEMPLATE={template}
 117             PACKER_VERSION={packer-version}
 118             UPDATE_CLOUD_IMAGE={update-cloud-image}
 119       - shell: !include-raw-escape:
 120           - ../shell/packer-install.sh
 121           - ../shell/packer-build.sh
 122       - shell: !include-raw:
 123           - ../shell/packer-clear-credentials.sh
 124
 125 - builder:
 126     name: lf-infra-packer-validate
 127     builders:
 128       - conditional-step:
 129           condition-kind: boolean-expression
 130           condition-expression: "{openstack}"
 131           steps:
 132             - config-file-provider:
 133                 files:
 134                   - file-id: clouds-yaml
 135                     target: "$HOME/.config/openstack/clouds.yaml"
 136             - inject:
 137                 properties-content: OS_CLOUD={openstack-cloud}
 138       - config-file-provider:
 139           files:
 140             - file-id: "{packer-cloud-settings}"
 141               variable: CLOUDENV
 142       - inject:
 143           properties-content: |
 144             PACKER_VERSION={packer-version}
 145       - shell: !include-raw-escape:
 146           - ../shell/packer-install.sh
 147           - ../shell/packer-validate.sh
 148       - shell: !include-raw:
 149           - ../shell/packer-clear-credentials.sh
 150
 151 - builder:
 152     name: lf-infra-update-packer-images
 153     builders:
 154       - shell: !include-raw: ../shell/update-cloud-images.sh
 155
 156 - builder:
 157     name: lf-infra-push-gerrit-patch
 158     builders:
 159       - inject:
 160           properties-content: |
 161             PROJECT={project}
 162             GERRIT_COMMIT_MESSAGE={gerrit-commit-message}
 163             GERRIT_HOST={gerrit-host}
 164             GERRIT_TOPIC={gerrit-topic}
 165             GERRIT_USER={gerrit-user}
 166             REVIEWERS_EMAIL={reviewers-email}
 167       - shell: !include-raw-escape: ../shell/gerrit-push-patch.sh
 168
 169 - builder:
 170     name: lf-infra-sysstat
 171     builders:
 172       - shell: !include-raw:
 173           - ../shell/sysstat.sh
 174
 175 - builder:
 176     name: lf-jacoco-nojava-workaround
 177     builders:
 178       - shell: "mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes"
 179
 180 - builder:
 181     name: lf-infra-deploy-maven-file
 182     builders:
 183       - lf-maven-install:
 184           mvn-version: "{mvn-version}"
 185       - lf-provide-maven-settings:
 186           global-settings-file: "{global-settings-file}"
 187           settings-file: "{settings-file}"
 188       - inject:
 189           properties-content: |
 190             MAVEN_REPO_URL={maven-repo-url}
 191             REPO_ID={repo-id}
 192             GROUP_ID={group-id}
 193             UPLOAD_FILES_PATH={upload-files-dir}
 194       - shell: !include-raw-escape:
 195           - ../shell/common-variables.sh
 196           - ../shell/deploy-maven-file.sh
 197       - lf-provide-maven-settings-cleanup
 198
 199 - builder:
 200     name: lf-maven-central
 201     builders:
 202       - conditional-step:
 203           condition-kind: boolean-expression
 204           condition-expression: "{mvn-central}"
 205           steps:
 206             - lf-provide-maven-settings:
 207                 global-settings-file: "{mvn-global-settings}"
 208                 settings-file: "{mvn-settings}"
 209             - lf-infra-create-netrc:
 210                 server-id: "ossrh"
 211             - inject:
 212                 properties-content: "OSSRH_PROFILE_ID={ossrh-profile-id}"
 213             - shell: !include-raw-escape: ../shell/maven-central.sh
 214             - lf-provide-maven-settings-cleanup
 215
 216 - builder:
 217     name: lf-maven-install
 218     builders:
 219       # Create a $HOME/.wgetrc to make the Maven download quiet.
 220       - shell: 'echo "quiet=on" > "$HOME/.wgetrc"'
 221       - maven-target:
 222           maven-version: "{mvn-version}"
 223           goals: "--version"
 224       - shell: 'rm "$HOME/.wgetrc"'
 225
 226 - builder:
 227     name: lf-provide-maven-settings
 228     builders:
 229       - config-file-provider:
 230           files:
 231             - file-id: "{global-settings-file}"
 232               variable: "GLOBAL_SETTINGS_FILE"
 233             - file-id: "{settings-file}"
 234               variable: "SETTINGS_FILE"
 235
 236 - builder:
 237     name: lf-pip-install
 238     builders:
 239       - inject:
 240           properties-content: PIP_PACKAGES={pip-packages}
 241       - shell: !include-raw-escape: ../shell/pip-install.sh
 242
 243 - builder:
 244     name: lf-provide-maven-settings-cleanup
 245     builders:
 246       - shell: |
 247           #!/bin/bash
 248           set +e  # DO NOT cause build failure if any of the rm calls fail.
 249
 250           rm -f "$GLOBAL_SETTINGS_FILE" "$SETTINGS_FILE"
 251
 252           # In some cases we use the lf-provide-maven-settings macro to produce
 253           # a "$HOME/.netrc" file containing credentials. Remove that file here
 254           # too if it exists.
 255           rm -f "$HOME/.netrc"
 256
 257           # DO NOT fail build if any of the above lines fail.
 258           exit 0
 259
 260 - builder:
 261     name: lf-rtd-trigger-build
 262     builders:
 263       - inject:
 264           properties-content: |
 265             RTD_BUILD_URL={rtd-build-url}
 266             RTD_TOKEN={rtd-token}
 267       - shell: !include-raw-escape: ../shell/rtd-trigger-build.sh
 268
 269 - builder:
 270     name: lf-rtd-verify
 271     builders:
 272       - inject:
 273           properties-content: |
 274             DOC_DIR={doc-dir}
 275             PYTHON={python-version}
 276       - shell: !include-raw-escape: ../shell/tox-install.sh
 277       - shell: !include-raw-escape: ../shell/rtd-verify.sh
 278
 279 - builder:
 280     name: lf-rtdv3-build
 281     builders:
 282       - inject:
 283           properties-content: |
 284             DEFAULT_VERSION={default-version}
 285       - shell: !include-raw: ../shell/rtdv3.sh
 286
 287 - builder:
 288     name: check-info-votes
 289     builders:
 290       - inject:
 291           properties-content: JENKINS_SSH_CREDENTIAL={jenkins-ssh-credential}
 292       - shell: !include-raw-escape: ../shell/check-info-votes.sh
 293
 294 - builder:
 295     name: lf-release
 296     builders:
 297       - config-file-provider:
 298           files:
 299             - file-id: lftoolsini
 300               target: "$HOME/.config/lftools/lftools.ini"
 301       - shell: !include-raw: ../shell/release-job.sh
 302
 303 - builder:
 304     name: lf-sigul-sign-dir
 305     # Requires that Jenkins be configured with SIGUL_BRIDGE_IP as a global
 306     # Environment variable
 307     builders:
 308       - conditional-step:
 309           condition-kind: boolean-expression
 310           condition-expression: "{sign-artifacts}"
 311           steps:
 312             - config-file-provider:
 313                 files:
 314                   - file-id: sigul-config
 315                     variable: SIGUL_CONFIG
 316                   - file-id: sigul-password
 317                     variable: SIGUL_PASSWORD
 318                   - file-id: sigul-pki
 319                     variable: SIGUL_PKI
 320             - shell: !include-raw-escape: ../shell/sigul-configuration.sh
 321             - shell: !include-raw-escape: ../shell/sigul-install.sh
 322             - inject:
 323                 properties-content: |
 324                   SIGN_DIR={sign-dir}
 325                   SIGN_MODE={sign-mode}
 326             - shell: !include-raw-escape:
 327                 - ../shell/sigul-sign-dir.sh
 328             - shell: !include-raw-escape: ../shell/sigul-configuration-cleanup.sh
 329
 330 - builder:
 331     name: lf-infra-provide-docker-cleanup
 332     builders:
 333       - shell: |
 334           #!/bin/bash
 335           set +e  # DO NOT cause build failure if docker rmi fails
 336           docker rmi -f $(docker images -a -q)
 337           exit 0
 338
 339 - builder:
 340     name: lf-infra-sonar
 341     # Run a Sonar Jenkins Plugin
 342     builders:
 343       - sonar:
 344           sonar-name: Sonar
 345           scanner-name: SonarQubeScanner
 346           task: "{sonar-task}"
 347           project: "{sonar-project-file}"
 348           properties: "{sonar-properties}"
 349           java-opts: "{sonar-java-opts}"
 350           additional-arguments: "{sonar-additional-args}"
 351
 352 - builder:
 353     name: lf-infra-sonar-with-prescan
 354     # Run a Sonar Jenkins Plugin
 355     builders:
 356       - lf-sonar-prescan # Must be defined by caller
 357       - sonar:
 358           sonar-name: Sonar
 359           scanner-name: SonarQubeScanner
 360           task: "{sonar-task}"
 361           project: "{sonar-project-file}"
 362           properties: "{sonar-properties}"
 363           java-opts: "{sonar-java-opts}"
 364           additional-arguments: "{sonar-additional-args}"
 365
 366 ##############
 367 # PARAMETERS #
 368 ##############
 369
 370 - parameter:
 371     name: lf-clm-parameters
 372     parameters:
 373       - string:
 374           name: NEXUS_IQ_STAGE
 375           default: "{nexus-iq-stage}"
 376           description: |
 377             Stage the policy evaluation will be run against on the Nexus IQ Server.
 378
 379 - parameter:
 380     name: lf-cmake-parameters
 381     # Parameters useful in CMake builds.
 382     parameters:
 383       - string:
 384           name: BUILD_DIR
 385           default: "{build-dir}"
 386           description: "Directory to build the project in."
 387       - string:
 388           name: CMAKE_OPTS
 389           default: "{cmake-opts}"
 390           description: |
 391             Options to pass to CMAKE.
 392             Example: -DCMAKE_INSTALL_PREFIX=$WORKSPACE/target/output
 393       - string:
 394           name: INSTALL_PREFIX
 395           default: "{install-prefix}"
 396           description: "Install prefix for C/C++ builds. Example: /usr/local"
 397       - string:
 398           name: MAKE_OPTS
 399           default: "{make-opts}"
 400           description: "Options to pass to make. Example: -j8"
 401
 402 - parameter:
 403     name: lf-infra-maven-parameters
 404     parameters:
 405       - string:
 406           name: M2_HOME
 407           # Sets an env var for shell scripts to be able to call the dynamically
 408           # installed maven without having to calculate the M2_HOME themselves.
 409           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}"
 410           description: "Maven selector to be used by shell scripts"
 411       - string:
 412           name: MAVEN_OPTS
 413           default: "{mvn-opts}"
 414           description: |
 415             Maven Java opts. Example: -Xmx1024m -XX:MaxPermSize=256m
 416       - string:
 417           name: MAVEN_PARAMS
 418           default: "{mvn-params}"
 419           description: |
 420             Maven parameters to pass to the mvn command.
 421       - string:
 422           name: MVN
 423           # Sets an env var for shell scripts to be able to call the dynamically
 424           # installed maven without having to calculate the path themselves.
 425           # yamllint disable-line rule:line-length
 426           default: "/w/tools/hudson.tasks.Maven_MavenInstallation/{mvn-version}/bin/mvn"
 427           description: "Maven selector to be used by shell scripts"
 428
 429 - parameter:
 430     name: lf-infra-openstack-parameters
 431     parameters:
 432       - string:
 433           name: OS_CLOUD
 434           default: "{os-cloud}"
 435           description: |
 436             The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
 437             variable name that is significant to openstack client as a
 438             environment variable. Please refer to the documentation for
 439             further details.
 440             https://docs.openstack.org/developer/python-openstackclient/
 441
 442 - parameter:
 443     name: lf-infra-parameters
 444     # Standard parameters used in the LF CI environments. Gerrit variables are
 445     # not used by GitHub projects, but defining them isn't harmful.
 446     parameters:
 447       - string:
 448           name: PROJECT
 449           default: "{project}"
 450           description: |
 451             Parameter to identify a SCM project to build. This is typically
 452             the project repo path. For example: ofextensions/circuitsw
 453       - string:
 454           name: STREAM
 455           default: "{stream}"
 456           description: |
 457             Stream is often set to the same name as 'branch' but can
 458             sometimes be used as a name representing a project's release code
 459             name.
 460       - string:
 461           name: GERRIT_PROJECT
 462           default: "{project}"
 463           description: |
 464             Gerrit Trigger provided parameter to identify Gerrit project that
 465             triggered the build. This is typically the project repo path as
 466             exists in Gerrit. For example: ofextensions/circuitsw
 467
 468             If using Gerrit, in a manual build this should match the PROJECT
 469             parameter above.
 470       - string:
 471           name: GERRIT_BRANCH
 472           default: "{branch}"
 473           description: |
 474             Gerrit Trigger provided parameter to identify a Gerrit branch.
 475
 476             If using Gerrit, in a manual build override with the branch to
 477             build against.
 478       - string:
 479           name: GERRIT_REFSPEC
 480           default: "refs/heads/{branch}"
 481           description: |
 482             Gerrit Trigger provided parameter to identify a refspec to fetch
 483             from Gerrit.
 484
 485             If using Gerrit, in a manual build override with a refspec.
 486             https://git-scm.com/book/en/v2/Git-Internals-The-Refspec
 487             For example: 'refs/heads/master'
 488       - string:
 489           name: sha1
 490           default: "origin/{branch}"
 491           description: |
 492             GitHub PR Trigger provided parameter for specifying the commit
 493             to checkout.
 494
 495             If using GitHub, in a manual build override with a branch path or
 496             sha1 hash to a specific commit. For example: 'origin/master'
 497
 498 - parameter:
 499     name: lf-infra-node-parameters
 500     parameters:
 501       - string:
 502           name: NODE_DIR
 503           default: "{node-dir}"
 504           description: Path to a Node project directory.
 505       - string:
 506           name: NODE_VERSION
 507           default: "{node-version}"
 508           description: Version of NodeJS to install.
 509
 510 - parameter:
 511     name: lf-infra-tox-parameters
 512     parameters:
 513       - string:
 514           name: TOX_DIR
 515           default: "{tox-dir}"
 516           description: |
 517             Path to directory containing tox.ini file.
 518       - string:
 519           name: TOX_ENVS
 520           default: "{tox-envs}"
 521           description: |
 522             Tox environments to run build against.
 523             Example: docs,py2,py3
 524
 525 - parameter:
 526     name: lf-build-with-parameters-maven-release
 527     parameters:
 528       - string:
 529           name: VERSION
 530           default: ""
 531           description: "Version to release, for example: 1.0.0"
 532       - string:
 533           name: GIT_TAG
 534           default: ""
 535           description: "Tag to push to git repo; optional, defaults to VERSION"
 536       - string:
 537           name: LOG_DIR
 538           default: ""
 539           description: "Log dir, for example: project-maven-stage-master/17/"
 540       - choice:
 541           name: DISTRIBUTION_TYPE
 542           choices:
 543             - None
 544             - container
 545             - maven
 546           description: "Set to maven for build with parameters"
 547       - bool:
 548           name: USE_RELEASE_FILE
 549           default: "{use-release-file}"
 550           description: "Set to False for job built with parameters"
 551
 552 ##############
 553 # PROPERTIES #
 554 ##############
 555
 556 - property:
 557     name: lf-infra-properties
 558     properties:
 559       - build-discarder:
 560           # Allow build data to be stored at a length configured by the
 561           # downstream project.
 562           days-to-keep: "{build-days-to-keep}"
 563           # Do not allow artifacts to be stored in Jenkins.
 564           artifact-num-to-keep: 0
 565
 566 ##############
 567 # PUBLISHERS #
 568 ##############
 569
 570 - publisher:
 571     name: lf-jacoco-report
 572     publishers:
 573       - jacoco:
 574           exec-pattern: "**/**.exec"
 575           class-pattern: "**/classes"
 576           source-pattern: "**/src/main/java"
 577           # yamllint disable-line rule:line-length
 578           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
 579           status-update: true
 580           targets:
 581             - branch:
 582                 healthy: 10
 583                 unhealthy: 20
 584             - method:
 585                 healthy: 50
 586                 unhealthy: 40
 587
 588 - publisher:
 589     name: lf-infra-publish
 590     # lf-infra macro to finish up a build.
 591     #
 592     # Handles the following:
 593     #   - Shipping logs to Nexus logs site repository
 594     #   - Cleanup workspace
 595     publishers:
 596       - postbuildscript:
 597           builders:
 598             - role: BOTH
 599               build-on:
 600                 - ABORTED
 601                 - FAILURE
 602                 - NOT_BUILT
 603                 - SUCCESS
 604                 - UNSTABLE
 605               build-steps:
 606                 - lf-infra-sysstat
 607                 - lf-infra-package-listing
 608                 - lf-infra-ship-logs
 609           mark-unstable-if-failed: true
 610       - workspace-cleanup:
 611           exclude:
 612             # Do not clean up *.jenkins-trigger files for jobs that use a
 613             # properties file as input for triggering another build.
 614             - "**/*.jenkins-trigger"
 615           fail-build: false
 616
 617 - publisher:
 618     name: lf-infra-publish-windows
 619     # lf-infra macro to finish up a build.
 620     #
 621     # Handles the following:
 622     #   - Shipping logs to Nexus logs site repository
 623     #   - Cleanup workspace
 624     publishers:
 625       # TODO: RELENG-1228 Develop log shipping script for Windows systems
 626       # - postbuildscript:
 627       #     builders:
 628       #       - role: BOTH
 629       #         build-on:
 630       #           - ABORTED
 631       #           - FAILURE
 632       #           - NOT_BUILT
 633       #           - SUCCESS
 634       #           - UNSTABLE
 635       #         build-steps:
 636       #           - lf-infra-ship-logs-windows
 637       #     mark-unstable-if-failed: true
 638       - workspace-cleanup:
 639           exclude:
 640             # Do not clean up *.jenkins-trigger files for jobs that use a
 641             # properties file as input for triggering another build.
 642             - "**/*.jenkins-trigger"
 643           fail-build: false
 644
 645 #######
 646 # SCM #
 647 #######
 648
 649 - scm:
 650     name: lf-infra-gerrit-scm
 651     scm:
 652       - git:
 653           credentials-id: "{jenkins-ssh-credential}"
 654           url: "{git-url}"
 655           refspec: "{refspec}"
 656           branches:
 657             - "refs/heads/{branch}"
 658           skip-tag: true
 659           wipe-workspace: true
 660           submodule:
 661             disable: "{submodule-disable}"
 662             recursive: "{submodule-recursive}"
 663             timeout: "{submodule-timeout}"
 664           choosing-strategy: "{choosing-strategy}"
 665
 666 - scm:
 667     name: lf-infra-github-scm
 668     scm:
 669       - git:
 670           credentials-id: "{jenkins-ssh-credential}"
 671           url: "{url}"
 672           refspec: "{refspec}"
 673           branches:
 674             - "{branch}"
 675           skip-tag: true
 676           wipe-workspace: true
 677           submodule:
 678             disable: "{submodule-disable}"
 679             recursive: "{submodule-recursive}"
 680             timeout: "{submodule-timeout}"
 681           choosing-strategy: "{choosing-strategy}"
 682
 683 ############
 684 # WRAPPERS #
 685 ############
 686
 687 - wrapper:
 688     name: lf-infra-wrappers-common
 689     wrappers:
 690       - mask-passwords
 691       - timeout:
 692           type: absolute
 693           timeout: "{build-timeout}"
 694           timeout-var: "BUILD_TIMEOUT"
 695           fail: true
 696       - timestamps
 697       - openstack:
 698           single-use: true
 699
 700 - wrapper:
 701     name: lf-infra-wrappers
 702     wrappers:
 703       - lf-infra-wrappers-common:
 704           build-timeout: "{build-timeout}"
 705       - config-file-provider:
 706           files:
 707             - file-id: npmrc
 708               target: "$HOME/.npmrc"
 709             - file-id: pipconf
 710               target: "$HOME/.config/pip/pip.conf"
 711       - ssh-agent-credentials:
 712           users:
 713             - "{jenkins-ssh-credential}"
 714
 715 - wrapper:
 716     name: lf-infra-wrappers-windows
 717     wrappers:
 718       - lf-infra-wrappers-common:
 719           build-timeout: "{build-timeout}"