From bcf758c9be496bcf1fdc934170cc31a914959508 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Thu, 10 Aug 2017 22:48:56 -0400 Subject: [PATCH] Reorganize template definition structure Having maintained this repo for awhile we can now see some common patterns emerge from our yaml structures. Typically for a given job there is a common builder anchor followed by 2 template sections, 1 for Gerrit and 1 for GitHub. Plus some common macros / anchors that may be used by multiple job types. This patch reorganizes the structure to treat a set of anchor-gerrit-github job definitions as a single job definition. This allows us to keep the job template close to their anchor definition so that looking up what the main anchor does will not cause as much back scroll. Additionally this patch refactors the confusingly named verify_merge anchors to a more understandable 'common' name. Change-Id: I95eb4609274ec21b50aef059b03ac4598894f671 Signed-off-by: Thanh Ha --- jjb/lf-ci-jobs.yaml | 311 +++++++++++++++++++++++++----------------------- jjb/lf-maven-jobs.yaml | 186 ++++++++++++++--------------- jjb/lf-python-jobs.yaml | 10 +- 3 files changed, 252 insertions(+), 255 deletions(-) diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index c130f26c..add785f4 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -49,9 +49,8 @@ - github-packer-merge - github-packer-verify - #################### -# Anchors & Macros # +# COMMON FUNCTIONS # #################### - builder: @@ -83,25 +82,6 @@ - compare-type: ANT pattern: 'packer/provision/lib/**' -- lf_jjb_merge_builders: &lf_jjb_merge_builders - name: lf-jjb-merge-builders - builders: - - lf-infra-jjbini - - shell: !include-raw-escape: - - ../shell/jjb-install.sh - - ../shell/jjb-merge-job.sh - -- lf_jjb_verify_builders: &lf_jjb_verify_builders - name: lf-jjb-verfy-builders - builders: - - lf-infra-jjbini - - shell: !include-raw-escape: - - ../shell/git-validate-jira-urls.sh - - ../shell/jjb-install.sh - - ../shell/jjb-verify-job.sh - - ../shell/jjb-check-unicode.sh - - lf-infra-gpg-verify-git-signature - - lf_packer: &lf_packer_common name: lf-packer-common # Adds wrappers and parameters sections common to packer jobs. @@ -126,39 +106,6 @@ - lf-infra-packer-parameters: packer-version: '{packer-version}' -- lf_packer_merge_builders: &lf_packer_merge_builders - name: lf-packer-merge-builders - # Adds builders section common to merge packer jobs. - # {packer-cloud-settings} is passed from the project, defines the managed - # file-id name created in Jenkins - # (default: packer-cloud-env) - # {packer-version}: is global defined in job group level, the version number - # can be updated to install the latest version of packer. - # {platforms}: defines the packer platform file, passed from the project. - # {template}: defines the packer template file, passed from the project. - builders: - - lf-infra-packer-validate: - packer-cloud-settings: '{packer-cloud-settings}' - packer-version: '{packer-version}' - - lf-infra-packer-build: - packer-cloud-settings: '{packer-cloud-settings}' - packer-version: '{packer-version}' - platform: '{platforms}' - template: '{templates}' - -- lf_packer_verify_builders: &lf_packer_verify_builders - # Adds builders section common to verify packer jobs. - # {packer-cloud-settings} is passed from the project, defines the managed - # file-id name created in Jenkins - # (default: packer-cloud-env) - # {packer-version}: is global defined in job group level, the version number - # can be updated to install the latest version of packer. - name: lf-packer-verify-builders - builders: - - lf-infra-packer-validate: - packer-cloud-settings: '{packer-cloud-settings}' - packer-version: '{packer-version}' - - parameter: name: lf-infra-jjb-parameters parameters: @@ -175,15 +122,8 @@ default: '{packer-version}' description: Packer version to download and install. - -################# -# Job Templates # -################# - -# JJB verify and merge jobs are the same except for their scm, trigger, and -# builders definition. This anchor is the common template -- lf_jjb_verify_merge: &lf_jjb_verify_merge - name: lf-jjb-verify-merge +- lf_jjb_common: &lf_jjb_common + name: lf-jjb-common project-type: freestyle node: '{build-node}' @@ -221,10 +161,22 @@ publishers: - lf-infra-publish +############# +# JJB Merge # +############# + +- lf_jjb_merge_builders: &lf_jjb_merge_builders + name: lf-jjb-merge-builders + builders: + - lf-infra-jjbini + - shell: !include-raw-escape: + - ../shell/jjb-install.sh + - ../shell/jjb-merge-job.sh + - job-template: name: '{project-name}-jjb-merge' id: gerrit-jjb-merge - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_jjb_merge_builders @@ -264,11 +216,64 @@ branch-pattern: '**/{branch}' <<: *lf_jjb_file_paths +- job-template: + name: '{project-name}-jjb-merge' + id: github-jjb-merge + <<: *lf_jjb_common + # yamllint disable-line rule:key-duplicates + <<: *lf_jjb_merge_builders + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - github + - pollscm: + cron: '' + - lf-infra-github-pr-trigger: + trigger-phrase: '^remerge$' + only-trigger-phrase: true + status-context: 'JJB Merge' + permit-all: false + github-hooks: true + github-org: '{github-org}' + github_pr_whitelist: '{obj:github_pr_whitelist}' + github_pr_admin_list: '{obj:github_pr_admin_list}' + +############## +# JJB Verify # +############## + +- lf_jjb_verify_builders: &lf_jjb_verify_builders + name: lf-jjb-verfy-builders + builders: + - lf-infra-jjbini + - shell: !include-raw-escape: + - ../shell/git-validate-jira-urls.sh + - ../shell/jjb-install.sh + - ../shell/jjb-verify-job.sh + - ../shell/jjb-check-unicode.sh + - lf-infra-gpg-verify-git-signature + - job-template: name: '{project-name}-jjb-verify' id: gerrit-jjb-verify concurrent: true - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_jjb_verify_builders @@ -312,49 +317,11 @@ branch-pattern: '**/{branch}' <<: *lf_jjb_file_paths -- job-template: - name: '{project-name}-jjb-merge' - id: github-jjb-merge - <<: *lf_jjb_verify_merge - # yamllint disable-line rule:key-duplicates - <<: *lf_jjb_merge_builders - - ##################### - # Job Configuration # - ##################### - - properties: - - github: - url: '{git-url}/{github-org}/{project}' - - scm: - - lf-infra-github-scm: - url: '{git-clone-url}{github-org}/{project}' - refspec: '' - branch: '{branch}' - submodule-recursive: '{submodule-recursive}' - choosing-strategy: default - jenkins-ssh-credential: '{jenkins-ssh-credential}' - - triggers: - - github - - pollscm: - cron: '' - - lf-infra-github-pr-trigger: - trigger-phrase: '^remerge$' - only-trigger-phrase: true - status-context: 'JJB Merge' - permit-all: false - github-hooks: true - github-org: '{github-org}' - github_pr_whitelist: '{obj:github_pr_whitelist}' - github_pr_admin_list: '{obj:github_pr_admin_list}' - - job-template: name: '{project-name}-jjb-verify' id: github-jjb-verify concurrent: true - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_jjb_verify_builders @@ -388,11 +355,37 @@ github_pr_admin_list: - '' +################ +# Packer Merge # +################ + +- lf_packer_merge_builders: &lf_packer_merge_builders + name: lf-packer-merge-builders + + # Adds builders section common to merge packer jobs. + # {packer-cloud-settings} is passed from the project, defines the managed + # file-id name created in Jenkins + # (default: packer-cloud-env) + # {packer-version}: is global defined in job group level, the version number + # can be updated to install the latest version of packer. + # {platforms}: defines the packer platform file, passed from the project. + # {template}: defines the packer template file, passed from the project. + + builders: + - lf-infra-packer-validate: + packer-cloud-settings: '{packer-cloud-settings}' + packer-version: '{packer-version}' + - lf-infra-packer-build: + packer-cloud-settings: '{packer-cloud-settings}' + packer-version: '{packer-version}' + platform: '{platforms}' + template: '{templates}' + - job-template: name: '{project-name}-packer-merge-{platforms}-{templates}' id: gerrit-packer-merge concurrent: true - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_packer_common # yamllint disable-line rule:key-duplicates @@ -435,11 +428,72 @@ branch-pattern: '**/{branch}' <<: *lf_packer_file_paths +- job-template: + name: '{project-name}-packer-merge-{platforms}-{templates}' + id: github-packer-merge + concurrent: true + <<: *lf_jjb_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packer_merge_builders + + ##################### + # Job Configuration # + ##################### + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - timed: '00 10 1 * *' + - github + - pollscm: + cron: '' + - lf-infra-github-pr-trigger: + trigger-phrase: '^remerge$' + only-trigger-phrase: true + status-context: 'Packer {platforms}-${templates} Merge' + permit-all: false + github-hooks: true + github-org: '{github-org}' + github_pr_whitelist: '{obj:github_pr_whitelist}' + github_pr_admin_list: '{obj:github_pr_admin_list}' + +################# +# Packer Verify # +################# + +- lf_packer_verify_builders: &lf_packer_verify_builders + name: lf-packer-verify-builders + + # Adds builders section common to verify packer jobs. + # {packer-cloud-settings} is passed from the project, defines the managed + # file-id name created in Jenkins + # (default: packer-cloud-env) + # {packer-version}: is global defined in job group level, the version number + # can be updated to install the latest version of packer. + + builders: + - lf-infra-packer-validate: + packer-cloud-settings: '{packer-cloud-settings}' + packer-version: '{packer-version}' + - job-template: name: '{project-name}-packer-verify' id: gerrit-packer-verify concurrent: true - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_packer_common # yamllint disable-line rule:key-duplicates @@ -486,54 +540,11 @@ branch-pattern: '**/{branch}' <<: *lf_packer_file_paths - -- job-template: - name: '{project-name}-packer-merge-{platforms}-{templates}' - id: github-packer-merge - concurrent: true - <<: *lf_jjb_verify_merge - # yamllint disable-line rule:key-duplicates - <<: *lf_packer_common - # yamllint disable-line rule:key-duplicates - <<: *lf_packer_merge_builders - - ##################### - # Job Configuration # - ##################### - - properties: - - github: - url: '{git-url}/{github-org}/{project}' - - scm: - - lf-infra-github-scm: - url: '{git-clone-url}{github-org}/{project}' - refspec: '' - branch: '{branch}' - submodule-recursive: '{submodule-recursive}' - choosing-strategy: default - jenkins-ssh-credential: '{jenkins-ssh-credential}' - - triggers: - - timed: '00 10 1 * *' - - github - - pollscm: - cron: '' - - lf-infra-github-pr-trigger: - trigger-phrase: '^remerge$' - only-trigger-phrase: true - status-context: 'Packer {platforms}-${templates} Merge' - permit-all: false - github-hooks: true - github-org: '{github-org}' - github_pr_whitelist: '{obj:github_pr_whitelist}' - github_pr_admin_list: '{obj:github_pr_admin_list}' - - job-template: name: '{project-name}-packer-verify' id: github-packer-verify concurrent: true - <<: *lf_jjb_verify_merge + <<: *lf_jjb_common # yamllint disable-line rule:key-duplicates <<: *lf_packer_common # yamllint disable-line rule:key-duplicates diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml index 3f6de2e8..2aa0bebb 100644 --- a/jjb/lf-maven-jobs.yaml +++ b/jjb/lf-maven-jobs.yaml @@ -22,9 +22,48 @@ - github-maven-release #################### -# Anchors & Macros # +# COMMON FUNCTIONS # #################### +- lf_maven_common: &lf_maven_common + name: lf-maven-common + + project-type: freestyle + node: '{build-node}' + jdk: '{java-version}' + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + - lf-infra-maven-parameters: + mvn-opts: '{mvn-opts}' + mvn-params: '{mvn-params}' + mvn-version: '{mvn-version}' + # Staging repos do not make sense for CLM jobs so set it blank. + staging-profile-id: '{staging-profile-id}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + publishers: + # TODO: Make email notification work. + # - lf-infra-email-notify: + # email-recipients: '{email-recipients}' + # email-prefix: '[releng]' + - lf-infra-publish + +############# +# Maven CLM # +############# + - lf_maven_clm: &lf_maven_clm name: lf-maven-clm @@ -41,15 +80,12 @@ # mvn-global-settings: The name of the Maven global settings to use for # Maven configuration. (default: global-settings) - project-type: freestyle - node: '{build-node}' - jdk: '{java-version}' - ###################### # Default parameters # ###################### branch: master + build-days-to-keep: 30 # 30 days for troubleshooting purposes build-timeout: 15 git-url: '$GIT_URL/$PROJECT' java-version: openjdk8 @@ -60,32 +96,13 @@ stream: master submodule-recursive: true + # Staging repos do not make sense for CLM jobs so set it blank. + staging-profile-id: '' + ##################### # Job Configuration # ##################### - properties: - - lf-infra-properties: - # Keep around for 30 days for troubleshooting purposes. - build-days-to-keep: 30 - - parameters: - - lf-infra-parameters: - project: '{project}' - branch: '{branch}' - stream: '{stream}' - - lf-infra-maven-parameters: - mvn-opts: '{mvn-opts}' - mvn-params: '{mvn-params}' - mvn-version: '{mvn-version}' - # Staging repos do not make sense for CLM jobs so set it blank. - staging-profile-id: '' - - wrappers: - - lf-infra-wrappers: - build-timeout: '{build-timeout}' - jenkins-ssh-credential: '{jenkins-ssh-credential}' - triggers: # Build weekly on Saturdays - timed: 'H H * * 6' @@ -105,8 +122,45 @@ - sonatype-clm: application-name: '{project-name}' - publishers: - - lf-infra-publish +- job-template: + name: '{project-name}-maven-clm-{stream}' + id: gerrit-maven-clm + <<: *lf_maven_common + # yamllint disable-line rule:key-duplicates + <<: *lf_maven_clm + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + +- job-template: + name: '{project-name}-maven-clm-{stream}' + id: github-maven-clm + <<: *lf_maven_common + # yamllint disable-line rule:key-duplicates + <<: *lf_maven_clm + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + +################# +# Maven Release # +################# - lf_maven_release: &lf_maven_release name: lf-maven-release @@ -128,15 +182,12 @@ # mvn-global-settings: The name of the Maven global settings to use for # Maven configuration. (default: global-settings) - project-type: freestyle - node: '{build-node}' - jdk: '{java-version}' - ###################### # Default parameters # ###################### branch: master + build-days-to-keep: 30 # 30 days in case a release takes long to get approved. build-timeout: 30 git-url: '$GIT_URL/$PROJECT' java-version: openjdk8 @@ -151,27 +202,6 @@ # Job Configuration # ##################### - properties: - - lf-infra-properties: - # Keep around for 30 days in case a release takes long to get approved. - build-days-to-keep: 30 - - parameters: - - lf-infra-parameters: - project: '{project}' - branch: '{branch}' - stream: '{stream}' - - lf-infra-maven-parameters: - mvn-opts: '{mvn-opts}' - mvn-params: '{mvn-params}' - mvn-version: '{mvn-version}' - staging-profile-id: '{staging-profile-id}' - - wrappers: - - lf-infra-wrappers: - build-timeout: '{build-timeout}' - jenkins-ssh-credential: '{jenkins-ssh-credential}' - builders: - lf-jacoco-nojava-workaround - lf-maven-install: @@ -189,53 +219,11 @@ - ../shell/maven-stage.sh - lf-provide-maven-settings-cleanup - publishers: - # TODO: Make email notification work. - # - lf-infra-email-notify: - # email-recipients: '{email-recipients}' - # email-prefix: '[releng]' - - lf-infra-publish - - -################# -# Job Templates # -################# - -- job-template: - name: '{project-name}-maven-clm-{stream}' - id: gerrit-maven-clm - <<: *lf_maven_clm - - scm: - - lf-infra-gerrit-scm: - jenkins-ssh-credential: '{jenkins-ssh-credential}' - git-url: '{git-url}' - refspec: '$GERRIT_REFSPEC' - branch: '$GERRIT_BRANCH' - submodule-recursive: '{submodule-recursive}' - choosing-strategy: default - -- job-template: - name: '{project-name}-maven-clm-{stream}' - id: github-maven-clm - <<: *lf_maven_clm - - properties: - - github: - url: '{git-url}/{github-org}/{project}' - - scm: - - lf-infra-github-scm: - url: '{git-clone-url}{github-org}/{project}' - refspec: '+refs/pull/*:refs/remotes/origin/pr/*' - branch: '{branch}' - submodule-recursive: '{submodule-recursive}' - choosing-strategy: default - jenkins-ssh-credential: '{jenkins-ssh-credential}' - - job-template: name: '{project-name}-maven-release-{stream}' id: gerrit-maven-release + <<: *lf_maven_common + # yamllint disable-line rule:key-duplicates <<: *lf_maven_release scm: @@ -250,6 +238,8 @@ - job-template: name: '{project-name}-maven-release-{stream}' id: github-maven-release + <<: *lf_maven_common + # yamllint disable-line rule:key-duplicates <<: *lf_maven_release properties: diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index 7f7fb50f..42e0967c 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -17,9 +17,9 @@ jobs: - github-tox-verify -#################### -# Anchors & Macros # -#################### +############## +# Tox Verify # +############## - lf_tox_verify: &lf_tox_verify name: lf-tox-verify @@ -80,10 +80,6 @@ publishers: - lf-infra-publish -################# -# Job Templates # -################# - - job-template: # Python projects typically use tox to run testing. name: '{project-name}-tox-verify-{stream}' -- 2.16.6