Reorganize template definition structure

Having maintained this repo for awhile we can now see some common
patterns emerge from our yaml structures. Typically for a given job
there is a common builder anchor followed by 2 template sections, 1
for Gerrit and 1 for GitHub. Plus some common macros / anchors that
may be used by multiple job types.

This patch reorganizes the structure to treat a set of
anchor-gerrit-github job definitions as a single job definition. This
allows us to keep the job template close to their anchor definition so
that looking up what the main anchor does will not cause as much back
scroll.

Additionally this patch refactors the confusingly named verify_merge
anchors to a more understandable 'common' name.

Change-Id: I95eb4609274ec21b50aef059b03ac4598894f671
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index c130f26..add785f 100644 (file)
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -49,9 +49,8 @@
       - github-packer-merge
       - github-packer-verify
 
-
 ####################
-# Anchors & Macros #
+# COMMON FUNCTIONS #
 ####################
 
 - builder:
@@ -83,25 +82,6 @@
       - compare-type: ANT
         pattern: 'packer/provision/lib/**'
 
-- lf_jjb_merge_builders: &lf_jjb_merge_builders
-    name: lf-jjb-merge-builders
-    builders:
-      - lf-infra-jjbini
-      - shell: !include-raw-escape:
-          - ../shell/jjb-install.sh
-          - ../shell/jjb-merge-job.sh
-
-- lf_jjb_verify_builders: &lf_jjb_verify_builders
-    name: lf-jjb-verfy-builders
-    builders:
-      - lf-infra-jjbini
-      - shell: !include-raw-escape:
-          - ../shell/git-validate-jira-urls.sh
-          - ../shell/jjb-install.sh
-          - ../shell/jjb-verify-job.sh
-          - ../shell/jjb-check-unicode.sh
-      - lf-infra-gpg-verify-git-signature
-
 - lf_packer: &lf_packer_common
     name: lf-packer-common
     # Adds wrappers and parameters sections common to packer jobs.
@@ -126,39 +106,6 @@
       - lf-infra-packer-parameters:
           packer-version: '{packer-version}'
 
-- lf_packer_merge_builders: &lf_packer_merge_builders
-    name: lf-packer-merge-builders
-    # Adds builders section common to merge packer jobs.
-    # {packer-cloud-settings} is passed from the project, defines the managed
-    #                   file-id name created in Jenkins
-    #                   (default: packer-cloud-env)
-    # {packer-version}: is global defined in job group level, the version number
-    #                   can be updated to install the latest version of packer.
-    # {platforms}:      defines the packer platform file, passed from the project.
-    # {template}:       defines the packer template file, passed from the project.
-    builders:
-      - lf-infra-packer-validate:
-          packer-cloud-settings: '{packer-cloud-settings}'
-          packer-version: '{packer-version}'
-      - lf-infra-packer-build:
-          packer-cloud-settings: '{packer-cloud-settings}'
-          packer-version: '{packer-version}'
-          platform: '{platforms}'
-          template: '{templates}'
-
-- lf_packer_verify_builders: &lf_packer_verify_builders
-    # Adds builders section common to verify packer jobs.
-    # {packer-cloud-settings} is passed from the project, defines the managed
-    #                   file-id name created in Jenkins
-    #                   (default: packer-cloud-env)
-    # {packer-version}: is global defined in job group level, the version number
-    #                   can be updated to install the latest version of packer.
-    name: lf-packer-verify-builders
-    builders:
-      - lf-infra-packer-validate:
-          packer-cloud-settings: '{packer-cloud-settings}'
-          packer-version: '{packer-version}'
-
 - parameter:
     name: lf-infra-jjb-parameters
     parameters:
@@ -175,15 +122,8 @@
           default: '{packer-version}'
           description: Packer version to download and install.
 
-
-#################
-# Job Templates #
-#################
-
-# JJB verify and merge jobs are the same except for their scm, trigger, and
-# builders definition. This anchor is the common template
-- lf_jjb_verify_merge: &lf_jjb_verify_merge
-    name: lf-jjb-verify-merge
+- lf_jjb_common: &lf_jjb_common
+    name: lf-jjb-common
     project-type: freestyle
 
     node: '{build-node}'
@@ -221,10 +161,22 @@
     publishers:
       - lf-infra-publish
 
+#############
+# JJB Merge #
+#############
+
+- lf_jjb_merge_builders: &lf_jjb_merge_builders
+    name: lf-jjb-merge-builders
+    builders:
+      - lf-infra-jjbini
+      - shell: !include-raw-escape:
+          - ../shell/jjb-install.sh
+          - ../shell/jjb-merge-job.sh
+
 - job-template:
     name: '{project-name}-jjb-merge'
     id: gerrit-jjb-merge
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_jjb_merge_builders
 
@@ -264,11 +216,64 @@
                   branch-pattern: '**/{branch}'
               <<: *lf_jjb_file_paths
 
+- job-template:
+    name: '{project-name}-jjb-merge'
+    id: github-jjb-merge
+    <<: *lf_jjb_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_jjb_merge_builders
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    properties:
+      - github:
+          url: '{git-url}/{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '{git-clone-url}{github-org}/{project}'
+          refspec: ''
+          branch: '{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      - github
+      - pollscm:
+          cron: ''
+      - lf-infra-github-pr-trigger:
+          trigger-phrase: '^remerge$'
+          only-trigger-phrase: true
+          status-context: 'JJB Merge'
+          permit-all: false
+          github-hooks: true
+          github-org: '{github-org}'
+          github_pr_whitelist: '{obj:github_pr_whitelist}'
+          github_pr_admin_list: '{obj:github_pr_admin_list}'
+
+##############
+# JJB Verify #
+##############
+
+- lf_jjb_verify_builders: &lf_jjb_verify_builders
+    name: lf-jjb-verfy-builders
+    builders:
+      - lf-infra-jjbini
+      - shell: !include-raw-escape:
+          - ../shell/git-validate-jira-urls.sh
+          - ../shell/jjb-install.sh
+          - ../shell/jjb-verify-job.sh
+          - ../shell/jjb-check-unicode.sh
+      - lf-infra-gpg-verify-git-signature
+
 - job-template:
     name: '{project-name}-jjb-verify'
     id: gerrit-jjb-verify
     concurrent: true
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_jjb_verify_builders
 
@@ -312,49 +317,11 @@
                   branch-pattern: '**/{branch}'
               <<: *lf_jjb_file_paths
 
-- job-template:
-    name: '{project-name}-jjb-merge'
-    id: github-jjb-merge
-    <<: *lf_jjb_verify_merge
-    # yamllint disable-line rule:key-duplicates
-    <<: *lf_jjb_merge_builders
-
-    #####################
-    # Job Configuration #
-    #####################
-
-    properties:
-      - github:
-          url: '{git-url}/{github-org}/{project}'
-
-    scm:
-      - lf-infra-github-scm:
-          url: '{git-clone-url}{github-org}/{project}'
-          refspec: ''
-          branch: '{branch}'
-          submodule-recursive: '{submodule-recursive}'
-          choosing-strategy: default
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-
-    triggers:
-      - github
-      - pollscm:
-          cron: ''
-      - lf-infra-github-pr-trigger:
-          trigger-phrase: '^remerge$'
-          only-trigger-phrase: true
-          status-context: 'JJB Merge'
-          permit-all: false
-          github-hooks: true
-          github-org: '{github-org}'
-          github_pr_whitelist: '{obj:github_pr_whitelist}'
-          github_pr_admin_list: '{obj:github_pr_admin_list}'
-
 - job-template:
     name: '{project-name}-jjb-verify'
     id: github-jjb-verify
     concurrent: true
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_jjb_verify_builders
 
@@ -388,11 +355,37 @@
           github_pr_admin_list:
             - ''
 
+################
+# Packer Merge #
+################
+
+- lf_packer_merge_builders: &lf_packer_merge_builders
+    name: lf-packer-merge-builders
+
+    # Adds builders section common to merge packer jobs.
+    # {packer-cloud-settings} is passed from the project, defines the managed
+    #                   file-id name created in Jenkins
+    #                   (default: packer-cloud-env)
+    # {packer-version}: is global defined in job group level, the version number
+    #                   can be updated to install the latest version of packer.
+    # {platforms}:      defines the packer platform file, passed from the project.
+    # {template}:       defines the packer template file, passed from the project.
+
+    builders:
+      - lf-infra-packer-validate:
+          packer-cloud-settings: '{packer-cloud-settings}'
+          packer-version: '{packer-version}'
+      - lf-infra-packer-build:
+          packer-cloud-settings: '{packer-cloud-settings}'
+          packer-version: '{packer-version}'
+          platform: '{platforms}'
+          template: '{templates}'
+
 - job-template:
     name: '{project-name}-packer-merge-{platforms}-{templates}'
     id: gerrit-packer-merge
     concurrent: true
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_packer_common
     # yamllint disable-line rule:key-duplicates
@@ -435,11 +428,72 @@
                   branch-pattern: '**/{branch}'
               <<: *lf_packer_file_paths
 
+- job-template:
+    name: '{project-name}-packer-merge-{platforms}-{templates}'
+    id: github-packer-merge
+    concurrent: true
+    <<: *lf_jjb_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_packer_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_packer_merge_builders
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    properties:
+      - github:
+          url: '{git-url}/{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '{git-clone-url}{github-org}/{project}'
+          refspec: ''
+          branch: '{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      - timed: '00 10 1 * *'
+      - github
+      - pollscm:
+          cron: ''
+      - lf-infra-github-pr-trigger:
+          trigger-phrase: '^remerge$'
+          only-trigger-phrase: true
+          status-context: 'Packer {platforms}-${templates} Merge'
+          permit-all: false
+          github-hooks: true
+          github-org: '{github-org}'
+          github_pr_whitelist: '{obj:github_pr_whitelist}'
+          github_pr_admin_list: '{obj:github_pr_admin_list}'
+
+#################
+# Packer Verify #
+#################
+
+- lf_packer_verify_builders: &lf_packer_verify_builders
+    name: lf-packer-verify-builders
+
+    # Adds builders section common to verify packer jobs.
+    # {packer-cloud-settings} is passed from the project, defines the managed
+    #                   file-id name created in Jenkins
+    #                   (default: packer-cloud-env)
+    # {packer-version}: is global defined in job group level, the version number
+    #                   can be updated to install the latest version of packer.
+
+    builders:
+      - lf-infra-packer-validate:
+          packer-cloud-settings: '{packer-cloud-settings}'
+          packer-version: '{packer-version}'
+
 - job-template:
     name: '{project-name}-packer-verify'
     id: gerrit-packer-verify
     concurrent: true
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_packer_common
     # yamllint disable-line rule:key-duplicates
@@ -486,54 +540,11 @@
                   branch-pattern: '**/{branch}'
               <<: *lf_packer_file_paths
 
-
-- job-template:
-    name: '{project-name}-packer-merge-{platforms}-{templates}'
-    id: github-packer-merge
-    concurrent: true
-    <<: *lf_jjb_verify_merge
-    # yamllint disable-line rule:key-duplicates
-    <<: *lf_packer_common
-    # yamllint disable-line rule:key-duplicates
-    <<: *lf_packer_merge_builders
-
-    #####################
-    # Job Configuration #
-    #####################
-
-    properties:
-      - github:
-          url: '{git-url}/{github-org}/{project}'
-
-    scm:
-      - lf-infra-github-scm:
-          url: '{git-clone-url}{github-org}/{project}'
-          refspec: ''
-          branch: '{branch}'
-          submodule-recursive: '{submodule-recursive}'
-          choosing-strategy: default
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-
-    triggers:
-      - timed: '00 10 1 * *'
-      - github
-      - pollscm:
-          cron: ''
-      - lf-infra-github-pr-trigger:
-          trigger-phrase: '^remerge$'
-          only-trigger-phrase: true
-          status-context: 'Packer {platforms}-${templates} Merge'
-          permit-all: false
-          github-hooks: true
-          github-org: '{github-org}'
-          github_pr_whitelist: '{obj:github_pr_whitelist}'
-          github_pr_admin_list: '{obj:github_pr_admin_list}'
-
 - job-template:
     name: '{project-name}-packer-verify'
     id: github-packer-verify
     concurrent: true
-    <<: *lf_jjb_verify_merge
+    <<: *lf_jjb_common
     # yamllint disable-line rule:key-duplicates
     <<: *lf_packer_common
     # yamllint disable-line rule:key-duplicates

diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml
index 3f6de2e..2aa0beb 100644 (file)
--- a/jjb/lf-maven-jobs.yaml
+++ b/jjb/lf-maven-jobs.yaml
@@ -22,9 +22,48 @@
       - github-maven-release
 
 ####################
-# Anchors & Macros #
+# COMMON FUNCTIONS #
 ####################
 
+- lf_maven_common: &lf_maven_common
+    name: lf-maven-common
+
+    project-type: freestyle
+    node: '{build-node}'
+    jdk: '{java-version}'
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          stream: '{stream}'
+      - lf-infra-maven-parameters:
+          mvn-opts: '{mvn-opts}'
+          mvn-params: '{mvn-params}'
+          mvn-version: '{mvn-version}'
+          # Staging repos do not make sense for CLM jobs so set it blank.
+          staging-profile-id: '{staging-profile-id}'
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    publishers:
+      # TODO: Make email notification work.
+      # - lf-infra-email-notify:
+      #     email-recipients: '{email-recipients}'
+      #     email-prefix: '[releng]'
+      - lf-infra-publish
+
+#############
+# Maven CLM #
+#############
+
 - lf_maven_clm: &lf_maven_clm
     name: lf-maven-clm
 
@@ -41,15 +80,12 @@
     #     mvn-global-settings: The name of the Maven global settings to use for
     #                          Maven configuration. (default: global-settings)
 
-    project-type: freestyle
-    node: '{build-node}'
-    jdk: '{java-version}'
-
     ######################
     # Default parameters #
     ######################
 
     branch: master
+    build-days-to-keep: 30  # 30 days for troubleshooting purposes
     build-timeout: 15
     git-url: '$GIT_URL/$PROJECT'
     java-version: openjdk8
@@ -60,32 +96,13 @@
     stream: master
     submodule-recursive: true
 
+    # Staging repos do not make sense for CLM jobs so set it blank.
+    staging-profile-id: ''
+
     #####################
     # Job Configuration #
     #####################
 
-    properties:
-      - lf-infra-properties:
-          # Keep around for 30 days for troubleshooting purposes.
-          build-days-to-keep: 30
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          stream: '{stream}'
-      - lf-infra-maven-parameters:
-          mvn-opts: '{mvn-opts}'
-          mvn-params: '{mvn-params}'
-          mvn-version: '{mvn-version}'
-          # Staging repos do not make sense for CLM jobs so set it blank.
-          staging-profile-id: ''
-
-    wrappers:
-      - lf-infra-wrappers:
-          build-timeout: '{build-timeout}'
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-
     triggers:
       # Build weekly on Saturdays
       - timed: 'H H * * 6'
@@ -105,8 +122,45 @@
       - sonatype-clm:
           application-name: '{project-name}'
 
-    publishers:
-      - lf-infra-publish
+- job-template:
+    name: '{project-name}-maven-clm-{stream}'
+    id: gerrit-maven-clm
+    <<: *lf_maven_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_maven_clm
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+
+- job-template:
+    name: '{project-name}-maven-clm-{stream}'
+    id: github-maven-clm
+    <<: *lf_maven_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_maven_clm
+
+    properties:
+      - github:
+          url: '{git-url}/{github-org}/{project}'
+
+    scm:
+      - lf-infra-github-scm:
+          url: '{git-clone-url}{github-org}/{project}'
+          refspec: '+refs/pull/*:refs/remotes/origin/pr/*'
+          branch: '{branch}'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+#################
+# Maven Release #
+#################
 
 - lf_maven_release: &lf_maven_release
     name: lf-maven-release
@@ -128,15 +182,12 @@
     #     mvn-global-settings: The name of the Maven global settings to use for
     #                          Maven configuration. (default: global-settings)
 
-    project-type: freestyle
-    node: '{build-node}'
-    jdk: '{java-version}'
-
     ######################
     # Default parameters #
     ######################
 
     branch: master
+    build-days-to-keep: 30  # 30 days in case a release takes long to get approved.
     build-timeout: 30
     git-url: '$GIT_URL/$PROJECT'
     java-version: openjdk8
@@ -151,27 +202,6 @@
     # Job Configuration #
     #####################
 
-    properties:
-      - lf-infra-properties:
-          # Keep around for 30 days in case a release takes long to get approved.
-          build-days-to-keep: 30
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          stream: '{stream}'
-      - lf-infra-maven-parameters:
-          mvn-opts: '{mvn-opts}'
-          mvn-params: '{mvn-params}'
-          mvn-version: '{mvn-version}'
-          staging-profile-id: '{staging-profile-id}'
-
-    wrappers:
-      - lf-infra-wrappers:
-          build-timeout: '{build-timeout}'
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-
     builders:
       - lf-jacoco-nojava-workaround
       - lf-maven-install:
@@ -189,53 +219,11 @@
           - ../shell/maven-stage.sh
       - lf-provide-maven-settings-cleanup
 
-    publishers:
-      # TODO: Make email notification work.
-      # - lf-infra-email-notify:
-      #     email-recipients: '{email-recipients}'
-      #     email-prefix: '[releng]'
-      - lf-infra-publish
-
-
-#################
-# Job Templates #
-#################
-
-- job-template:
-    name: '{project-name}-maven-clm-{stream}'
-    id: gerrit-maven-clm
-    <<: *lf_maven_clm
-
-    scm:
-      - lf-infra-gerrit-scm:
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-          git-url: '{git-url}'
-          refspec: '$GERRIT_REFSPEC'
-          branch: '$GERRIT_BRANCH'
-          submodule-recursive: '{submodule-recursive}'
-          choosing-strategy: default
-
-- job-template:
-    name: '{project-name}-maven-clm-{stream}'
-    id: github-maven-clm
-    <<: *lf_maven_clm
-
-    properties:
-      - github:
-          url: '{git-url}/{github-org}/{project}'
-
-    scm:
-      - lf-infra-github-scm:
-          url: '{git-clone-url}{github-org}/{project}'
-          refspec: '+refs/pull/*:refs/remotes/origin/pr/*'
-          branch: '{branch}'
-          submodule-recursive: '{submodule-recursive}'
-          choosing-strategy: default
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-
 - job-template:
     name: '{project-name}-maven-release-{stream}'
     id: gerrit-maven-release
+    <<: *lf_maven_common
+    # yamllint disable-line rule:key-duplicates
     <<: *lf_maven_release
 
     scm:
@@ -250,6 +238,8 @@
 - job-template:
     name: '{project-name}-maven-release-{stream}'
     id: github-maven-release
+    <<: *lf_maven_common
+    # yamllint disable-line rule:key-duplicates
     <<: *lf_maven_release
 
     properties:

diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml
index 7f7fb50..42e0967 100644 (file)
--- a/jjb/lf-python-jobs.yaml
+++ b/jjb/lf-python-jobs.yaml
@@ -17,9 +17,9 @@
     jobs:
       - github-tox-verify
 
-####################
-# Anchors & Macros #
-####################
+##############
+# Tox Verify #
+##############
 
 - lf_tox_verify: &lf_tox_verify
     name: lf-tox-verify
@@ -80,10 +80,6 @@
     publishers:
       - lf-infra-publish
 
-#################
-# Job Templates #
-#################
-
 - job-template:
     # Python projects typically use tox to run testing.
     name: '{project-name}-tox-verify-{stream}'