From: Houa Yang <hyang@contractor.linuxfoundation.org>
Date: Mon, 2 Dec 2019 16:30:08 +0000 (-0600)
Subject: Fix lf-release-jobs
X-Git-Tag: v0.49.0~17
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=41f85fa1f59549b274230fac987002f7147b7d03;p=releng%2Fglobal-jjb.git

Fix lf-release-jobs

Add -m to deal with merge commits.

-m
By default, git diff-tree --stdin does not show differences
for merge commits. With this flag, it shows differences
to that commit from all of its parents.

Refactor packagecloud release/merge parameters and add distribution_type
as environmental variable.
Remove distribution_type from pypi and packagecloud schema.
Add variable values for packagecloud in release-job.sh.

ISSUE-ID: RELENG-2567
Signed-off-by: Houa Yang <hyang@contractor.linuxfoundation.org>
Change-Id: I4bf718874b754bd24c03e22a9184e0b1566075e9
---

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index 8b5fa3d9..cafab869 100644
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -49,8 +49,7 @@ For example, the parameters for a Maven release are as follows::
 Maven Release Files
 -------------------
 
-An example of a maven release file appears below. Name of release file
-must start with "maven". eg. releases/maven-1.0.0-test.yaml
+An example of a maven release file appears below.
 
 .. code-block:: none
 
@@ -181,16 +180,16 @@ PyPI Release Files
 ------------------
 
 An example of a PyPI release file appears below. Name of the release file must
-start with "pypi". eg. releases/pypi-1.0.0-mypackage.yaml
+start with "pypi". For example releases/pypi-1.0.0-mypackage.yaml
 
 .. code-block:: none
 
     $ cat releases/pypi-1.0.0-mypackage.yaml
     ---
-    distribution_type: pypi
     pypi_project: mypackage
     python_version: '3.4'
     version: 1.0.0
+    log_dir: example-project-pypi-merge-master/17
 
 
 The following parameters must appear in the PyPI release yaml file.
@@ -200,7 +199,6 @@ packages.
 
 :Required Parameters:
 
-    :distribution_type: Must be "pypi".
     :log_dir: The suffix of the logs URL reported on completion by the
         Jenkins merge job that created and pushed the distribution files
         to the staging repository.  For example, use value
@@ -223,15 +221,12 @@ The JSON schema for a PyPI release file appears below.
     $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml"
 
     required:
-      - "distribution_type"
       - "log_dir"
       - "pypi_project"
       - "python_version"
       - "version"
 
     properties:
-      distribution_type:
-        type: "string"
       log_dir:
         type: "string"
       pypi_project:
@@ -246,16 +241,15 @@ PackageCloud Release Files
 --------------------------
 
 An example of a PackageCloud release file appears below. Name of release file
-must start with "packagecloud". eg. releases/packagecloud-1.6.0-tree.yaml
+must start with "packagecloud". For example releases/packagecloud-1.6-tree.yaml
 
 .. code-block:: none
 
-    $ cat releases/packagecloud-1.6.0-tree.yaml
+    $ cat releases/packagecloud-1.6-tree.yaml
     ---
-    distribution_type: packagecloud
     package_name:
-        - name: 'tree-1.6.0-10.el7.x86_64.rpm'
-        - name: 'test.rpm'
+        - name: tree-1.6.0-10.el7.x86_64.rpm
+        - name: test.rpm
 
 The following parameters must appear in the PackageCloud release yaml file.
 These are not part of the Jenkins job definition to allow independent
@@ -264,11 +258,10 @@ packages.
 
 :Required Parameters:
 
-    :distribution_type: Must be "packagecloud".
     :package_name: A list of names that specify the packages to promote.
-        (Found via jenkins log when using gem to initally push package up eg.
+        (Found in jenkins console log when using gem to push package eg.
         "Pushing /path/of/package/name-of-package.rpm... success!"
-        OR using rest api call with generated token from packagecloud.io
+        OR using rest api call to query packagecloud.io repo
         "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
         | yq -r .[].filename"
 
@@ -282,7 +275,6 @@ The JSON schema for a PackageCloud release file appears below.
 
     required:
       - "package_name"
-      - "distribution_type"
 
     properties:
       package_name:
@@ -290,8 +282,6 @@ The JSON schema for a PackageCloud release file appears below.
         properties:
           name:
             type: "string"
-    distribution_type:
-      type: "string"
 
 Jenkins Jobs
 ------------
@@ -390,7 +380,7 @@ This template supports Maven and Container release jobs.
         **default**::
 
             - compare-type: REG_EXP
-              pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)'
+              pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
 
 
 PyPI Release Merge
diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 3df511de..4d4a9ca2 100644
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -131,7 +131,7 @@
                   branch-pattern: "**"
               file-paths:
                 - compare-type: REG_EXP
-                  pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)'
+                  pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
           skip-vote:
             successful: "{gerrit-skip-vote}"
             failed: "{gerrit-skip-vote}"
@@ -240,7 +240,7 @@
                   branch-pattern: "**"
               file-paths:
                 - compare-type: REG_EXP
-                  pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)'
+                  pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
 
 ################
 # PyPI RELEASE #
@@ -489,12 +489,12 @@
             - "{branch}"
           included-regions: "{obj:github_release_included_regions}"
 
-###############################
-# PackageCloud RELEASE VERIFY #
-###############################
+########################
+# PackageCloud RELEASE #
+########################
 
-- lf_packagecloud_verify: &lf_packagecloud_verify
-    name: lf-packagecloud-verify
+- lf_packagecloud_release_common: &lf_packagecloud_release_common
+    name: lf-packagecloud-release-common
 
     ######################
     # Default parameters #
@@ -506,16 +506,8 @@
     git-url: "$GIT_URL/$PROJECT"
     gerrit-skip-vote: false
     use-release-file: true
-
-    gerrit_verify_triggers:
-      - patchset-created-event:
-          exclude-drafts: true
-          exclude-trivial-rebase: false
-          exclude-no-code-change: false
-      - draft-published-event
-      - comment-added-contains-event:
-          # yamllint disable-line rule:line-length
-          comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
+    account-name: "{account-name}"
+    source-repo: "{source-repo}"
 
     #####################
     # Job Configuration #
@@ -528,8 +520,19 @@
           project: "{project}"
           branch: "$GERRIT_BRANCH"
           stream: "$GERRIT_BRANCH"
-      - lf-build-with-parameters-maven-release:
-          use-release-file: "{use-release-file}"
+      - string:
+          name: DISTRIBUTION_TYPE
+          default: "packagecloud"
+          description: "The Jenkins release job distribution type."
+      - bool:
+          name: DRY_RUN
+          default: false
+          description: |
+            If DRY_RUN is enabled artifacts are not promoted.
+      - bool:
+          name: USE_RELEASE_FILE
+          default: "{use-release-file}"
+          description: "Set to False for job built with parameters"
 
     builders:
       - lf-infra-pre-build
@@ -537,14 +540,15 @@
           files:
             - file-id: "packagecloud_api"
               target: "$HOME/packagecloud_api"
+            - file-id: packagecloud-account
+              variable: ACCOUNT_NAME_FILE
       - lf-release
 
 - job-template:
     name: "{project-name}-packagecloud-release-verify"
     id: gerrit-packagecloud-release-verify
     <<: *lf_release_common
-    # yamllint disable-line rule:key-duplicates
-    <<: *lf_packagecloud_verify
+    <<: *lf_packagecloud_release_common
 
     wrappers:
       - lf-infra-wrappers:
@@ -565,7 +569,15 @@
     triggers:
       - gerrit:
           server-name: "{gerrit-server-name}"
-          trigger-on: "{obj:gerrit_verify_triggers}"
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: true
+                exclude-trivial-rebase: false
+                exclude-no-code-change: false
+            - draft-published-event
+            - comment-added-contains-event:
+                # yamllint disable-line rule:line-length
+                comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
           projects:
             - project-compare-type: "ANT"
               project-pattern: "{project}"
@@ -574,69 +586,18 @@
                   branch-pattern: "**"
               file-paths:
                 - compare-type: REG_EXP
-                  pattern: '(releases\/packagecloud.*\.yaml|\.releases\
-                    /packagecloud.*\.yaml)'
+                  pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
           skip-vote:
             successful: "{gerrit-skip-vote}"
             failed: "{gerrit-skip-vote}"
             unstable: "{gerrit-skip-vote}"
             notbuilt: "{gerrit-skip-vote}"
 
-##############################
-# PackageCloud RELEASE MERGE #
-##############################
-
-- lf_packagecloud_merge: &lf_packagecloud_merge
-    name: lf-packagecloud-merge
-
-    ######################
-    # Default parameters #
-    ######################
-
-    build-days-to-keep: 7
-    build-timeout: 15
-    disable-job: false
-    git-url: "$GIT_URL/$PROJECT"
-    use-release-file: true
-
-    gerrit_merge_triggers:
-      - change-merged-event
-      - comment-added-contains-event:
-          comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
-
-    #####################
-    # Job Configuration #
-    #####################
-
-    disabled: "{disable-job}"
-
-    parameters:
-      - lf-infra-parameters:
-          project: "{project}"
-          branch: "$GERRIT_BRANCH"
-          stream: "$GERRIT_BRANCH"
-      - lf-build-with-parameters-maven-release:
-          use-release-file: "{use-release-file}"
-      - bool:
-          name: DRY_RUN
-          default: false
-          description: |
-            If DRY_RUN is enabled artifacts are not published.
-
-    builders:
-      - lf-infra-pre-build
-      - config-file-provider:
-          files:
-            - file-id: "packagecloud_api"
-              target: "$HOME/packagecloud_api"
-      - lf-release
-
 - job-template:
     name: "{project-name}-packagecloud-release-merge"
     id: gerrit-packagecloud-release-merge
     <<: *lf_release_common
-    # yamllint disable-line rule:key-duplicates
-    <<: *lf_packagecloud_merge
+    <<: *lf_packagecloud_release_common
 
     wrappers:
       - lf-infra-wrappers:
@@ -657,7 +618,10 @@
     triggers:
       - gerrit:
           server-name: "{gerrit-server-name}"
-          trigger-on: "{obj:gerrit_merge_triggers}"
+          trigger-on:
+            - change-merged-event
+            - comment-added-contains-event:
+                comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
           projects:
             - project-compare-type: "ANT"
               project-pattern: "{project}"
@@ -666,5 +630,4 @@
                   branch-pattern: "**"
               file-paths:
                 - compare-type: REG_EXP
-                  pattern: '(releases\/packagecloud.*\.yaml|\.releases\
-                    /packagecloud.*\.yaml)'
+                  pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
diff --git a/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml b/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml
new file mode 100644
index 00000000..87ac180b
--- /dev/null
+++ b/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml
@@ -0,0 +1,11 @@
+---
+prelude: >
+  A project may need more than one type of release job
+  this patch changes the trigger to match on release file name
+  for pypi and packagecloud release jobs.
+features:
+  - |
+    Release jobs now support non-ff merges.
+issues:
+  - |
+    Release jobs still trigger when a release file is deleted.
diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml
index 15ba511d..f17d1741 100644
--- a/schema/release-packagecloud-schema.yaml
+++ b/schema/release-packagecloud-schema.yaml
@@ -13,7 +13,6 @@ $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud
 
 required:
   - "package_name"
-  - "distribution_type"
 
 properties:
   package_name:
@@ -21,5 +20,3 @@ properties:
     properties:
       name:
         type: "string"
-  distribution_type:
-    type: "string"
diff --git a/schema/release-pypi-schema.yaml b/schema/release-pypi-schema.yaml
index 7edc4942..734198cb 100644
--- a/schema/release-pypi-schema.yaml
+++ b/schema/release-pypi-schema.yaml
@@ -12,15 +12,12 @@ $schema: "http://json-schema.org/schema#"
 $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml"
 
 required:
-  - "distribution_type"
   - "log_dir"
   - "pypi_project"
   - "python_version"
   - "version"
 
 properties:
-  distribution_type:
-    type: "string"
   log_dir:
     type: "string"
   pypi_project:
diff --git a/shell/release-job.sh b/shell/release-job.sh
index d852a022..f8837e81 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -30,7 +30,7 @@ set_variables_common(){
     NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
     # Verify if using release file or parameters
     if $USE_RELEASE_FILE ; then
-        release_files=$(git diff-tree --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
+        release_files=$(git diff-tree -m --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
         if (( $(grep -c . <<<"$release_files") > 1 )); then
           echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files"
           echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported."
@@ -142,7 +142,7 @@ set_variables_pypi(){
 
 verify_schema(){
     echo "INFO: Verifying $release_file schema."
-    lftools schema verify "$release_file" "$RELEASE_SCHEMA"
+    lftools schema verify "$release_file" "$release_schema"
 }
 
 verify_version(){
@@ -337,29 +337,29 @@ pypi_release_file(){
 }
 
 packagecloud_verify(){
-    echo "INFO: ---> Verifying $1 exists in staging..."
+    echo "INFO: Verifying $1 exists in staging..."
     if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
-        https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+        https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
         | yq -r .[].filename) ]]; then
         echo "INFO: $1 exists in staging!"
         echo "INFO: Existing package location: https://packagecloud.io$(curl \
             --netrc-file ~/packagecloud_api --silent \
-            https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+            https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
             | yq -r .[].package_html_url)"
     else
-        echo "ERROR: $package_name does not exist in staging"
+        echo "ERROR: $1 does not exist in staging"
         exit 1
     fi
 }
 
 packagecloud_promote(){
-    echo "INFO: ---> Preparing to promote $1..."
+    echo "INFO: Preparing to promote $1..."
     promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
-        --silent https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+        --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
         | yq -r .[].promote_url)"
     echo "INFO: Promoting $1 from staging to release"
     curl --netrc-file ~/packagecloud_api -X POST -F \
-        destination=o-ran-sc/release "$promote_url" \
+        destination="$2/release" "$promote_url" \
         | echo "INFO: Promoted package location: \
         https://packagecloud.io$(yq -r .package_html_url)"
 }
@@ -374,55 +374,66 @@ set_variables_common
 #   - maven, release-schema.yaml
 #   - pypi,  release-pypi-schema.yaml
 
-if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
-    if $USE_RELEASE_FILE ; then
-        RELEASE_SCHEMA="release-schema.yaml"
-        echo "INFO: Fetching schema $RELEASE_SCHEMA"
-        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
-        verify_schema
-    fi
-    set_variables_maven
-    verify_version
-    verify_version_match_release
-    maven_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then
-    if $USE_RELEASE_FILE ; then
-        RELEASE_SCHEMA="release-container-schema.yaml"
-        echo "INFO: Fetching schema $RELEASE_SCHEMA"
-        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
-        verify_schema
-    fi
-    set_variables_container
-    verify_version
-    container_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then
-    if $USE_RELEASE_FILE ; then
-        RELEASE_SCHEMA="release-pypi-schema.yaml"
-        echo "INFO: Fetching schema $RELEASE_SCHEMA"
-        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
-        verify_schema
-    fi
-    set_variables_pypi
-    verify_version
-    verify_pypi_match_release
-    pypi_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then
-    RELEASE_SCHEMA="release-packagecloud-schema.yaml"
-    package_name=$(yq -r '.package_name' $release_file)
-    username_repo="o-ran-sc/staging"
-    echo "INFO: Fetching schema $RELEASE_SCHEMA"
-    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
-    verify_schema
-    for name in $(yq -r '.package_name[].name' $release_file); do
-        package_name=$name
-        packagecloud_verify "$package_name" "$username_repo"
-        if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
-            packagecloud_promote "$package_name" "$username_repo"
+case $DISTRIBUTION_TYPE in
+
+    maven)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema
+            verify_schema
         fi
-    done
-else
-    echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
-    exit 1
-fi
+        set_variables_maven
+        verify_version
+        verify_version_match_release
+        maven_release_file
+        ;;
+
+    container)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-container-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_container
+        verify_version
+        container_release_file
+        ;;
+
+    pypi)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-pypi-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_pypi
+        verify_version
+        verify_pypi_match_release
+        pypi_release_file
+        ;;
+
+    packagecloud)
+        release_schema="release-packagecloud-schema.yaml"
+        package_name=$(yq -r '.package_name' $release_file)
+        packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
+        echo "INFO: Fetching schema $release_schema"
+        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+        verify_schema
+        for name in $(yq -r '.package_name[].name' $release_file); do
+            package_name=$name
+            packagecloud_verify "$package_name" "$packagecloud_account"
+            if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
+                packagecloud_promote "$package_name" "$packagecloud_account"
+            fi
+        done
+        ;;
+
+    *)
+        echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
+        exit 1
+        ;;
+esac
 
 echo "---> release-job.sh ends"