From: Houa Yang <hyang@contractor.linuxfoundation.org>
Date: Sat, 16 Nov 2019 00:12:41 +0000 (-0600)
Subject: Add packagecloud support global jjb
X-Git-Tag: v0.49.0~28
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=0ad9506f94f815ebdfa84f7b5ecd8b97cc0ba531;p=releng%2Fglobal-jjb.git

Add packagecloud support global jjb

Add support for automating promotion of packages in packagecloud.

Signed-off-by: Houa Yang <hyang@contractor.linuxfoundation.org>
Change-Id: I97dfa3822462e0a1076e5546f97c216e18343795
---

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index 77b46d19..73e04c76 100644
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -240,6 +240,56 @@ The JSON schema for a PyPI release file appears below.
         type: "string"
 
 
+PackageCloud Release Files
+--------------------------
+
+An example of a PackageCloud release file appears below.
+
+.. code-block:: none
+
+    $ cat releases/packagecloud-release.yaml
+    ---
+    distribution_type: packagecloud
+    package_name:
+        - name: 'tree-1.6.0-10.el7.x86_64.rpm'
+        - name: 'test.rpm'
+
+The following parameters must appear in the PackageCloud release yaml file.
+These are not part of the Jenkins job definition to allow independent
+self-release of a package maintained in a git repository with other
+packages.
+
+:Required Parameters:
+
+    :distribution_type: Must be "packagecloud".
+    :package_name: A list of names that specify the packages to promote.
+        (Found via jenkins log when using gem to initally push package up eg.
+        "Pushing /path/of/package/name-of-package.rpm... success!"
+        OR using rest api call with generated token from packagecloud.io
+        "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
+        | yq -r .[].filename"
+
+The JSON schema for a PackageCloud release file appears below.
+
+.. code-block:: none
+
+    ---
+    $schema: "http://json-schema.org/schema#"
+    $id: "https://github.com/lfit/releng-global-jjb/blob/master/packagecloud-release-schema"
+
+    required:
+      - "package_name"
+      - "distribution_type"
+
+    properties:
+      package_name:
+        type: "array"
+        properties:
+          name:
+            type: "string"
+    distribution_type:
+      type: "string"
+
 Jenkins Jobs
 ------------
 
@@ -417,6 +467,71 @@ verify template accepts neither a branch nor a stream parameter.
     :use-release-file: Whether to use the release file. (default: true)
 
 
+PackageCloud Release Verify
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This template supports PackageCloud release jobs.
+
+:Template Name: {project-name}-packagecloud-verify
+
+:Comment Trigger: recheck|reverify
+
+:Required Parameters:
+
+    :build-node: The node to run build on.
+    :jenkins-ssh-credential: Credential to use for SSH. (Generally set
+        in defaults.yaml)
+    :project: Git repository name
+    :project-name: Jenkins job name prefix
+
+:Optional Parameters:
+
+    :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+    :build-node: The node to run build on.
+    :build-timeout: Timeout in minutes before aborting build. (default: 15)
+    :gerrit-skip-vote: Skip voting for this job. (default: false)
+    :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+
+    :gerrit_verify_triggers: Override Gerrit Triggers.
+    :gerrit_trigger_file_paths: Override file paths filter which checks which
+        file modifications will trigger a build.
+        **default**::
+
+            - compare-type: REG_EXP
+              pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+
+
+PackageCloud Release Merge
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This template supports PackageCloud release jobs.
+
+:template name: {project-name}-packagecloud-merge
+
+:comment trigger: remerge
+
+:required parameters:
+
+    :build-node: the node to run build on.
+    :jenkins-ssh-release-credential: credential to use for ssh. (generally set
+        in defaults.yaml)
+    :project: git repository name
+    :project-name: jenkins job name prefix
+
+:optional parameters:
+
+    :build-days-to-keep: days to keep build logs in jenkins. (default: 7)
+    :build-timeout: timeout in minutes before aborting build. (default: 15)
+
+    :gerrit_merge_triggers: override gerrit triggers.
+    :gerrit_trigger_file_paths: override file paths filter which checks which
+        file modifications will trigger a build.
+        **default**::
+
+            - compare-type: reg_exp
+              pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+
+
 Setup for LFID, Nexus, Jenkins and Gerrit
 -----------------------------------------
 
diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 07b988ff..ace4f842 100644
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -488,3 +488,181 @@
           white-list-target-branches:
             - "{branch}"
           included-regions: "{obj:github_release_included_regions}"
+
+###############################
+# PackageCloud RELEASE VERIFY #
+###############################
+
+- lf_packagecloud_verify: &lf_packagecloud_verify
+    name: lf-packagecloud-verify
+
+    ######################
+    # Default parameters #
+    ######################
+
+    build-days-to-keep: 7
+    build-timeout: 15
+    disable-job: false
+    git-url: "$GIT_URL/$PROJECT"
+    gerrit-skip-vote: false
+    use-release-file: true
+
+    gerrit_verify_triggers:
+      - patchset-created-event:
+          exclude-drafts: true
+          exclude-trivial-rebase: false
+          exclude-no-code-change: false
+      - draft-published-event
+      - comment-added-contains-event:
+          # yamllint disable-line rule:line-length
+          comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    disabled: "{disable-job}"
+
+    parameters:
+      - lf-infra-parameters:
+          project: "{project}"
+          branch: "$GERRIT_BRANCH"
+          stream: "$GERRIT_BRANCH"
+      - lf-build-with-parameters-maven-release:
+          use-release-file: "{use-release-file}"
+
+    builders:
+      - lf-infra-pre-build
+      - config-file-provider:
+          files:
+            - file-id: "packagecloud_api"
+              target: "$HOME/packagecloud_api"
+      - lf-release
+
+- job-template:
+    name: "{project-name}-packagecloud-verify"
+    id: gerrit-packagecloud-release-verify
+    <<: *lf_release_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_packagecloud_verify
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: "{build-timeout}"
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    scm:
+      - lf-infra-gerrit-scm:
+          git-url: "{git-url}"
+          refspec: "$GERRIT_REFSPEC"
+          branch: "$GERRIT_BRANCH"
+          submodule-disable: true
+          submodule-recursive: false
+          submodule-timeout: 10
+          choosing-strategy: gerrit
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    triggers:
+      - gerrit:
+          server-name: "{gerrit-server-name}"
+          trigger-on: "{obj:gerrit_verify_triggers}"
+          projects:
+            - project-compare-type: "ANT"
+              project-pattern: "{project}"
+              branches:
+                - branch-compare-type: "ANT"
+                  branch-pattern: "**"
+              file-paths:
+                - compare-type: REG_EXP
+                  pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+          skip-vote:
+            successful: "{gerrit-skip-vote}"
+            failed: "{gerrit-skip-vote}"
+            unstable: "{gerrit-skip-vote}"
+            notbuilt: "{gerrit-skip-vote}"
+
+##############################
+# PackageCloud RELEASE MERGE #
+##############################
+
+- lf_packagecloud_merge: &lf_packagecloud_merge
+    name: lf-packagecloud-merge
+
+    ######################
+    # Default parameters #
+    ######################
+
+    build-days-to-keep: 7
+    build-timeout: 15
+    disable-job: false
+    git-url: "$GIT_URL/$PROJECT"
+    use-release-file: true
+
+    gerrit_merge_triggers:
+      - change-merged-event
+      - comment-added-contains-event:
+          comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    disabled: "{disable-job}"
+
+    parameters:
+      - lf-infra-parameters:
+          project: "{project}"
+          branch: "$GERRIT_BRANCH"
+          stream: "$GERRIT_BRANCH"
+      - lf-build-with-parameters-maven-release:
+          use-release-file: "{use-release-file}"
+      - bool:
+          name: DRY_RUN
+          default: false
+          description: |
+            If DRY_RUN is enabled artifacts are not published.
+
+    builders:
+      - lf-infra-pre-build
+      - config-file-provider:
+          files:
+            - file-id: "packagecloud_api"
+              target: "$HOME/packagecloud_api"
+      - lf-release
+
+- job-template:
+    name: "{project-name}-packagecloud-merge"
+    id: gerrit-packagecloud-merge
+    <<: *lf_release_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_packagecloud_merge
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: "{build-timeout}"
+          jenkins-ssh-credential: "{jenkins-ssh-release-credential}"
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+          git-url: "{git-url}"
+          refspec: "$GERRIT_REFSPEC"
+          branch: "$GERRIT_BRANCH"
+          submodule-disable: true
+          submodule-recursive: false
+          submodule-timeout: 10
+          choosing-strategy: default
+
+    triggers:
+      - gerrit:
+          server-name: "{gerrit-server-name}"
+          trigger-on: "{obj:gerrit_merge_triggers}"
+          projects:
+            - project-compare-type: "ANT"
+              project-pattern: "{project}"
+              branches:
+                - branch-compare-type: "ANT"
+                  branch-pattern: "**"
+              file-paths:
+                - compare-type: REG_EXP
+                  pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
diff --git a/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml b/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml
new file mode 100644
index 00000000..a7ab2796
--- /dev/null
+++ b/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add support for automation of promoting packagecloud packages.
diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml
new file mode 100644
index 00000000..15ba511d
--- /dev/null
+++ b/schema/release-packagecloud-schema.yaml
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+---
+$schema: "http://json-schema.org/schema#"
+$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud-schema"
+
+required:
+  - "package_name"
+  - "distribution_type"
+
+properties:
+  package_name:
+    type: "array"
+    properties:
+      name:
+        type: "string"
+  distribution_type:
+    type: "string"
diff --git a/shell/release-job.sh b/shell/release-job.sh
index 5ee06f55..7c78a870 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -336,6 +336,36 @@ pypi_release_file(){
     tag
 }
 
+packagecloud_verify(){
+    echo "INFO: ---> Verifying $1 exists in staging..."
+    if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
+        https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+        | yq -r .[].filename) ]]; then
+        echo "INFO: $1 exists in staging!"
+        echo "INFO: Existing package location: https://packagecloud.io$(curl \
+            --netrc-file ~/packagecloud_api --silent \
+            https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+            | yq -r .[].package_html_url)"
+    else
+        echo "ERROR: $package_name does not exist in staging"
+        exit 1
+    fi
+}
+
+packagecloud_promote(){
+    echo "INFO: ---> Preparing to promote $1..."
+    promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
+        --silent https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+        | yq -r .[].promote_url)"
+    echo "INFO: Promoting $1 from staging to release"
+    curl --netrc-file ~/packagecloud_api -X POST -F \
+        destination=o-ran-sc/release "$promote_url" \
+        | echo "INFO: Promoted package location: \
+        https://packagecloud.io$(yq -r .package_html_url)"
+}
+
+##############################  End Function Declarations  ################################
+
 # Set common environment variables
 set_variables_common
 
@@ -376,6 +406,21 @@ elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then
     verify_version
     verify_pypi_match_release
     pypi_release_file
+elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then
+    RELEASE_SCHEMA="release-packagecloud-schema.yaml"
+    package_name=$(yq -r '.package_name' $release_file)
+    username_repo="o-ran-sc/staging"
+    echo "INFO: Fetching schema $RELEASE_SCHEMA"
+    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/\
+        schema/${RELEASE_SCHEMA}
+    verify_schema
+    for name in $(yq -r '.package_name[].name' $release_file); do
+        package_name=$name
+        packagecloud_verify "$package_name" "$username_repo"
+        if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
+            packagecloud_promote "$package_name" "$username_repo"
+        fi
+    done
 else
     echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
     exit 1