From 0ad9506f94f815ebdfa84f7b5ecd8b97cc0ba531 Mon Sep 17 00:00:00 2001 From: Houa Yang Date: Fri, 15 Nov 2019 18:12:41 -0600 Subject: [PATCH] Add packagecloud support global jjb Add support for automating promotion of packages in packagecloud. Signed-off-by: Houa Yang Change-Id: I97dfa3822462e0a1076e5546f97c216e18343795 --- docs/jjb/lf-release-jobs.rst | 115 +++++++++++++ jjb/lf-release-jobs.yaml | 178 +++++++++++++++++++++ ...ecloud-support-global-jjb-8324889b369a9d8a.yaml | 4 + schema/release-packagecloud-schema.yaml | 25 +++ shell/release-job.sh | 45 ++++++ 5 files changed, 367 insertions(+) create mode 100644 releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml create mode 100644 schema/release-packagecloud-schema.yaml diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index 77b46d19..73e04c76 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -240,6 +240,56 @@ The JSON schema for a PyPI release file appears below. type: "string" +PackageCloud Release Files +-------------------------- + +An example of a PackageCloud release file appears below. + +.. code-block:: none + + $ cat releases/packagecloud-release.yaml + --- + distribution_type: packagecloud + package_name: + - name: 'tree-1.6.0-10.el7.x86_64.rpm' + - name: 'test.rpm' + +The following parameters must appear in the PackageCloud release yaml file. +These are not part of the Jenkins job definition to allow independent +self-release of a package maintained in a git repository with other +packages. + +:Required Parameters: + + :distribution_type: Must be "packagecloud". + :package_name: A list of names that specify the packages to promote. + (Found via jenkins log when using gem to initally push package up eg. + "Pushing /path/of/package/name-of-package.rpm... success!" + OR using rest api call with generated token from packagecloud.io + "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q= + | yq -r .[].filename" + +The JSON schema for a PackageCloud release file appears below. + +.. code-block:: none + + --- + $schema: "http://json-schema.org/schema#" + $id: "https://github.com/lfit/releng-global-jjb/blob/master/packagecloud-release-schema" + + required: + - "package_name" + - "distribution_type" + + properties: + package_name: + type: "array" + properties: + name: + type: "string" + distribution_type: + type: "string" + Jenkins Jobs ------------ @@ -417,6 +467,71 @@ verify template accepts neither a branch nor a stream parameter. :use-release-file: Whether to use the release file. (default: true) +PackageCloud Release Verify +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This template supports PackageCloud release jobs. + +:Template Name: {project-name}-packagecloud-verify + +:Comment Trigger: recheck|reverify + +:Required Parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + :project: Git repository name + :project-name: Jenkins job name prefix + +:Optional Parameters: + + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-node: The node to run build on. + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :gerrit-skip-vote: Skip voting for this job. (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + + :gerrit_verify_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: REG_EXP + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + + +PackageCloud Release Merge +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This template supports PackageCloud release jobs. + +:template name: {project-name}-packagecloud-merge + +:comment trigger: remerge + +:required parameters: + + :build-node: the node to run build on. + :jenkins-ssh-release-credential: credential to use for ssh. (generally set + in defaults.yaml) + :project: git repository name + :project-name: jenkins job name prefix + +:optional parameters: + + :build-days-to-keep: days to keep build logs in jenkins. (default: 7) + :build-timeout: timeout in minutes before aborting build. (default: 15) + + :gerrit_merge_triggers: override gerrit triggers. + :gerrit_trigger_file_paths: override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: reg_exp + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + + Setup for LFID, Nexus, Jenkins and Gerrit ----------------------------------------- diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml index 07b988ff..ace4f842 100644 --- a/jjb/lf-release-jobs.yaml +++ b/jjb/lf-release-jobs.yaml @@ -488,3 +488,181 @@ white-list-target-branches: - "{branch}" included-regions: "{obj:github_release_included_regions}" + +############################### +# PackageCloud RELEASE VERIFY # +############################### + +- lf_packagecloud_verify: &lf_packagecloud_verify + name: lf-packagecloud-verify + + ###################### + # Default parameters # + ###################### + + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + git-url: "$GIT_URL/$PROJECT" + gerrit-skip-vote: false + use-release-file: true + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + # yamllint disable-line rule:line-length + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "$GERRIT_BRANCH" + stream: "$GERRIT_BRANCH" + - lf-build-with-parameters-maven-release: + use-release-file: "{use-release-file}" + + builders: + - lf-infra-pre-build + - config-file-provider: + files: + - file-id: "packagecloud_api" + target: "$HOME/packagecloud_api" + - lf-release + +- job-template: + name: "{project-name}-packagecloud-verify" + id: gerrit-packagecloud-release-verify + <<: *lf_release_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packagecloud_verify + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + scm: + - lf-infra-gerrit-scm: + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-disable: true + submodule-recursive: false + submodule-timeout: 10 + choosing-strategy: gerrit + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_verify_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**" + file-paths: + - compare-type: REG_EXP + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + skip-vote: + successful: "{gerrit-skip-vote}" + failed: "{gerrit-skip-vote}" + unstable: "{gerrit-skip-vote}" + notbuilt: "{gerrit-skip-vote}" + +############################## +# PackageCloud RELEASE MERGE # +############################## + +- lf_packagecloud_merge: &lf_packagecloud_merge + name: lf-packagecloud-merge + + ###################### + # Default parameters # + ###################### + + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + git-url: "$GIT_URL/$PROJECT" + use-release-file: true + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "$GERRIT_BRANCH" + stream: "$GERRIT_BRANCH" + - lf-build-with-parameters-maven-release: + use-release-file: "{use-release-file}" + - bool: + name: DRY_RUN + default: false + description: | + If DRY_RUN is enabled artifacts are not published. + + builders: + - lf-infra-pre-build + - config-file-provider: + files: + - file-id: "packagecloud_api" + target: "$HOME/packagecloud_api" + - lf-release + +- job-template: + name: "{project-name}-packagecloud-merge" + id: gerrit-packagecloud-merge + <<: *lf_release_common + # yamllint disable-line rule:key-duplicates + <<: *lf_packagecloud_merge + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-release-credential}" + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-disable: true + submodule-recursive: false + submodule-timeout: 10 + choosing-strategy: default + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_merge_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**" + file-paths: + - compare-type: REG_EXP + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' diff --git a/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml b/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml new file mode 100644 index 00000000..a7ab2796 --- /dev/null +++ b/releasenotes/notes/add-packagecloud-support-global-jjb-8324889b369a9d8a.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for automation of promoting packagecloud packages. diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml new file mode 100644 index 00000000..15ba511d --- /dev/null +++ b/schema/release-packagecloud-schema.yaml @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +--- +$schema: "http://json-schema.org/schema#" +$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud-schema" + +required: + - "package_name" + - "distribution_type" + +properties: + package_name: + type: "array" + properties: + name: + type: "string" + distribution_type: + type: "string" diff --git a/shell/release-job.sh b/shell/release-job.sh index 5ee06f55..7c78a870 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -336,6 +336,36 @@ pypi_release_file(){ tag } +packagecloud_verify(){ + echo "INFO: ---> Verifying $1 exists in staging..." + if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \ + https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + | yq -r .[].filename) ]]; then + echo "INFO: $1 exists in staging!" + echo "INFO: Existing package location: https://packagecloud.io$(curl \ + --netrc-file ~/packagecloud_api --silent \ + https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + | yq -r .[].package_html_url)" + else + echo "ERROR: $package_name does not exist in staging" + exit 1 + fi +} + +packagecloud_promote(){ + echo "INFO: ---> Preparing to promote $1..." + promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \ + --silent https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + | yq -r .[].promote_url)" + echo "INFO: Promoting $1 from staging to release" + curl --netrc-file ~/packagecloud_api -X POST -F \ + destination=o-ran-sc/release "$promote_url" \ + | echo "INFO: Promoted package location: \ + https://packagecloud.io$(yq -r .package_html_url)" +} + +############################## End Function Declarations ################################ + # Set common environment variables set_variables_common @@ -376,6 +406,21 @@ elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then verify_version verify_pypi_match_release pypi_release_file +elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then + RELEASE_SCHEMA="release-packagecloud-schema.yaml" + package_name=$(yq -r '.package_name' $release_file) + username_repo="o-ran-sc/staging" + echo "INFO: Fetching schema $RELEASE_SCHEMA" + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/\ + schema/${RELEASE_SCHEMA} + verify_schema + for name in $(yq -r '.package_name[].name' $release_file); do + package_name=$name + packagecloud_verify "$package_name" "$username_repo" + if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then + packagecloud_promote "$package_name" "$username_repo" + fi + done else echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported" exit 1 -- 2.16.6