From 41f85fa1f59549b274230fac987002f7147b7d03 Mon Sep 17 00:00:00 2001 From: Houa Yang Date: Mon, 2 Dec 2019 10:30:08 -0600 Subject: [PATCH] Fix lf-release-jobs Add -m to deal with merge commits. -m By default, git diff-tree --stdin does not show differences for merge commits. With this flag, it shows differences to that commit from all of its parents. Refactor packagecloud release/merge parameters and add distribution_type as environmental variable. Remove distribution_type from pypi and packagecloud schema. Add variable values for packagecloud in release-job.sh. ISSUE-ID: RELENG-2567 Signed-off-by: Houa Yang Change-Id: I4bf718874b754bd24c03e22a9184e0b1566075e9 --- docs/jjb/lf-release-jobs.rst | 30 ++--- jjb/lf-release-jobs.yaml | 119 +++++++------------ .../notes/release-job-1d9b641b8f2c5259.yaml | 11 ++ schema/release-packagecloud-schema.yaml | 3 - schema/release-pypi-schema.yaml | 3 - shell/release-job.sh | 127 +++++++++++---------- 6 files changed, 131 insertions(+), 162 deletions(-) create mode 100644 releasenotes/notes/release-job-1d9b641b8f2c5259.yaml diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index 8b5fa3d9..cafab869 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -49,8 +49,7 @@ For example, the parameters for a Maven release are as follows:: Maven Release Files ------------------- -An example of a maven release file appears below. Name of release file -must start with "maven". eg. releases/maven-1.0.0-test.yaml +An example of a maven release file appears below. .. code-block:: none @@ -181,16 +180,16 @@ PyPI Release Files ------------------ An example of a PyPI release file appears below. Name of the release file must -start with "pypi". eg. releases/pypi-1.0.0-mypackage.yaml +start with "pypi". For example releases/pypi-1.0.0-mypackage.yaml .. code-block:: none $ cat releases/pypi-1.0.0-mypackage.yaml --- - distribution_type: pypi pypi_project: mypackage python_version: '3.4' version: 1.0.0 + log_dir: example-project-pypi-merge-master/17 The following parameters must appear in the PyPI release yaml file. @@ -200,7 +199,6 @@ packages. :Required Parameters: - :distribution_type: Must be "pypi". :log_dir: The suffix of the logs URL reported on completion by the Jenkins merge job that created and pushed the distribution files to the staging repository. For example, use value @@ -223,15 +221,12 @@ The JSON schema for a PyPI release file appears below. $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml" required: - - "distribution_type" - "log_dir" - "pypi_project" - "python_version" - "version" properties: - distribution_type: - type: "string" log_dir: type: "string" pypi_project: @@ -246,16 +241,15 @@ PackageCloud Release Files -------------------------- An example of a PackageCloud release file appears below. Name of release file -must start with "packagecloud". eg. releases/packagecloud-1.6.0-tree.yaml +must start with "packagecloud". For example releases/packagecloud-1.6-tree.yaml .. code-block:: none - $ cat releases/packagecloud-1.6.0-tree.yaml + $ cat releases/packagecloud-1.6-tree.yaml --- - distribution_type: packagecloud package_name: - - name: 'tree-1.6.0-10.el7.x86_64.rpm' - - name: 'test.rpm' + - name: tree-1.6.0-10.el7.x86_64.rpm + - name: test.rpm The following parameters must appear in the PackageCloud release yaml file. These are not part of the Jenkins job definition to allow independent @@ -264,11 +258,10 @@ packages. :Required Parameters: - :distribution_type: Must be "packagecloud". :package_name: A list of names that specify the packages to promote. - (Found via jenkins log when using gem to initally push package up eg. + (Found in jenkins console log when using gem to push package eg. "Pushing /path/of/package/name-of-package.rpm... success!" - OR using rest api call with generated token from packagecloud.io + OR using rest api call to query packagecloud.io repo "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q= | yq -r .[].filename" @@ -282,7 +275,6 @@ The JSON schema for a PackageCloud release file appears below. required: - "package_name" - - "distribution_type" properties: package_name: @@ -290,8 +282,6 @@ The JSON schema for a PackageCloud release file appears below. properties: name: type: "string" - distribution_type: - type: "string" Jenkins Jobs ------------ @@ -390,7 +380,7 @@ This template supports Maven and Container release jobs. **default**:: - compare-type: REG_EXP - pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)' + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' PyPI Release Merge diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml index 3df511de..4d4a9ca2 100644 --- a/jjb/lf-release-jobs.yaml +++ b/jjb/lf-release-jobs.yaml @@ -131,7 +131,7 @@ branch-pattern: "**" file-paths: - compare-type: REG_EXP - pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)' + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' skip-vote: successful: "{gerrit-skip-vote}" failed: "{gerrit-skip-vote}" @@ -240,7 +240,7 @@ branch-pattern: "**" file-paths: - compare-type: REG_EXP - pattern: '(releases\/maven.*\.yaml|\.releases\/maven.*\.yaml)' + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' ################ # PyPI RELEASE # @@ -489,12 +489,12 @@ - "{branch}" included-regions: "{obj:github_release_included_regions}" -############################### -# PackageCloud RELEASE VERIFY # -############################### +######################## +# PackageCloud RELEASE # +######################## -- lf_packagecloud_verify: &lf_packagecloud_verify - name: lf-packagecloud-verify +- lf_packagecloud_release_common: &lf_packagecloud_release_common + name: lf-packagecloud-release-common ###################### # Default parameters # @@ -506,16 +506,8 @@ git-url: "$GIT_URL/$PROJECT" gerrit-skip-vote: false use-release-file: true - - gerrit_verify_triggers: - - patchset-created-event: - exclude-drafts: true - exclude-trivial-rebase: false - exclude-no-code-change: false - - draft-published-event - - comment-added-contains-event: - # yamllint disable-line rule:line-length - comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + account-name: "{account-name}" + source-repo: "{source-repo}" ##################### # Job Configuration # @@ -528,8 +520,19 @@ project: "{project}" branch: "$GERRIT_BRANCH" stream: "$GERRIT_BRANCH" - - lf-build-with-parameters-maven-release: - use-release-file: "{use-release-file}" + - string: + name: DISTRIBUTION_TYPE + default: "packagecloud" + description: "The Jenkins release job distribution type." + - bool: + name: DRY_RUN + default: false + description: | + If DRY_RUN is enabled artifacts are not promoted. + - bool: + name: USE_RELEASE_FILE + default: "{use-release-file}" + description: "Set to False for job built with parameters" builders: - lf-infra-pre-build @@ -537,14 +540,15 @@ files: - file-id: "packagecloud_api" target: "$HOME/packagecloud_api" + - file-id: packagecloud-account + variable: ACCOUNT_NAME_FILE - lf-release - job-template: name: "{project-name}-packagecloud-release-verify" id: gerrit-packagecloud-release-verify <<: *lf_release_common - # yamllint disable-line rule:key-duplicates - <<: *lf_packagecloud_verify + <<: *lf_packagecloud_release_common wrappers: - lf-infra-wrappers: @@ -565,7 +569,15 @@ triggers: - gerrit: server-name: "{gerrit-server-name}" - trigger-on: "{obj:gerrit_verify_triggers}" + trigger-on: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + # yamllint disable-line rule:line-length + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' projects: - project-compare-type: "ANT" project-pattern: "{project}" @@ -574,69 +586,18 @@ branch-pattern: "**" file-paths: - compare-type: REG_EXP - pattern: '(releases\/packagecloud.*\.yaml|\.releases\ - /packagecloud.*\.yaml)' + pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)' skip-vote: successful: "{gerrit-skip-vote}" failed: "{gerrit-skip-vote}" unstable: "{gerrit-skip-vote}" notbuilt: "{gerrit-skip-vote}" -############################## -# PackageCloud RELEASE MERGE # -############################## - -- lf_packagecloud_merge: &lf_packagecloud_merge - name: lf-packagecloud-merge - - ###################### - # Default parameters # - ###################### - - build-days-to-keep: 7 - build-timeout: 15 - disable-job: false - git-url: "$GIT_URL/$PROJECT" - use-release-file: true - - gerrit_merge_triggers: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' - - ##################### - # Job Configuration # - ##################### - - disabled: "{disable-job}" - - parameters: - - lf-infra-parameters: - project: "{project}" - branch: "$GERRIT_BRANCH" - stream: "$GERRIT_BRANCH" - - lf-build-with-parameters-maven-release: - use-release-file: "{use-release-file}" - - bool: - name: DRY_RUN - default: false - description: | - If DRY_RUN is enabled artifacts are not published. - - builders: - - lf-infra-pre-build - - config-file-provider: - files: - - file-id: "packagecloud_api" - target: "$HOME/packagecloud_api" - - lf-release - - job-template: name: "{project-name}-packagecloud-release-merge" id: gerrit-packagecloud-release-merge <<: *lf_release_common - # yamllint disable-line rule:key-duplicates - <<: *lf_packagecloud_merge + <<: *lf_packagecloud_release_common wrappers: - lf-infra-wrappers: @@ -657,7 +618,10 @@ triggers: - gerrit: server-name: "{gerrit-server-name}" - trigger-on: "{obj:gerrit_merge_triggers}" + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' projects: - project-compare-type: "ANT" project-pattern: "{project}" @@ -666,5 +630,4 @@ branch-pattern: "**" file-paths: - compare-type: REG_EXP - pattern: '(releases\/packagecloud.*\.yaml|\.releases\ - /packagecloud.*\.yaml)' + pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)' diff --git a/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml b/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml new file mode 100644 index 00000000..87ac180b --- /dev/null +++ b/releasenotes/notes/release-job-1d9b641b8f2c5259.yaml @@ -0,0 +1,11 @@ +--- +prelude: > + A project may need more than one type of release job + this patch changes the trigger to match on release file name + for pypi and packagecloud release jobs. +features: + - | + Release jobs now support non-ff merges. +issues: + - | + Release jobs still trigger when a release file is deleted. diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml index 15ba511d..f17d1741 100644 --- a/schema/release-packagecloud-schema.yaml +++ b/schema/release-packagecloud-schema.yaml @@ -13,7 +13,6 @@ $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud required: - "package_name" - - "distribution_type" properties: package_name: @@ -21,5 +20,3 @@ properties: properties: name: type: "string" - distribution_type: - type: "string" diff --git a/schema/release-pypi-schema.yaml b/schema/release-pypi-schema.yaml index 7edc4942..734198cb 100644 --- a/schema/release-pypi-schema.yaml +++ b/schema/release-pypi-schema.yaml @@ -12,15 +12,12 @@ $schema: "http://json-schema.org/schema#" $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml" required: - - "distribution_type" - "log_dir" - "pypi_project" - "python_version" - "version" properties: - distribution_type: - type: "string" log_dir: type: "string" pypi_project: diff --git a/shell/release-job.sh b/shell/release-job.sh index d852a022..f8837e81 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -30,7 +30,7 @@ set_variables_common(){ NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" # Verify if using release file or parameters if $USE_RELEASE_FILE ; then - release_files=$(git diff-tree --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/") + release_files=$(git diff-tree -m --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/") if (( $(grep -c . <<<"$release_files") > 1 )); then echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files" echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported." @@ -142,7 +142,7 @@ set_variables_pypi(){ verify_schema(){ echo "INFO: Verifying $release_file schema." - lftools schema verify "$release_file" "$RELEASE_SCHEMA" + lftools schema verify "$release_file" "$release_schema" } verify_version(){ @@ -337,29 +337,29 @@ pypi_release_file(){ } packagecloud_verify(){ - echo "INFO: ---> Verifying $1 exists in staging..." + echo "INFO: Verifying $1 exists in staging..." if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \ - https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \ | yq -r .[].filename) ]]; then echo "INFO: $1 exists in staging!" echo "INFO: Existing package location: https://packagecloud.io$(curl \ --netrc-file ~/packagecloud_api --silent \ - https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \ | yq -r .[].package_html_url)" else - echo "ERROR: $package_name does not exist in staging" + echo "ERROR: $1 does not exist in staging" exit 1 fi } packagecloud_promote(){ - echo "INFO: ---> Preparing to promote $1..." + echo "INFO: Preparing to promote $1..." promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \ - --silent https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \ + --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \ | yq -r .[].promote_url)" echo "INFO: Promoting $1 from staging to release" curl --netrc-file ~/packagecloud_api -X POST -F \ - destination=o-ran-sc/release "$promote_url" \ + destination="$2/release" "$promote_url" \ | echo "INFO: Promoted package location: \ https://packagecloud.io$(yq -r .package_html_url)" } @@ -374,55 +374,66 @@ set_variables_common # - maven, release-schema.yaml # - pypi, release-pypi-schema.yaml -if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then - if $USE_RELEASE_FILE ; then - RELEASE_SCHEMA="release-schema.yaml" - echo "INFO: Fetching schema $RELEASE_SCHEMA" - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml - verify_schema - fi - set_variables_maven - verify_version - verify_version_match_release - maven_release_file -elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then - if $USE_RELEASE_FILE ; then - RELEASE_SCHEMA="release-container-schema.yaml" - echo "INFO: Fetching schema $RELEASE_SCHEMA" - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA} - verify_schema - fi - set_variables_container - verify_version - container_release_file -elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then - if $USE_RELEASE_FILE ; then - RELEASE_SCHEMA="release-pypi-schema.yaml" - echo "INFO: Fetching schema $RELEASE_SCHEMA" - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA} - verify_schema - fi - set_variables_pypi - verify_version - verify_pypi_match_release - pypi_release_file -elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then - RELEASE_SCHEMA="release-packagecloud-schema.yaml" - package_name=$(yq -r '.package_name' $release_file) - username_repo="o-ran-sc/staging" - echo "INFO: Fetching schema $RELEASE_SCHEMA" - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA} - verify_schema - for name in $(yq -r '.package_name[].name' $release_file); do - package_name=$name - packagecloud_verify "$package_name" "$username_repo" - if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then - packagecloud_promote "$package_name" "$username_repo" +case $DISTRIBUTION_TYPE in + + maven) + if $USE_RELEASE_FILE ; then + release_schema="release-schema.yaml" + echo "INFO: Fetching schema $release_schema" + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema + verify_schema fi - done -else - echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported" - exit 1 -fi + set_variables_maven + verify_version + verify_version_match_release + maven_release_file + ;; + + container) + if $USE_RELEASE_FILE ; then + release_schema="release-container-schema.yaml" + echo "INFO: Fetching schema $release_schema" + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema} + verify_schema + fi + set_variables_container + verify_version + container_release_file + ;; + + pypi) + if $USE_RELEASE_FILE ; then + release_schema="release-pypi-schema.yaml" + echo "INFO: Fetching schema $release_schema" + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema} + verify_schema + fi + set_variables_pypi + verify_version + verify_pypi_match_release + pypi_release_file + ;; + + packagecloud) + release_schema="release-packagecloud-schema.yaml" + package_name=$(yq -r '.package_name' $release_file) + packagecloud_account=$(cat "$ACCOUNT_NAME_FILE") + echo "INFO: Fetching schema $release_schema" + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema} + verify_schema + for name in $(yq -r '.package_name[].name' $release_file); do + package_name=$name + packagecloud_verify "$package_name" "$packagecloud_account" + if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then + packagecloud_promote "$package_name" "$packagecloud_account" + fi + done + ;; + + *) + echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported" + exit 1 + ;; +esac echo "---> release-job.sh ends" -- 2.16.6