From d473edacae5c8da2b4da50e6d41b2a1c52316813 Mon Sep 17 00:00:00 2001 From: Aric Gardner Date: Wed, 16 Jan 2019 16:45:30 -0500 Subject: [PATCH] Automate Artifact release This will be adapted to work for ONAP/Other projects once we are okay with this. projects that want self-serve releases will create a tagname.yaml file in the releases/ directory of their git repo. Here is the change I have been testing in the sandbox https://git.opendaylight.org/gerrit/#/c/80613/ example of a releases file: $ cat releases/1.0.0.yaml --- distribution_type: 'maven' version: '1.0.0' project: 'zzz-test-release' log_dir: 'zzz-test-release-maven-stage-master/17/' maven_central_url: 'oss.sonatype.org' Issue: RELENG-1657 Change-Id: I02dc935a84425d194cdd8eaabe5b43c5920ce319 Signed-off-by: Aric Gardner --- docs/jjb/lf-macros.rst | 13 ++ docs/jjb/lf-release-jobs.rst | 134 ++++++++++++++ jjb/lf-macros.yaml | 9 + jjb/lf-release-jobs.yaml | 198 +++++++++++++++++++++ .../notes/lf-release-jobs-a618ab1220044253.yaml | 18 ++ shell/release-job.sh | 108 +++++++++++ 6 files changed, 480 insertions(+) create mode 100644 docs/jjb/lf-release-jobs.rst create mode 100644 jjb/lf-release-jobs.yaml create mode 100644 releasenotes/notes/lf-release-jobs-a618ab1220044253.yaml create mode 100644 shell/release-job.sh diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 2471b98c..ab70a9fa 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -247,6 +247,19 @@ check-info-votes Calls shell script to validate votes on a change to an INFO.yaml +lf-releases +----------- + +realises lftools.ini (required) +needed to push to nexus. + +[nexus] +username= +password= + +Then runs ../shell/release-job.sh + + lf-sigul-sign-dir ----------------- diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst new file mode 100644 index 00000000..47559fa6 --- /dev/null +++ b/docs/jjb/lf-release-jobs.rst @@ -0,0 +1,134 @@ +.. _lf-global-jjb-release: + +#################### +Releng Release Files +#################### + +Projects can create a releases directory and then place a release file in it. +Jenkins will pick this up and then promote the artifact from the staging log +directory (log_dir) and tag the release with the defined version. +if a maven_central_url is given artifact will be pushed there as well. + +example of a projects release file + +.. code-block:: bash + + $ cat releases/1.0.0.yaml + --- + distribution_type: 'maven' + version: '1.0.0' + project: 'zzz-test-release' + log_dir: 'zzz-test-release-maven-stage-master/17/' + maven_central_url: 'oss.sonatype.org' + +lftools nexus release is used so there must be a lftoolsini section in jenkins +configfiles with a [nexus] section for auth. + +Macros +====== + +lf-releases +---------------------- + +Release verify and merge jobs are the same except for their scm, trigger, and +builders definition. This anchor is the common template. + +Job Templates +============= + +Release Merge +------------- + +Runs: + +- sigul-install +- sigul-configuration +- checkout ref from taglist.log +- applies the $PROJECT.bundle +- signs, tags and pushes + +.. code-block:: bash + + lftools nexus release --server $NEXUS_URL $STAGING_REPO + + +:Template Name: + - {project-name}-releases-merge + +:Comment Trigger: remerge + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + :stream: run this job against: master + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :project-pattern: Project to trigger build against. (default: \*\*) + :submodule-recursive: Whether to checkout submodules recursively. + (default: false) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + + :gerrit_merge_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: ANT + pattern: 'releases/*.yaml' + + +Release Verify +------------------ + +Release verify job checks the schema and ensures that the staging-repo.txt.gz +is available on the job. + +- sigul-install +- sigul-configuration +- checkout ref from taglist.log +- applies the $PROJECT.bundle +- signs and shows signature + + +:Template Names: + - {project-name}-releases-verify + +:Comment Trigger: recheck|reverify + +:Required Parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + :stream: run this job against: master + +:Optional Parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-node: The node to run build on. + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :doc-dir: Directory where tox will place built docs. + as defined in the tox.ini (default: docs/_build/html) + :gerrit-skip-vote: Skip voting for this job. (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :project-pattern: Project to trigger build against. (default: \*\*) + :submodule-recursive: Whether to checkout submodules recursively. + (default: false) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + + :gerrit_verify_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: ANT + pattern: 'releases/*.yaml' diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 6804f482..82dc8404 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -271,6 +271,15 @@ properties-content: JENKINS_SSH_CREDENTIAL={jenkins-ssh-credential} - shell: !include-raw-escape: ../shell/check-info-votes.sh +- builder: + name: lf-releases + builders: + - config-file-provider: + files: + - file-id: lftoolsini + target: '$HOME/.config/lftools/lftools.ini' + - shell: !include-raw: ../shell/release-job.sh + - builder: name: lf-sigul-sign-dir # Requires that Jenkins be configured with SIGUL_BRIDGE_IP as a global diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml new file mode 100644 index 00000000..2852d4a8 --- /dev/null +++ b/jjb/lf-release-jobs.yaml @@ -0,0 +1,198 @@ +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +#################### +# COMMON FUNCTIONS # +#################### + +- lf_releases_common: &lf_releases_common + name: lf-releases-common + + ###################### + # Default parameters # + ###################### + + gerrit_trigger_file_paths: + - compare-type: ANT + pattern: 'releases/*.yaml' + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + + properties: + - lf-infra-properties: + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: 'master' + stream: '{stream}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + publishers: + - lf-infra-publish + +################## +# RELEASE VERIFY # +################## + +- lf_releases_verify: &lf_releases_verify + name: lf-releases-verify + + ###################### + # Default parameters # + ###################### + + branch: master + stream: master + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + git-url: '$GIT_URL/$PROJECT' + project-pattern: '**' + submodule-recursive: true + submodule-timeout: 10 + gerrit-skip-vote: false + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$' + + + ##################### + # Job Configuration # + ##################### + + disabled: '{disable-job}' + + builders: + - lf-infra-pre-build + - config-file-provider: + files: + - file-id: sigul-config + variable: SIGUL_CONFIG + - file-id: sigul-password + variable: SIGUL_PASSWORD + - file-id: sigul-pki + variable: SIGUL_PKI + - shell: !include-raw-escape: ../shell/sigul-configuration.sh + - shell: !include-raw-escape: ../shell/sigul-install.sh + - lf-releases + +- job-template: + name: '{project-name}-releases-verify' + id: gerrit-releases-verify + <<: *lf_releases_common + # yamllint disable-line rule:key-duplicates + <<: *lf_releases_verify + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: 'master' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: '{obj:gerrit_verify_triggers}' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project-pattern}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + file-paths: '{obj:gerrit_trigger_file_paths}' + skip-vote: + successful: '{gerrit-skip-vote}' + failed: '{gerrit-skip-vote}' + unstable: '{gerrit-skip-vote}' + notbuilt: '{gerrit-skip-vote}' + + +################# +# RELEASE MERGE # +################# + +- lf_releases_merge: &lf_releases_merge + name: lf-releases-merge + + ###################### + # Default parameters # + ###################### + + branch: master + stream: master + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + project-pattern: '**' + submodule-recursive: true + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: remerge$ + + ##################### + # Job Configuration # + ##################### + + disabled: '{disable-job}' + + builders: + - lf-infra-pre-build + - config-file-provider: + files: + - file-id: sigul-config + variable: SIGUL_CONFIG + - file-id: sigul-password + variable: SIGUL_PASSWORD + - file-id: sigul-pki + variable: SIGUL_PKI + - shell: !include-raw-escape: ../shell/sigul-configuration.sh + - shell: !include-raw-escape: ../shell/sigul-install.sh + - lf-releases + +- job-template: + name: '{project-name}-releases-merge' + id: gerrit-releases-merge + <<: *lf_releases_common + # yamllint disable-line rule:key-duplicates + <<: *lf_releases_merge + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: '{obj:gerrit_merge_triggers}' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project-pattern}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + file-paths: '{obj:gerrit_trigger_file_paths}' diff --git a/releasenotes/notes/lf-release-jobs-a618ab1220044253.yaml b/releasenotes/notes/lf-release-jobs-a618ab1220044253.yaml new file mode 100644 index 00000000..ead9dec9 --- /dev/null +++ b/releasenotes/notes/lf-release-jobs-a618ab1220044253.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + New ``lf-release-job-merge`` and ``lf-release-job-verify`` templates allow + projects to have self-serve releases. Project will create a tagname.yaml + file in the releases/ directory of their git repo. + example: + + .. code-block:: bash + + $ cat releases/4.0.0.yaml + --- + distribution_type: 'maven' + version: '4.0.0' + project: 'odlparent' + log_dir: 'odlparent-maven-release-master/11/' + #below is optional + maven_central_url: 'oss.sonatype.org' diff --git a/shell/release-job.sh b/shell/release-job.sh new file mode 100644 index 00000000..a5ab1169 --- /dev/null +++ b/shell/release-job.sh @@ -0,0 +1,108 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +set -eu -o pipefail + +if [ -d "/opt/pyenv" ]; then + echo "---> Setting up pyenv" + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" +fi +PYTHONPATH=$(pwd) +export PYTHONPATH +pyenv local 3.6.4 +export PYENV_VERSION="3.6.4" + +pip install --user lftools[nexus] jsonschema niet + +echo "########### Start Script release-job.sh ###################################" + +LOGS_SERVER="${LOGS_SERVER:-None}" +MAVEN_CENTRAL_URL="${MAVEN_CENTRAL_URL:-None}" + +#OPTIONAL +if grep -q "\.maven_central_url" "$release_file"; then + MAVEN_CENTRAL_URL="$(niet ".maven_central_url" "$release_file")" +fi + +if [ "${LOGS_SERVER}" == 'None' ]; then + echo "FAILED: log server not found" + exit 1 +fi + +NEXUS_URL="${ODLNEXUSPROXY:-$NEXUS_URL}" + +release_files=$(git diff HEAD^1 --name-only -- "releases/") +echo "RELEASE FILES ARE AS FOLLOWS: $release_files" + +for release_file in $release_files; do + echo "This is the release file: $release_file" + echo "--> Verifying $release_file Schema." + echo "DUMMY CODE:" + #Make sure the schema check catches a missing trailing / on log_dir + #lftools schema is written, but not the schema file (yet) + echo "lftools schema verify [OPTIONS] $release_file $SCHEMAFILE" + + VERSION="$(niet ".version" "$release_file")" + PROJECT="$(niet ".project" "$release_file")" + LOG_DIR="$(niet ".log_dir" "$release_file")" + + + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" + LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" + PATCH_DIR="$(mktemp -d)" + + pushd "$PATCH_DIR" + wget --quiet "${LOGS_URL}"staging-repo.txt.gz + STAGING_REPO="$(zcat staging-repo.txt)" + + #INFO + echo "INFO:" + echo "RELEASE_FILE: $release_file" + echo "LOGS_SERVER: $LOGS_SERVER" + echo "NEXUS_URL: $NEXUS_URL" + echo "NEXUS_PATH: $NEXUS_PATH" + echo "ODLNEXUSPROXY: $ODLNEXUSPROXY" + echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME" + echo "SILO: $SILO" + echo "PROJECT: $PROJECT" + echo "STAGING_REPO: $STAGING_REPO" + echo "VERSION: $VERSION" + echo "PROJECT: $PROJECT" + echo "LOG DIR: $LOG_DIR" + + wget --quiet "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz} + gunzip taglist.log.gz + cat "$PATCH_DIR"/taglist.log + popd + + git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")" + git fetch "$PATCH_DIR/$PROJECT.bundle" + git merge --ff-only FETCH_HEAD + git tag -am "$PROJECT $VERSION" "v$VERSION" + sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" v"$VERSION" < "$SIGUL_PASSWORD" + echo "Showing latest signature for $PROJECT:" + git log --show-signature -n1 + + + ########## Merge Part ############## + if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Running merge" + git push origin "v$VERSION" + lftools nexus release --server "$NEXUS_URL" "$STAGING_REPO" + if [ "${MAVEN_CENTRAL_URL}" == 'None' ]; then + echo "No Maven central url specified, not pushing to maven central" + else + lftools nexus release --server "$MAVEN_CENTRAL_URL" "$STAGING_REPO" + fi + fi + +done +echo "########### End Script release-job.sh ###################################" -- 2.16.6