From 1396009703a2baf17938da83891714ea3db52d83 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Fri, 2 Aug 2019 19:34:31 +1000 Subject: [PATCH] Handle multiple repos listed in staging-repo. Handle multiple repos listed in staging-repo for ODL. Also multiple release files in the same commit will not work. exiting if this case is detected Also git diff HEAD^1 --name-only is bad. if there is another merge it fails. Change to git diff-tree --no-commit-id -r $GERRIT_PATCHSET_REVISION \ --name-only -- "releases/" StrictHostKeyChecking is enabled on the builder. disabled Also needed to change {jenkins-ssh-credential} to {jenkins-ssh-release-credential} Also Give group ``self-serve-release`` Forge Committer rights on refs/tags/ and allow on Create Signed Tag Create Annotated Tag Also - file-id: signing-pubkey target: SIGNING_PUBKEY Also fix nexus url needs https:// Issue: RELENG-2129 Change-Id: I55dc0a4022f93a6c3c0f389c04c88c38c53eff64 Signed-off-by: Anil Belur --- docs/jjb/lf-release-jobs.rst | 14 ++ jjb/lf-release-jobs.yaml | 14 +- ...-all-listed-staging-repos-b3318b925b09d2ab.yaml | 8 + shell/release-job.sh | 173 +++++++++++---------- 4 files changed, 123 insertions(+), 86 deletions(-) create mode 100644 releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index 73187b13..00c43cfb 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -106,6 +106,10 @@ In Gerrit create a new group called ``self-serve-release`` and give it direct pu Add ``RELEASE_USERNAME`` to group ``self-serve-release`` and group ``Non-Interactive Users`` +Give group ``self-serve-release`` Forge Committer rights on ``refs/tags/*`` +Give group ``self-serve-release`` Allow on ``Create Signed Tag`` +Give group ``self-serve-release`` Allow on ``Create Annotated Tag`` + Jenkins ======= @@ -118,6 +122,16 @@ Jenkins configre -> Global properties -> Environment variables ``RELEASE_USERNAME = $RELEASE_USERNAME`` ``RELEASE_EMAIL = $RELEASE_EMAIL`` +Jenkins configre -> Managed Files -> Custom File + +id: signing-pubkey +Name: SIGNING_PUBKEY (optional) +Comment: SIGNING_PUBKEY (optional) + +Content: (ask andy) +-----BEGIN PGP PUBLIC KEY BLOCK----- + + Add or edit the managed file in Jenkins called ``lftoolsini``, appending a nexus section: Jenkins Settings -> Managed files -> Add (or edit) -> Custom file diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml index 02111ef7..e081dd0b 100644 --- a/jjb/lf-release-jobs.yaml +++ b/jjb/lf-release-jobs.yaml @@ -42,7 +42,7 @@ wrappers: - lf-infra-wrappers: build-timeout: "{build-timeout}" - jenkins-ssh-credential: "{jenkins-ssh-credential}" + jenkins-ssh-credential: "{jenkins-ssh-release-credential}" publishers: - lf-infra-publish @@ -91,8 +91,8 @@ variable: SIGUL_PASSWORD - file-id: sigul-pki variable: SIGUL_PKI - - file-id: onap-pubkey - target: "/tmp/onap-pubkey" + - file-id: signing-pubkey + target: SIGNING_PUBKEY - shell: !include-raw-escape: ../shell/sigul-configuration.sh - shell: !include-raw-escape: ../shell/sigul-install.sh - lf-release @@ -113,7 +113,7 @@ submodule-recursive: false submodule-timeout: 10 choosing-strategy: gerrit - jenkins-ssh-credential: "{jenkins-ssh-credential}" + jenkins-ssh-credential: "{jenkins-ssh-release-credential}" triggers: - gerrit: @@ -171,8 +171,8 @@ variable: SIGUL_PASSWORD - file-id: sigul-pki variable: SIGUL_PKI - - file-id: onap-pubkey - target: "/tmp/onap-pubkey" + - file-id: signing-pubkey + target: SIGNING_PUBKEY - shell: !include-raw-escape: ../shell/sigul-configuration.sh - shell: !include-raw-escape: ../shell/sigul-install.sh - lf-release @@ -186,7 +186,7 @@ scm: - lf-infra-gerrit-scm: - jenkins-ssh-credential: "{jenkins-ssh-credential}" + jenkins-ssh-credential: "{jenkins-ssh-release-credential}" git-url: "{git-url}" refspec: "$GERRIT_REFSPEC" branch: "$GERRIT_BRANCH" diff --git a/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml b/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml new file mode 100644 index 00000000..0c013992 --- /dev/null +++ b/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + The self-release jobs does not handle multiple repositories listed in + staging-repo.txt file. This fixes the issue by deriving the NEXUS_URL + and the STAGING_REPO from each entry in the file. This approach also + eliminates the need for having multiple release.yaml files for every + staging-repo. diff --git a/shell/release-job.sh b/shell/release-job.sh index 5b163c89..68f35db3 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -39,87 +39,102 @@ NEXUS_URL="${NEXUSPROXY:-$NEXUS_URL}" # Fetch the release-schema.yaml wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml -release_files=$(git diff HEAD^1 --name-only -- "releases/") +release_files=$(git diff-tree --no-commit-id -r $GERRIT_PATCHSET_REVISION --name-only -- "releases/") echo "RELEASE FILES ARE AS FOLLOWS: $release_files" -for release_file in $release_files; do - echo "--> Verifying $release_file schema." - lftools schema verify $release_file release-schema.yaml - - #OPTIONAL - if grep -q "\.maven_central_url" "$release_file"; then - MAVEN_CENTRAL_URL="$(niet ".maven_central_url" "$release_file")" - fi - - VERSION="$(niet ".version" "$release_file")" - PROJECT="$(niet ".project" "$release_file")" - LOG_DIR="$(niet ".log_dir" "$release_file")" - - NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" - LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" - PATCH_DIR="$(mktemp -d)" - - pushd "$PATCH_DIR" - wget --quiet "${LOGS_URL}"staging-repo.txt.gz - STAGING_REPO="$(zcat staging-repo.txt)" - - #INFO - echo "INFO:" - echo "RELEASE_FILE: $release_file" - echo "LOGS_SERVER: $LOGS_SERVER" - echo "NEXUS_URL: $NEXUS_URL" - echo "NEXUS_PATH: $NEXUS_PATH" - echo "NEXUSPROXY: $NEXUSPROXY" - echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME" - echo "SILO: $SILO" - echo "PROJECT: $PROJECT" - echo "STAGING_REPO: $STAGING_REPO" - echo "VERSION: $VERSION" - echo "PROJECT: $PROJECT" - echo "LOG DIR: $LOG_DIR" - - wget --quiet "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz} - gunzip taglist.log.gz - cat "$PATCH_DIR"/taglist.log - popd - - # Verify allowed versions - # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer - allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$" - if [[ ! $VERSION =~ $allowed_version_regex ]]; then - echo "The version $VERSION is not a semantic valid version" - echo "Allowed versions are "v#.#.#" or "#.#.#" aka SemVer" - echo "See https://semver.org/ for more details on SemVer" - exit 1 - fi - - git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")" - git fetch "$PATCH_DIR/$PROJECT.bundle" - git merge --ff-only FETCH_HEAD - git tag -am "$PROJECT $VERSION" "$VERSION" - sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD" - - echo "Showing latest signature for $PROJECT:" - gpg --import /tmp/onap-pubkey - echo "git tag -v "$VERSION"" - git tag -v "$VERSION" - - - ########## Merge Part ############## - if [[ "$JOB_NAME" =~ "merge" ]]; then - echo "Running merge" - gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}') - git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/$PROJECT - git config user.name "$RELEASE_USERNAME" - git config user.email "$RELEASE_EMAIL" - git push origin "$VERSION" - lftools nexus release --server "$NEXUS_URL" "$STAGING_REPO" - if [ "${MAVEN_CENTRAL_URL}" == 'None' ]; then - echo "No Maven central url specified, not pushing to maven central" - else - lftools nexus release --server "$MAVEN_CENTRAL_URL" "$STAGING_REPO" - fi - fi +if (( $(grep -c . <<<"$release_files") > 1 )); then + echo "multiple release files in the same commit do not make sense" + exit 1 +else + release_file="$release_files" + echo "RELEASE FILE IS AS FOLLOWS: $release_file" +fi +echo "--> Verifying $release_file schema." +lftools schema verify $release_file release-schema.yaml + +VERSION="$(niet ".version" "$release_file")" +PROJECT="$(niet ".project" "$release_file")" +LOG_DIR="$(niet ".log_dir" "$release_file")" + +NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" +LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" +PATCH_DIR="$(mktemp -d)" + +wget --quiet -P "$PATCH_DIR" "${LOGS_URL}"staging-repo.txt.gz + +nexus_release(){ +for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do + # extract the domain name from URL + NEXUS_URL=$(echo $staging_url | sed -e 's|^[^/]*//||' -e 's|/.*$||') + # extract the staging repo from URL + STAGING_REPO=${staging_url#*repositories/} + echo "Merge will run" + echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO" + if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Promoting $STAGING_REPO on $NEXUS_URL." + lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO" + fi done +} + + +#INFO +echo "INFO:" +echo "RELEASE_FILE: $release_file" +echo "LOGS_SERVER: $LOGS_SERVER" +echo "NEXUS_URL: $NEXUS_URL" +echo "NEXUS_PATH: $NEXUS_PATH" +echo "NEXUSPROXY: $NEXUSPROXY" +echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME" +echo "SILO: $SILO" +echo "PROJECT: $PROJECT" +echo "VERSION: $VERSION" +echo "PROJECT: $PROJECT" +echo "LOG DIR: $LOG_DIR" + +pushd "$PATCH_DIR" + wget --quiet "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz} + gunzip taglist.log.gz + cat "$PATCH_DIR"/taglist.log +popd + +# Verify allowed versions +# Allowed versions are "v#.#.#" or "#.#.#" aka SemVer +allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$" +if [[ ! $VERSION =~ $allowed_version_regex ]]; then + echo "The version $VERSION is not a semantic valid version" + echo "Allowed versions are "v#.#.#" or "#.#.#" aka SemVer" + echo "See https://semver.org/ for more details on SemVer" + exit 1 +fi + +git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")" +git fetch "$PATCH_DIR/$PROJECT.bundle" +git merge --ff-only FETCH_HEAD +git tag -am "$PROJECT $VERSION" "$VERSION" +sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD" + +echo "Showing latest signature for $PROJECT:" +gpg --import "$SIGNING_PUBKEY" +echo "git tag -v "$VERSION"" +git tag -v "$VERSION" + +########## Merge Part ############## +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Running merge" + gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}') + git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/$PROJECT + git config user.name "$RELEASE_USERNAME" + git config user.email "$RELEASE_EMAIL" + echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + chmod 600 ~/.ssh/config + git push origin "$VERSION" + + +fi + +# This function: if merge push to nexus. If verify output the proposed push command. +nexus_release + echo "########### End Script release-job.sh ###################################" -- 2.16.6