From: Anil Belur <abelur@linuxfoundation.org>
Date: Fri, 2 Aug 2019 09:34:31 +0000 (+1000)
Subject: Handle multiple repos listed in staging-repo.
X-Git-Tag: v0.40.1~3
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=1396009703a2baf17938da83891714ea3db52d83;p=releng%2Fglobal-jjb.git

Handle multiple repos listed in staging-repo.

Handle multiple repos listed in staging-repo for ODL.

Also multiple release files in the same commit will not work.
exiting if this case is detected

Also
git diff HEAD^1 --name-only
is bad. if there is another merge it fails.

Change to
git diff-tree --no-commit-id -r $GERRIT_PATCHSET_REVISION \
    --name-only -- "releases/"

StrictHostKeyChecking is enabled on the builder.
disabled

Also needed to change

{jenkins-ssh-credential}
to
{jenkins-ssh-release-credential}

Also
Give group ``self-serve-release`` Forge Committer rights
on refs/tags/
and allow on Create Signed Tag Create Annotated Tag

Also
- file-id: signing-pubkey
target: SIGNING_PUBKEY

Also fix nexus url needs https://

Issue: RELENG-2129
Change-Id: I55dc0a4022f93a6c3c0f389c04c88c38c53eff64
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>
---

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index 73187b13..00c43cfb 100644
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -106,6 +106,10 @@ In Gerrit create a new group called ``self-serve-release`` and give it direct pu
 
 Add ``RELEASE_USERNAME`` to group ``self-serve-release`` and group ``Non-Interactive Users``
 
+Give group ``self-serve-release`` Forge Committer rights on ``refs/tags/*``
+Give group ``self-serve-release`` Allow on ``Create Signed Tag``
+Give group ``self-serve-release`` Allow on ``Create Annotated Tag``
+
 Jenkins
 =======
 
@@ -118,6 +122,16 @@ Jenkins configre -> Global properties -> Environment variables
 ``RELEASE_USERNAME = $RELEASE_USERNAME``
 ``RELEASE_EMAIL = $RELEASE_EMAIL``
 
+Jenkins configre -> Managed Files -> Custom File
+
+id: signing-pubkey
+Name: SIGNING_PUBKEY (optional)
+Comment: SIGNING_PUBKEY (optional)
+
+Content: (ask andy)
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+
 Add or edit the managed file in Jenkins called ``lftoolsini``, appending a nexus section:
 Jenkins Settings -> Managed files -> Add (or edit) -> Custom file
 
diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 02111ef7..e081dd0b 100644
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -42,7 +42,7 @@
     wrappers:
       - lf-infra-wrappers:
           build-timeout: "{build-timeout}"
-          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+          jenkins-ssh-credential: "{jenkins-ssh-release-credential}"
 
     publishers:
       - lf-infra-publish
@@ -91,8 +91,8 @@
               variable: SIGUL_PASSWORD
             - file-id: sigul-pki
               variable: SIGUL_PKI
-            - file-id: onap-pubkey
-              target: "/tmp/onap-pubkey"
+            - file-id: signing-pubkey
+              target: SIGNING_PUBKEY
       - shell: !include-raw-escape: ../shell/sigul-configuration.sh
       - shell: !include-raw-escape: ../shell/sigul-install.sh
       - lf-release
@@ -113,7 +113,7 @@
           submodule-recursive: false
           submodule-timeout: 10
           choosing-strategy: gerrit
-          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+          jenkins-ssh-credential: "{jenkins-ssh-release-credential}"
 
     triggers:
       - gerrit:
@@ -171,8 +171,8 @@
               variable: SIGUL_PASSWORD
             - file-id: sigul-pki
               variable: SIGUL_PKI
-            - file-id: onap-pubkey
-              target: "/tmp/onap-pubkey"
+            - file-id: signing-pubkey
+              target: SIGNING_PUBKEY
       - shell: !include-raw-escape: ../shell/sigul-configuration.sh
       - shell: !include-raw-escape: ../shell/sigul-install.sh
       - lf-release
@@ -186,7 +186,7 @@
 
     scm:
       - lf-infra-gerrit-scm:
-          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+          jenkins-ssh-credential: "{jenkins-ssh-release-credential}"
           git-url: "{git-url}"
           refspec: "$GERRIT_REFSPEC"
           branch: "$GERRIT_BRANCH"
diff --git a/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml b/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml
new file mode 100644
index 00000000..0c013992
--- /dev/null
+++ b/releasenotes/notes/promote-all-listed-staging-repos-b3318b925b09d2ab.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    The self-release jobs does not handle multiple repositories listed in
+    staging-repo.txt file. This fixes the issue by deriving the NEXUS_URL
+    and the STAGING_REPO from each entry in the file. This approach also
+    eliminates the need for having multiple release.yaml files for every
+    staging-repo.
diff --git a/shell/release-job.sh b/shell/release-job.sh
index 5b163c89..68f35db3 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -39,87 +39,102 @@ NEXUS_URL="${NEXUSPROXY:-$NEXUS_URL}"
 # Fetch the release-schema.yaml
 wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
 
-release_files=$(git diff HEAD^1 --name-only -- "releases/")
+release_files=$(git diff-tree --no-commit-id -r $GERRIT_PATCHSET_REVISION --name-only -- "releases/")
 echo "RELEASE FILES ARE AS FOLLOWS: $release_files"
 
-for release_file in $release_files; do
-    echo "--> Verifying $release_file schema."
-    lftools schema verify $release_file release-schema.yaml
-
-    #OPTIONAL
-    if grep -q "\.maven_central_url" "$release_file"; then
-        MAVEN_CENTRAL_URL="$(niet ".maven_central_url" "$release_file")"
-    fi
-
-    VERSION="$(niet ".version" "$release_file")"
-    PROJECT="$(niet ".project" "$release_file")"
-    LOG_DIR="$(niet ".log_dir" "$release_file")"
-
-    NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
-    LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
-    PATCH_DIR="$(mktemp -d)"
-
-    pushd "$PATCH_DIR"
-        wget --quiet "${LOGS_URL}"staging-repo.txt.gz
-        STAGING_REPO="$(zcat staging-repo.txt)"
-
-        #INFO
-        echo "INFO:"
-        echo "RELEASE_FILE: $release_file"
-        echo "LOGS_SERVER: $LOGS_SERVER"
-        echo "NEXUS_URL: $NEXUS_URL"
-        echo "NEXUS_PATH: $NEXUS_PATH"
-        echo "NEXUSPROXY: $NEXUSPROXY"
-        echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME"
-        echo "SILO: $SILO"
-        echo "PROJECT: $PROJECT"
-        echo "STAGING_REPO: $STAGING_REPO"
-        echo "VERSION: $VERSION"
-        echo "PROJECT: $PROJECT"
-        echo "LOG DIR: $LOG_DIR"
-
-        wget --quiet  "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz}
-        gunzip taglist.log.gz
-        cat "$PATCH_DIR"/taglist.log
-    popd
-
-    # Verify allowed versions
-    # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer
-    allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$"
-    if [[ ! $VERSION =~ $allowed_version_regex ]]; then
-        echo "The version $VERSION is not a semantic valid version"
-        echo "Allowed versions are "v#.#.#" or "#.#.#" aka SemVer"
-        echo "See https://semver.org/ for more details on SemVer"
-        exit 1
-    fi
-
-    git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")"
-    git fetch "$PATCH_DIR/$PROJECT.bundle"
-    git merge --ff-only FETCH_HEAD
-    git tag -am "$PROJECT $VERSION" "$VERSION"
-    sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD"
-
-    echo "Showing latest signature for $PROJECT:"
-    gpg --import /tmp/onap-pubkey
-    echo "git tag -v "$VERSION""
-    git tag -v "$VERSION"
-
-
-    ########## Merge Part ##############
-    if [[ "$JOB_NAME" =~ "merge" ]]; then
-        echo "Running merge"
-        gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}')
-        git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/$PROJECT
-        git config user.name "$RELEASE_USERNAME"
-        git config user.email "$RELEASE_EMAIL"
-        git push origin "$VERSION"
-        lftools nexus release --server "$NEXUS_URL" "$STAGING_REPO"
-        if [ "${MAVEN_CENTRAL_URL}" == 'None' ]; then
-            echo "No Maven central url specified, not pushing to maven central"
-        else
-            lftools nexus release --server "$MAVEN_CENTRAL_URL" "$STAGING_REPO"
-        fi
-    fi
+if (( $(grep -c . <<<"$release_files") > 1 )); then
+  echo "multiple release files in the same commit do not make sense"
+  exit 1
+else
+  release_file="$release_files"
+  echo "RELEASE FILE IS AS FOLLOWS: $release_file"
+fi
 
+echo "--> Verifying $release_file schema."
+lftools schema verify $release_file release-schema.yaml
+
+VERSION="$(niet ".version" "$release_file")"
+PROJECT="$(niet ".project" "$release_file")"
+LOG_DIR="$(niet ".log_dir" "$release_file")"
+
+NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
+LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+PATCH_DIR="$(mktemp -d)"
+
+wget --quiet -P "$PATCH_DIR" "${LOGS_URL}"staging-repo.txt.gz
+
+nexus_release(){
+for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
+  # extract the domain name from URL
+  NEXUS_URL=$(echo $staging_url | sed -e 's|^[^/]*//||' -e 's|/.*$||')
+  # extract the staging repo from URL
+  STAGING_REPO=${staging_url#*repositories/}
+  echo "Merge will run"
+  echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO"
+  if [[ "$JOB_NAME" =~ "merge" ]]; then
+    echo "Promoting $STAGING_REPO on $NEXUS_URL."
+    lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO"
+  fi
 done
+}
+
+
+#INFO
+echo "INFO:"
+echo "RELEASE_FILE: $release_file"
+echo "LOGS_SERVER: $LOGS_SERVER"
+echo "NEXUS_URL: $NEXUS_URL"
+echo "NEXUS_PATH: $NEXUS_PATH"
+echo "NEXUSPROXY: $NEXUSPROXY"
+echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME"
+echo "SILO: $SILO"
+echo "PROJECT: $PROJECT"
+echo "VERSION: $VERSION"
+echo "PROJECT: $PROJECT"
+echo "LOG DIR: $LOG_DIR"
+
+pushd "$PATCH_DIR"
+  wget --quiet  "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz}
+  gunzip taglist.log.gz
+  cat "$PATCH_DIR"/taglist.log
+popd
+
+# Verify allowed versions
+# Allowed versions are "v#.#.#" or "#.#.#" aka SemVer
+allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$"
+if [[ ! $VERSION =~ $allowed_version_regex ]]; then
+  echo "The version $VERSION is not a semantic valid version"
+  echo "Allowed versions are "v#.#.#" or "#.#.#" aka SemVer"
+  echo "See https://semver.org/ for more details on SemVer"
+  exit 1
+fi
+
+git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")"
+git fetch "$PATCH_DIR/$PROJECT.bundle"
+git merge --ff-only FETCH_HEAD
+git tag -am "$PROJECT $VERSION" "$VERSION"
+sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD"
+
+echo "Showing latest signature for $PROJECT:"
+gpg --import "$SIGNING_PUBKEY"
+echo "git tag -v "$VERSION""
+git tag -v "$VERSION"
+
+########## Merge Part ##############
+if [[ "$JOB_NAME" =~ "merge" ]]; then
+  echo "Running merge"
+  gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}')
+  git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/$PROJECT
+  git config user.name "$RELEASE_USERNAME"
+  git config user.email "$RELEASE_EMAIL"
+  echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+  chmod 600 ~/.ssh/config
+  git push origin "$VERSION"
+
+
+fi
+
+# This function: if merge push to nexus. If verify output the proposed push command.
+nexus_release
+
 echo "########### End Script release-job.sh ###################################"