shell/release-job.sh

   1 #!/bin/bash -l
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2019 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 echo "---> release-job.sh"
  12 set -eu -o pipefail
  13
  14 #Python bits. Remove when centos 7.7 builder is avaliable.
  15 if [ -d "/opt/pyenv" ]; then
  16     echo "INFO: Setting up pyenv"
  17     export PYENV_ROOT="/opt/pyenv"
  18     export PATH="$PYENV_ROOT/bin:$PATH"
  19 fi
  20 PYTHONPATH=$(pwd)
  21 export PYTHONPATH
  22 pyenv local 3.6.4
  23 export PYENV_VERSION="3.6.4"
  24 pip install --user lftools[nexus] jsonschema niet yq
  25
  26 #Functions.
  27
  28 set_variables_common(){
  29     echo "INFO: Setting all common variables"
  30     LOGS_SERVER="${LOGS_SERVER:-None}"
  31     if [ "${LOGS_SERVER}" == 'None' ]; then
  32         echo "ERROR: log server not found"
  33         exit 1
  34     fi
  35     NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
  36     # Verify if using release file or parameters
  37     if $USE_RELEASE_FILE ; then
  38         release_files=$(git diff-tree --no-commit-id -r "$GERRIT_PATCHSET_REVISION" --name-only -- "releases/" ".releases/")
  39         if (( $(grep -c . <<<"$release_files") > 1 )); then
  40           echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files"
  41           echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported."
  42           exit 1
  43         else
  44           release_file="$release_files"
  45           echo "INFO: RELEASE FILE: $release_files"
  46         fi
  47     else
  48         echo "INFO: This job is built with parameters, no release file needed. Continuing..."
  49         release_file="None"
  50     fi
  51
  52     DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}"
  53     if [[ $DISTRIBUTION_TYPE == "None" ]]; then
  54         DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")"
  55     fi
  56
  57     PATCH_DIR="$(mktemp -d)"
  58
  59     # Displaying Release Information (Common variables)
  60     printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO:
  61     printf "\t%-30s %s\n" RELEASE_FILE: $release_file
  62     printf "\t%-30s %s\n" LOGS_SERVER: $LOGS_SERVER
  63     printf "\t%-30s %s\n" NEXUS_PATH: $NEXUS_PATH
  64     printf "\t%-30s %s\n" JENKINS_HOSTNAME: $JENKINS_HOSTNAME
  65     printf "\t%-30s %s\n" SILO: $SILO
  66     printf "\t%-30s %s\n" PROJECT: $PROJECT
  67     printf "\t%-30s %s\n" PROJECT-DASHED: ${PROJECT//\//-}
  68     printf "\t%-30s %s\n" DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE
  69 }
  70
  71 set_variables_maven(){
  72     VERSION="${VERSION:-None}"
  73     if [[ $VERSION == "None" ]]; then
  74         VERSION="$(niet ".version" "$release_file")"
  75     fi
  76     LOG_DIR="${LOG_DIR:-None}"
  77     if [[ $LOG_DIR == "None" ]]; then
  78         LOG_DIR="$(niet ".log_dir" "$release_file")"
  79     fi
  80     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
  81     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
  82
  83     # Continuing displaying Release Information (Maven)
  84     printf "\t%-30s\n" RELEASE_MAVEN_INFO:
  85     printf "\t%-30s %s\n" VERSION: $VERSION
  86     printf "\t%-30s %s\n" LOG DIR: $LOG_DIR
  87     printf "\t%-30s %s\n" LOGS URL: $LOGS_URL
  88 }
  89
  90 set_variables_container(){
  91     VERSION="${VERSION:-None}"
  92     if [[ $VERSION == "None" ]]; then
  93         VERSION="$(niet ".container_release_tag" "$release_file")"
  94     fi
  95     if grep -q "container_pull_registry" "$release_file" ; then
  96         CONTAINER_PULL_REGISTRY="$(niet ".container_pull_registry" "$release_file")"
  97     fi
  98     if grep -q "container_push_registry" "$release_file" ; then
  99         CONTAINER_PUSH_REGISTRY="$(niet ".container_push_registry" "$release_file")"
 100     fi
 101     # Make sure both pull and push registries are defined
 102     if [ -z ${CONTAINER_PULL_REGISTRY+x} ] || [ -z ${CONTAINER_PUSH_REGISTRY+x} ]; then
 103         echo "ERROR: CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be defined"
 104         exit 1
 105     fi
 106     ref="$(niet ".ref" "$release_file")"
 107
 108     # Continuing displaying Release Information (Container)
 109     printf "\t%-30s\n" RELEASE_CONTAINER_INFO:
 110     printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: $VERSION
 111     printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: $CONTAINER_PULL_REGISTRY
 112     printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: $CONTAINER_PUSH_REGISTRY
 113     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref
 114 }
 115
 116 verify_schema(){
 117     echo "INFO: Verifying $release_file schema."
 118     lftools schema verify "$release_file" "$RELEASE_SCHEMA"
 119 }
 120
 121 verify_version(){
 122     # Verify allowed versions
 123     # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer
 124     allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$"
 125     if [[ ! $VERSION =~ $allowed_version_regex ]]; then
 126         echo "INFO: The version $VERSION is not a semantic valid version"
 127         echo "INFO: Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer"
 128         echo "INFO: See https://semver.org/ for more details on SemVer"
 129         exit 1
 130     fi
 131 }
 132
 133 tag(){
 134     # Import public signing key
 135     gpg --import "$SIGNING_PUBKEY"
 136     if git tag -v "$VERSION"; then
 137         echo "OK: Repo already tagged $VERSION Continuting to release"
 138     else
 139         echo "INFO: Repo has not yet been tagged $VERSION"
 140         git tag -am "${PROJECT//\//-} $VERSION" "$VERSION"
 141         sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD"
 142         echo "INFO: Showing latest signature for $PROJECT:"
 143         echo "INFO: git tag -v $VERSION"
 144         git tag -v "$VERSION"
 145
 146         ########## Merge Part ##############
 147         if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then
 148             echo "INFO: Running merge, pushing tag"
 149             gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}')
 150             git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/"$PROJECT"
 151             git config user.name "$RELEASE_USERNAME"
 152             git config user.email "$RELEASE_EMAIL"
 153             echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 154             chmod 600 ~/.ssh/config
 155             git push origin "$VERSION"
 156         fi
 157     fi
 158 }
 159
 160 nexus_release(){
 161     for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
 162         # extract the domain name from URL
 163         NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
 164         echo "INFO: NEXUS_URL: $NEXUS_URL"
 165         # extract the staging repo from URL
 166         STAGING_REPO=${staging_url#*repositories/}
 167         echo "INFO: Running Nexus Verify"
 168         lftools nexus release -v --server https://"$NEXUS_URL" "$STAGING_REPO"
 169         echo "INFO: Merge will run:"
 170         echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO"
 171     done
 172
 173     #Run the loop twice, to catch errors on either nexus repo
 174     if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then
 175         for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
 176           NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
 177           STAGING_REPO=${staging_url#*repositories/}
 178           echo "INFO: Promoting $STAGING_REPO on $NEXUS_URL."
 179           lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO"
 180         done
 181     fi
 182 }
 183
 184 container_release_file(){
 185     echo "INFO: Processing container release"
 186     local lfn_umbrella
 187     lfn_umbrella="$(echo "$GERRIT_HOST" | awk -F"." '{print $2}')"
 188
 189     for namequoted in $(cat $release_file | yq '.containers[].name'); do
 190         versionquoted=$(cat $release_file | yq ".containers[] |select(.name==$namequoted) |.version")
 191
 192         #Remove extra yaml quotes
 193         name="${namequoted#\"}"
 194         name="${name%\"}"
 195         version="${versionquoted#\"}"
 196         version="${version%\"}"
 197
 198         echo "$name"
 199         echo "$version"
 200         echo "INFO: Merge will release $name $version as $VERSION"
 201         # Attempt to pull from releases registry to see if the image has been released.
 202         if docker pull "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"; then
 203             echo "OK: $VERSION is already released for image $name, Continuing..."
 204         else
 205             echo "OK: $VERSION not found in releases, release will be prepared. Continuing..."
 206             docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version"
 207             container_image_id="$(docker images | grep $name | grep $version | awk '{print $3}')"
 208             echo "INFO: Merge will run the following commands:"
 209             echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
 210             echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
 211             if [[ "$JOB_NAME" =~ "merge" ]]; then
 212                 docker tag "$container_image_id" "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"
 213                 docker push "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"
 214             fi
 215             echo "#########################"
 216         fi
 217     done
 218
 219     echo "INFO: Merge will tag ref: $ref"
 220     git checkout "$ref"
 221     tag
 222 }
 223
 224 maven_release_file(){
 225     echo "INFO: wget -P $PATCH_DIR ${LOGS_URL}/staging-repo.txt.gz"
 226     wget -P "$PATCH_DIR" "${LOGS_URL}/"staging-repo.txt.gz
 227     pushd "$PATCH_DIR"
 228         echo "INFO: wget ${LOGS_URL}/patches/{${PROJECT//\//-}.bundle,taglist.log.gz}"
 229         wget "${LOGS_URL}"/patches/{"${PROJECT//\//-}".bundle,taglist.log.gz}
 230         gunzip taglist.log.gz
 231         cat "$PATCH_DIR"/taglist.log
 232     popd
 233     git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")"
 234     git fetch "$PATCH_DIR/${PROJECT//\//-}.bundle"
 235     git merge --ff-only FETCH_HEAD
 236     nexus_release
 237     tag
 238 }
 239
 240 echo "########### Start Script release-job.sh ###################################"
 241
 242 # Check if this is a container or maven release: release-container-schema.yaml vs release-schema.yaml
 243 # Logic to determine what we are releasing.
 244 ##########################################
 245
 246 # Set common environment variables
 247 set_variables_common
 248
 249 if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
 250     wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
 251     RELEASE_SCHEMA="release-schema.yaml"
 252     if $USE_RELEASE_FILE ; then
 253         verify_schema
 254     fi
 255     set_variables_maven
 256     verify_version
 257     maven_release_file
 258 elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then
 259     wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-container-schema.yaml
 260     RELEASE_SCHEMA="release-container-schema.yaml"
 261     verify_schema
 262     set_variables_container
 263     verify_version
 264     container_release_file
 265 else
 266     echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
 267     echo "Must be maven or container"
 268     exit 1
 269 fi
 270 ##########################################
 271
 272 echo "########### End Script release-job.sh ###################################"