X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=blobdiff_plain;f=shell%2Frelease-job.sh;h=f8837e81e8c34772fd33e7567d60dab8dc3a4e5c;hb=568b22ef45e66def2e9b3361b3d438fd5b32fd97;hp=b764ac1919789a80b2d55c20f5773b0e1358b5b1;hpb=200c13dbad2552db1ba9cfc185b0dc40db863930;p=releng%2Fglobal-jjb.git

diff --git a/shell/release-job.sh b/shell/release-job.sh
index b764ac19..f8837e81 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -11,12 +11,12 @@
 echo "---> release-job.sh"
 set -eu -o pipefail
 
-set +u
-python3 -m venv /tmp/v/venv/
-# shellcheck disable=SC1091
-source /tmp/v/venv/bin/activate
-set -u
-python -m pip install lftools[nexus] jsonschema niet yq
+echo "INFO: creating virtual environment"
+virtualenv -p python3 /tmp/venv
+PATH=/tmp/venv/bin:$PATH
+pipup="python -m pip install -q --upgrade pip lftools[nexus] jsonschema niet twine yq"
+echo "INFO: $pipup"
+$pipup
 
 #Functions.
 
@@ -30,7 +30,7 @@ set_variables_common(){
     NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
     # Verify if using release file or parameters
     if $USE_RELEASE_FILE ; then
-        release_files=$(git diff-tree --no-commit-id -r "$GERRIT_PATCHSET_REVISION" --name-only -- "releases/" ".releases/")
+        release_files=$(git diff-tree -m --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
         if (( $(grep -c . <<<"$release_files") > 1 )); then
           echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files"
           echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported."
@@ -78,8 +78,8 @@ set_variables_maven(){
     # Continuing displaying Release Information (Maven)
     printf "\t%-30s\n" RELEASE_MAVEN_INFO:
     printf "\t%-30s %s\n" VERSION: $VERSION
-    printf "\t%-30s %s\n" LOG DIR: $LOG_DIR
-    printf "\t%-30s %s\n" LOGS URL: $LOGS_URL
+    printf "\t%-30s %s\n" LOG_DIR: $LOG_DIR
+    printf "\t%-30s %s\n" LOGS_URL: $LOGS_URL
 }
 
 set_variables_container(){
@@ -108,16 +108,50 @@ set_variables_container(){
     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref
 }
 
+set_variables_pypi(){
+    # use Jenkins parameter if set; else get value from release file
+    echo "INFO: Setting pypi variables"
+    LOG_DIR="${LOG_DIR:-None}"
+    if [[ $LOG_DIR == "None" ]]; then
+        LOG_DIR="$(yq -er .log_dir "$release_file")"
+    fi
+    LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+    LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
+    PYPI_PROJECT="${PYPI_PROJECT:-None}"
+    if [[ $PYPI_PROJECT == "None" ]]; then
+        PYPI_PROJECT="$(yq -er .pypi_project "$release_file")"
+    fi
+    PYTHON_VERSION="${PYTHON_VERSION:-None}"
+    if [[ $PYTHON_VERSION == "None" ]]; then
+        PYTHON_VERSION="$(yq -er .python_version "$release_file")"
+    fi
+    VERSION="${VERSION:-None}"
+    if [[ $VERSION == "None" ]]; then
+        VERSION="$(yq -er .version "$release_file")"
+    fi
+
+    # Continuing displaying Release Information (pypi)
+    printf "\t%-30s\n" RELEASE_PYPI_INFO:
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$LOGS_URL"
+    printf "\t%-30s %s\n" PYPI_INDEX: "$PYPI_INDEX" # from job configuration
+    printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
+    printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
+    printf "\t%-30s %s\n" VERSION: "$VERSION"
+}
+
 verify_schema(){
     echo "INFO: Verifying $release_file schema."
-    lftools schema verify "$release_file" "$RELEASE_SCHEMA"
+    lftools schema verify "$release_file" "$release_schema"
 }
 
 verify_version(){
-    # Verify allowed versions
-    # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer
+    # Verify allowed patterns "v#.#.#" or "#.#.#" aka SemVer
+    echo "INFO: Verifying version string $VERSION"
     allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$"
-    if [[ ! $VERSION =~ $allowed_version_regex ]]; then
+    if [[ $VERSION =~ $allowed_version_regex ]]; then
+        echo "INFO: The version $VERSION is a valid semantic version"
+    else
         echo "INFO: The version $VERSION is not a semantic valid version"
         echo "INFO: Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer"
         echo "INFO: See https://semver.org/ for more details on SemVer"
@@ -125,6 +159,33 @@ verify_version(){
     fi
 }
 
+verify_version_match_release(){
+    wget -P /tmp "${LOGS_URL}/"console.log.gz
+    echo "INFO: Comparing version $VERSION with log snippet from maven-stage:"
+    if zgrep "Successfully uploaded" /tmp/console.log.gz | grep "$VERSION"; then
+        echo "INFO: version $VERSION matches maven-stage artifacts"
+    else
+        echo "ERROR: Defined version in release file does not match staging repo artifacts version to be released"
+        echo "       Please make sure maven-stage job selected as candidate and release version are correct"
+        exit 1
+    fi
+}
+
+# check prerequisites to detect mistakes in the release YAML file
+verify_pypi_match_release(){
+    wget -q -P /tmp "${LOGS_URL}/"console.log.gz
+    echo "INFO: Searching for strings >$PYPI_PROJECT< and >$VERSION< in job log"
+    # pypi-upload.sh generates success message with file list
+    if zgrep -i "uploaded" /tmp/console.log.gz | grep "$PYPI_PROJECT" | grep "$VERSION" ; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+
+# sigul is only available on Centos
+# TODO: write tag_github function
 tag(){
     # Import public signing key
     gpg --import "$SIGNING_PUBKEY"
@@ -232,36 +293,147 @@ maven_release_file(){
     tag
 }
 
-echo "########### Start Script release-job.sh ###################################"
+# calls pip to download binary and source distributions from the specified index,
+# which requires a recent-in-2019 version.  Uploads the files it received.
+pypi_release_file(){
+    tgtdir=dist
+    mkdir $tgtdir
+    pip_pfx="pip download -d $tgtdir --no-deps --python-version $PYTHON_VERSION -i $PYPI_INDEX"
+    module="$PYPI_PROJECT==$VERSION"
+    pip_bin="$pip_pfx $module"
+    echo "INFO: downloading binary: $pip_bin"
+    if ! $pip_bin ; then
+        echo "WARN: failed to download binary distribution"
+    fi
+    pip_src="$pip_pfx --no-binary=:all: $module"
+    echo "INFO: downloading source: $pip_src"
+    if ! $pip_src ; then
+        echo "WARN: failed to download source distribution"
+    fi
+    echo "INFO: Checking files in $tgtdir"
+    filecount=$(ls $tgtdir | wc -l)
+    if [[ $filecount = 0 ]] ; then
+        echo "ERROR: no files downloaded"
+        exit 1
+    else
+        # shellcheck disable=SC2046
+        echo "INFO: downloaded $filecount distributions: " $(ls $tgtdir)
+    fi
+
+    if [[ ! "$JOB_NAME" =~ "merge" ]] ; then
+        echo "INFO: not a merge job, not uploading files"
+        return
+    fi
+
+    cmd="twine upload -r $REPOSITORY $tgtdir/*"
+    if $DRY_RUN; then
+        echo "INFO: dry-run is set, echoing command only"
+        echo "$cmd"
+    else
+        echo "INFO: uploading $filecount distributions to repo $REPOSITORY"
+        $cmd
+    fi
+    tag
+}
+
+packagecloud_verify(){
+    echo "INFO: Verifying $1 exists in staging..."
+    if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
+        https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+        | yq -r .[].filename) ]]; then
+        echo "INFO: $1 exists in staging!"
+        echo "INFO: Existing package location: https://packagecloud.io$(curl \
+            --netrc-file ~/packagecloud_api --silent \
+            https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+            | yq -r .[].package_html_url)"
+    else
+        echo "ERROR: $1 does not exist in staging"
+        exit 1
+    fi
+}
 
-# Check if this is a container or maven release: release-container-schema.yaml vs release-schema.yaml
-# Logic to determine what we are releasing.
-##########################################
+packagecloud_promote(){
+    echo "INFO: Preparing to promote $1..."
+    promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
+        --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+        | yq -r .[].promote_url)"
+    echo "INFO: Promoting $1 from staging to release"
+    curl --netrc-file ~/packagecloud_api -X POST -F \
+        destination="$2/release" "$promote_url" \
+        | echo "INFO: Promoted package location: \
+        https://packagecloud.io$(yq -r .package_html_url)"
+}
+
+##############################  End Function Declarations  ################################
 
 # Set common environment variables
 set_variables_common
 
-if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
-    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
-    RELEASE_SCHEMA="release-schema.yaml"
-    if $USE_RELEASE_FILE ; then
+# Determine the type of release:
+#   - container, release-container-schema.yaml
+#   - maven, release-schema.yaml
+#   - pypi,  release-pypi-schema.yaml
+
+case $DISTRIBUTION_TYPE in
+
+    maven)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema
+            verify_schema
+        fi
+        set_variables_maven
+        verify_version
+        verify_version_match_release
+        maven_release_file
+        ;;
+
+    container)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-container-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_container
+        verify_version
+        container_release_file
+        ;;
+
+    pypi)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-pypi-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_pypi
+        verify_version
+        verify_pypi_match_release
+        pypi_release_file
+        ;;
+
+    packagecloud)
+        release_schema="release-packagecloud-schema.yaml"
+        package_name=$(yq -r '.package_name' $release_file)
+        packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
+        echo "INFO: Fetching schema $release_schema"
+        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
         verify_schema
-    fi
-    set_variables_maven
-    verify_version
-    maven_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then
-    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-container-schema.yaml
-    RELEASE_SCHEMA="release-container-schema.yaml"
-    verify_schema
-    set_variables_container
-    verify_version
-    container_release_file
-else
-    echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
-    echo "Must be maven or container"
-    exit 1
-fi
-##########################################
-
-echo "########### End Script release-job.sh ###################################"
+        for name in $(yq -r '.package_name[].name' $release_file); do
+            package_name=$name
+            packagecloud_verify "$package_name" "$packagecloud_account"
+            if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
+                packagecloud_promote "$package_name" "$packagecloud_account"
+            fi
+        done
+        ;;
+
+    *)
+        echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
+        exit 1
+        ;;
+esac
+
+echo "---> release-job.sh ends"