X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=blobdiff_plain;f=shell%2Frelease-job.sh;fp=shell%2Frelease-job.sh;h=5ee06f559b55cbfec8ea0db2b89ac9ceebb02dc0;hb=c53e11db14ab7abede74c74801bec9ca381c8cc6;hp=7b22b280fe636b04f6a8229a9bce4d140e8f9b75;hpb=d0dfd24055995c4a2ed27fb7bc9e3dee65ba0d16;p=releng%2Fglobal-jjb.git

diff --git a/shell/release-job.sh b/shell/release-job.sh
index 7b22b280..5ee06f55 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -11,12 +11,12 @@
 echo "---> release-job.sh"
 set -eu -o pipefail
 
-set +u
-python3 -m venv /tmp/v/venv/
-# shellcheck disable=SC1091
-source /tmp/v/venv/bin/activate
-set -u
-python -m pip install lftools[nexus] jsonschema niet yq
+echo "INFO: creating virtual environment"
+virtualenv -p python3 /tmp/venv
+PATH=/tmp/venv/bin:$PATH
+pipup="python -m pip install -q --upgrade pip lftools[nexus] jsonschema niet twine yq"
+echo "INFO: $pipup"
+$pipup
 
 #Functions.
 
@@ -30,7 +30,7 @@ set_variables_common(){
     NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
     # Verify if using release file or parameters
     if $USE_RELEASE_FILE ; then
-        release_files=$(git diff-tree --no-commit-id -r "$GERRIT_PATCHSET_REVISION" --name-only -- "releases/" ".releases/")
+        release_files=$(git diff-tree --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
         if (( $(grep -c . <<<"$release_files") > 1 )); then
           echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files"
           echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported."
@@ -108,16 +108,50 @@ set_variables_container(){
     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref
 }
 
+set_variables_pypi(){
+    # use Jenkins parameter if set; else get value from release file
+    echo "INFO: Setting pypi variables"
+    LOG_DIR="${LOG_DIR:-None}"
+    if [[ $LOG_DIR == "None" ]]; then
+        LOG_DIR="$(yq -er .log_dir "$release_file")"
+    fi
+    LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+    LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
+    PYPI_PROJECT="${PYPI_PROJECT:-None}"
+    if [[ $PYPI_PROJECT == "None" ]]; then
+        PYPI_PROJECT="$(yq -er .pypi_project "$release_file")"
+    fi
+    PYTHON_VERSION="${PYTHON_VERSION:-None}"
+    if [[ $PYTHON_VERSION == "None" ]]; then
+        PYTHON_VERSION="$(yq -er .python_version "$release_file")"
+    fi
+    VERSION="${VERSION:-None}"
+    if [[ $VERSION == "None" ]]; then
+        VERSION="$(yq -er .version "$release_file")"
+    fi
+
+    # Continuing displaying Release Information (pypi)
+    printf "\t%-30s\n" RELEASE_PYPI_INFO:
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$LOGS_URL"
+    printf "\t%-30s %s\n" PYPI_INDEX: "$PYPI_INDEX" # from job configuration
+    printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
+    printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
+    printf "\t%-30s %s\n" VERSION: "$VERSION"
+}
+
 verify_schema(){
     echo "INFO: Verifying $release_file schema."
     lftools schema verify "$release_file" "$RELEASE_SCHEMA"
 }
 
 verify_version(){
-    # Verify allowed versions
-    # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer
+    # Verify allowed patterns "v#.#.#" or "#.#.#" aka SemVer
+    echo "INFO: Verifying version string $VERSION"
     allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$"
-    if [[ ! $VERSION =~ $allowed_version_regex ]]; then
+    if [[ $VERSION =~ $allowed_version_regex ]]; then
+        echo "INFO: The version $VERSION is a valid semantic version"
+    else
         echo "INFO: The version $VERSION is not a semantic valid version"
         echo "INFO: Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer"
         echo "INFO: See https://semver.org/ for more details on SemVer"
@@ -137,6 +171,21 @@ verify_version_match_release(){
     fi
 }
 
+# check prerequisites to detect mistakes in the release YAML file
+verify_pypi_match_release(){
+    wget -q -P /tmp "${LOGS_URL}/"console.log.gz
+    echo "INFO: Searching for strings >$PYPI_PROJECT< and >$VERSION< in job log"
+    # pypi-upload.sh generates success message with file list
+    if zgrep -i "uploaded" /tmp/console.log.gz | grep "$PYPI_PROJECT" | grep "$VERSION" ; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+
+# sigul is only available on Centos
+# TODO: write tag_github function
 tag(){
     # Import public signing key
     gpg --import "$SIGNING_PUBKEY"
@@ -244,19 +293,62 @@ maven_release_file(){
     tag
 }
 
-echo "########### Start Script release-job.sh ###################################"
+# calls pip to download binary and source distributions from the specified index,
+# which requires a recent-in-2019 version.  Uploads the files it received.
+pypi_release_file(){
+    tgtdir=dist
+    mkdir $tgtdir
+    pip_pfx="pip download -d $tgtdir --no-deps --python-version $PYTHON_VERSION -i $PYPI_INDEX"
+    module="$PYPI_PROJECT==$VERSION"
+    pip_bin="$pip_pfx $module"
+    echo "INFO: downloading binary: $pip_bin"
+    if ! $pip_bin ; then
+        echo "WARN: failed to download binary distribution"
+    fi
+    pip_src="$pip_pfx --no-binary=:all: $module"
+    echo "INFO: downloading source: $pip_src"
+    if ! $pip_src ; then
+        echo "WARN: failed to download source distribution"
+    fi
+    echo "INFO: Checking files in $tgtdir"
+    filecount=$(ls $tgtdir | wc -l)
+    if [[ $filecount = 0 ]] ; then
+        echo "ERROR: no files downloaded"
+        exit 1
+    else
+        # shellcheck disable=SC2046
+        echo "INFO: downloaded $filecount distributions: " $(ls $tgtdir)
+    fi
 
-# Check if this is a container or maven release: release-container-schema.yaml vs release-schema.yaml
-# Logic to determine what we are releasing.
-##########################################
+    if [[ ! "$JOB_NAME" =~ "merge" ]] ; then
+        echo "INFO: not a merge job, not uploading files"
+        return
+    fi
+
+    cmd="twine upload -r $REPOSITORY $tgtdir/*"
+    if $DRY_RUN; then
+        echo "INFO: dry-run is set, echoing command only"
+        echo "$cmd"
+    else
+        echo "INFO: uploading $filecount distributions to repo $REPOSITORY"
+        $cmd
+    fi
+    tag
+}
 
 # Set common environment variables
 set_variables_common
 
+# Determine the type of release:
+#   - container, release-container-schema.yaml
+#   - maven, release-schema.yaml
+#   - pypi,  release-pypi-schema.yaml
+
 if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
-    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
-    RELEASE_SCHEMA="release-schema.yaml"
     if $USE_RELEASE_FILE ; then
+        RELEASE_SCHEMA="release-schema.yaml"
+        echo "INFO: Fetching schema $RELEASE_SCHEMA"
+        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
         verify_schema
     fi
     set_variables_maven
@@ -264,17 +356,29 @@ if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
     verify_version_match_release
     maven_release_file
 elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then
-    wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-container-schema.yaml
-    RELEASE_SCHEMA="release-container-schema.yaml"
-    verify_schema
+    if $USE_RELEASE_FILE ; then
+        RELEASE_SCHEMA="release-container-schema.yaml"
+        echo "INFO: Fetching schema $RELEASE_SCHEMA"
+        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
+        verify_schema
+    fi
     set_variables_container
     verify_version
     container_release_file
+elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then
+    if $USE_RELEASE_FILE ; then
+        RELEASE_SCHEMA="release-pypi-schema.yaml"
+        echo "INFO: Fetching schema $RELEASE_SCHEMA"
+        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
+        verify_schema
+    fi
+    set_variables_pypi
+    verify_version
+    verify_pypi_match_release
+    pypi_release_file
 else
     echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
-    echo "Must be maven or container"
     exit 1
 fi
-##########################################
 
-echo "########### End Script release-job.sh ###################################"
+echo "---> release-job.sh ends"