From: Bengt Thuree <bthuree@linuxfoundation.org>
Date: Fri, 5 Mar 2021 03:29:41 +0000 (+0000)
Subject: Merge "Update the create script with V3 flavors."
X-Git-Tag: v0.62.0~3
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=4d1aaad578d2bc185b281964c95980267300ecb2;hp=8745a8672c70d42e46b3ee37e11dcfa7c0c19daf;p=releng%2Fglobal-jjb.git

Merge "Update the create script with V3 flavors."
---

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3b8d1f13..7141eeda 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -16,7 +16,7 @@ repos:
       - id: gitlint
 
   - repo: https://github.com/jumanjihouse/pre-commit-hooks
-    rev: 2.1.1
+    rev: 2.1.5
     hooks:
       - id: shellcheck
 
diff --git a/releasenotes/notes/artifacts-self-releases-support-0448d5269ac8f30f.yaml b/releasenotes/notes/artifacts-self-releases-support-0448d5269ac8f30f.yaml
new file mode 100644
index 00000000..3f26ae28
--- /dev/null
+++ b/releasenotes/notes/artifacts-self-releases-support-0448d5269ac8f30f.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Add artifact distribution type for self releases.
+    This support allows customers to specify a particular name
+    and path of an artifact in Nexus which will be downloaded locally,
+    re-tagged and posted into the releases repository of the matching repo.
diff --git a/releasenotes/notes/deprecate-niet-release-jobs-64454a3317b7936f.yaml b/releasenotes/notes/deprecate-niet-release-jobs-64454a3317b7936f.yaml
index 1c0fa182..6129a503 100644
--- a/releasenotes/notes/deprecate-niet-release-jobs-64454a3317b7936f.yaml
+++ b/releasenotes/notes/deprecate-niet-release-jobs-64454a3317b7936f.yaml
@@ -1,5 +1,5 @@
 ---
-fixess:
+fixes:
   - |
     Standardize yaml parsing tools used across global-jjb.
     Switch from using niet package to yq for parsing yaml files
diff --git a/releasenotes/notes/fix-rtd-verify-9af795a7c4c6ecc6.yaml b/releasenotes/notes/fix-rtd-verify-9af795a7c4c6ecc6.yaml
new file mode 100644
index 00000000..31d8a491
--- /dev/null
+++ b/releasenotes/notes/fix-rtd-verify-9af795a7c4c6ecc6.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    The old version of rtd-verify is missing the "--init" flag, which causes it
+    to not add new submodules. Additionally, the "--recursive" flag has been
+    included to ensure proper loading of recursive submodules.
diff --git a/releasenotes/notes/instance-metadata-skip-f80ccdd0a6c696e3.yaml b/releasenotes/notes/instance-metadata-skip-f80ccdd0a6c696e3.yaml
new file mode 100644
index 00000000..6c0b0087
--- /dev/null
+++ b/releasenotes/notes/instance-metadata-skip-f80ccdd0a6c696e3.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Updates the 'capture-instance-metadata.sh' script to skip attempting
+    to capture instance metadata needed for job-cost.sh if the build is
+    being run on an unsupported cloud or platform.
diff --git a/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml b/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml
index aef5ced0..6adcc3e1 100644
--- a/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml
+++ b/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml
@@ -1,6 +1,7 @@
 ---
-feature: >
-  Global job that triggers on any docs changes.
-  Creates docs project if absent
-  Creates subproject association with master doc project
-  Triggers docs build
+features:
+  - |
+    Global job that triggers on any docs changes.
+    Creates docs project if absent
+    Creates subproject association with master doc project
+    Triggers docs build
diff --git a/releasenotes/notes/maven-deploy-87ad380c53d8e2c2.yaml b/releasenotes/notes/maven-deploy-87ad380c53d8e2c2.yaml
index e6167196..5229379d 100644
--- a/releasenotes/notes/maven-deploy-87ad380c53d8e2c2.yaml
+++ b/releasenotes/notes/maven-deploy-87ad380c53d8e2c2.yaml
@@ -13,7 +13,7 @@ fixes:
     lf-activate-venv is a more error resistant way to make sure a pip
     package is installed
 
-deprecates:
+deprecations:
   - |
     lftools_activate should no longer be used and will be removed
     in a future release.
diff --git a/releasenotes/notes/pip-cryptography-module-3.3.2-40839530eb33b88d.yaml b/releasenotes/notes/pip-cryptography-module-3.3.2-40839530eb33b88d.yaml
new file mode 100644
index 00000000..885abb38
--- /dev/null
+++ b/releasenotes/notes/pip-cryptography-module-3.3.2-40839530eb33b88d.yaml
@@ -0,0 +1,24 @@
+---
+fixes:
+  - |
+    Pin cryptography to 3.3.2
+
+    The latest version of module breaks compatibility with the latest version
+    of pip.
+
+
+    Error:
+
+    .. code-block:: bash
+
+       Traceback (most recent call last):
+         File "<string>", line 1, in <module>
+         File "/tmp/pip-build-vqk6fya9/cryptography/setup.py", line 14,
+         in <module> from setuptools_rust import RustExtension
+       ModuleNotFoundError: No module named 'setuptools_rust'
+
+
+    Reference:
+
+       `PYCA#5753 <https://github.com/pyca/cryptography/issues/5753>`
+       `PYCA#5771 <https://github.com/pyca/cryptography/issues/5771>`
diff --git a/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml
index 7d6bb561..b2f6cca7 100644
--- a/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml
+++ b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml
@@ -1,5 +1,5 @@
 ---
-fix:
+fixes:
   - |
     Builders may have diffrent pyenv versions installed.
     Programically pick the latest pyenv version.
diff --git a/releasenotes/notes/schema-artifact-releases-9f7503eff53a7d29.yaml b/releasenotes/notes/schema-artifact-releases-9f7503eff53a7d29.yaml
new file mode 100644
index 00000000..867cf4dc
--- /dev/null
+++ b/releasenotes/notes/schema-artifact-releases-9f7503eff53a7d29.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Enable support for artifact type releases. Add initial schema for
+    artifact release verification.
diff --git a/releasenotes/notes/self-releases-reorder-7b9c9b12c9bd5f26.yaml b/releasenotes/notes/self-releases-reorder-7b9c9b12c9bd5f26.yaml
new file mode 100644
index 00000000..a465989b
--- /dev/null
+++ b/releasenotes/notes/self-releases-reorder-7b9c9b12c9bd5f26.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Reorder functions and add function labels to make release-job.sh
+    easier to read.
diff --git a/schema/release-artifact-schema.yaml b/schema/release-artifact-schema.yaml
new file mode 100644
index 00000000..e9cc0852
--- /dev/null
+++ b/schema/release-artifact-schema.yaml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2021 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+---
+$schema: "http://json-schema.org/schema#"
+$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-artifact-schema.yaml"
+
+required:
+  - "distribution_type"
+  - "project"
+  - "version"
+  - "artifacts"
+
+properties:
+  artifacts:
+    type: "array"
+    properties:
+      name:
+        type: "string"
+      path:
+        type: "string"
+    additionalProperties: false
+  distribution_type:
+    type: "string"
+  git_tag:
+    type: "string"
+  project:
+    type: "string"
+  tag_release:
+    type: "boolean"
+  version:
+    type: "string"
diff --git a/shell/capture-instance-metadata.sh b/shell/capture-instance-metadata.sh
index a5914ad9..c102d53c 100644
--- a/shell/capture-instance-metadata.sh
+++ b/shell/capture-instance-metadata.sh
@@ -17,6 +17,11 @@ if [ -n "${NOMAD_DC}" ]; then
     exit 0
 fi
 
+if [[ ! -f /run/cloud-init/result.json ]]; then
+    echo "INFO: Running in unsupported cloud, no metadata"
+    exit 0
+fi
+
 # AWS not supported, exit
 cloudtype="$(jq -r .v1.datasource /run/cloud-init/result.json)"
 if [[ $cloudtype == "DataSourceEc2Local" ]]; then
diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh
index 062beb56..59783053 100644
--- a/shell/python-tools-install.sh
+++ b/shell/python-tools-install.sh
@@ -29,6 +29,7 @@ python-openstackclient
 python-magnumclient
 kubernetes
 niet~=1.4.2
+cryptography<3.4
 yq
 
 # PINNED INDIRECT DEPENDENCIES
diff --git a/shell/release-job.sh b/shell/release-job.sh
index 9ff67994..80521924 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -21,7 +21,31 @@ lf-activate-venv lftools pip idna==2.9 lftools jsonschema twine yq readline
 python -m pip --version
 python -m pip freeze
 
-#Functions.
+##########################
+# Set Variables Functions.
+##########################
+
+set_variables_artifact(){
+     echo "INFO: Setting artifact variables"
+     if [[ -z ${VERSION:-} ]]; then
+         VERSION=$(yq -r ".version" "$release_file")
+     fi
+     if [[ -z ${GIT_TAG:-} ]]; then
+         if grep -q "git_tag" "$release_file" ; then
+             GIT_TAG=$(yq -r ".git_tag" "$release_file")
+         else
+             GIT_TAG="$VERSION"
+         fi
+     fi
+     if [[ -z ${REF:-} ]]; then
+         REF=$(yq -r ".ref" "$release_file")
+     fi
+
+     printf "\t%-30s\n" RELEASE_ARTIFACT_INFO:
+     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: "$REF"
+     printf "\t%-30s %s\n" VERSION: "$VERSION"
+     printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+}
 
 set_variables_common(){
     echo "INFO: Setting common variables"
@@ -83,7 +107,7 @@ set_variables_common(){
     # Displaying Release Information (Common variables)
     printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO:
     printf "\t%-30s %s\n" RELEASE_FILE: "$release_file"
-    printf "\t%-30s %s\n" LOGS_SERVER: "$LOGS_SERVER"
+    printf "\t%-30s %s\n" LOGS_SERVER: "${LOGS_SERVER:-None}"
     printf "\t%-30s %s\n" CDN_URL: "${CDN_URL:-None}"
     printf "\t%-30s %s\n" NEXUS_PATH: "$NEXUS_PATH"
     printf "\t%-30s %s\n" JENKINS_HOSTNAME: "$JENKINS_HOSTNAME"
@@ -95,30 +119,6 @@ set_variables_common(){
     printf "\t%-30s %s\n" OVERRIDE_SEMVER_REGEX: "${OVERRIDE_SEMVER_REGEX:-None}"
 }
 
-set_variables_maven(){
-    echo "INFO: Setting maven variables"
-    if [[ -z ${VERSION:-} ]]; then
-        VERSION=$(yq -r ".version" "$release_file")
-    fi
-    if [[ -z ${GIT_TAG:-} ]]; then
-        if grep -q "git_tag" "$release_file" ; then
-            GIT_TAG=$(yq -r ".git_tag" "$release_file")
-        else
-            GIT_TAG="$VERSION"
-        fi
-    fi
-    if [[ -z ${LOG_DIR:-} ]]; then
-        LOG_DIR=$(yq -r ".log_dir" "$release_file")
-    fi
-
-    # Continuing displaying Release Information (Maven)
-    printf "\t%-30s\n" RELEASE_MAVEN_INFO:
-    printf "\t%-30s %s\n" VERSION: "$VERSION"
-    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
-    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
-    printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
-}
-
 set_variables_container(){
     echo "INFO: Setting container variables"
     if [[ -z ${VERSION:-} ]]; then
@@ -153,18 +153,8 @@ set_variables_container(){
     printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
 }
 
-set_variables_pypi(){
-    echo "INFO: Setting pypi variables"
-    if [[ -z ${LOG_DIR:-} ]]; then
-        LOG_DIR=$(yq -r ".log_dir" "$release_file")
-    fi
-
-    if [[ -z ${PYPI_PROJECT:-} ]]; then
-        PYPI_PROJECT=$(yq -r ".pypi_project" "$release_file")
-    fi
-    if [[ -z ${PYTHON_VERSION:-} ]]; then
-        PYTHON_VERSION=$(yq -r ".python_version" "$release_file")
-    fi
+set_variables_maven(){
+    echo "INFO: Setting maven variables"
     if [[ -z ${VERSION:-} ]]; then
         VERSION=$(yq -r ".version" "$release_file")
     fi
@@ -174,17 +164,17 @@ set_variables_pypi(){
         else
             GIT_TAG="$VERSION"
         fi
-   fi
+    fi
+    if [[ -z ${LOG_DIR:-} ]]; then
+        LOG_DIR=$(yq -r ".log_dir" "$release_file")
+    fi
 
-    # Continuing displaying Release Information (pypi)
-    printf "\t%-30s\n" RELEASE_PYPI_INFO:
-    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
-    printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
-    printf "\t%-30s %s\n" PYPI_INDEX: "$PYPI_INDEX" # from job configuration
-    printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
-    printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
+    # Continuing displaying Release Information (Maven)
+    printf "\t%-30s\n" RELEASE_MAVEN_INFO:
     printf "\t%-30s %s\n" VERSION: "$VERSION"
     printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
 }
 
 set_variables_packagecloud(){
@@ -217,13 +207,75 @@ set_variables_packagecloud(){
      printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
 }
 
+set_variables_pypi(){
+    echo "INFO: Setting pypi variables"
+    if [[ -z ${LOG_DIR:-} ]]; then
+        LOG_DIR=$(yq -r ".log_dir" "$release_file")
+    fi
+
+    if [[ -z ${PYPI_PROJECT:-} ]]; then
+        PYPI_PROJECT=$(yq -r ".pypi_project" "$release_file")
+    fi
+    if [[ -z ${PYTHON_VERSION:-} ]]; then
+        PYTHON_VERSION=$(yq -r ".python_version" "$release_file")
+    fi
+    if [[ -z ${VERSION:-} ]]; then
+        VERSION=$(yq -r ".version" "$release_file")
+    fi
+    if [[ -z ${GIT_TAG:-} ]]; then
+        if grep -q "git_tag" "$release_file" ; then
+            GIT_TAG=$(yq -r ".git_tag" "$release_file")
+        else
+            GIT_TAG="$VERSION"
+        fi
+   fi
+
+    # Continuing displaying Release Information (pypi)
+    printf "\t%-30s\n" RELEASE_PYPI_INFO:
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
+    printf "\t%-30s %s\n" PYPI_INDEX: "$PYPI_INDEX" # from job configuration
+    printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
+    printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
+    printf "\t%-30s %s\n" VERSION: "$VERSION"
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+}
+
+##########################
+# Verify Schema Functions.
+##########################
+
+verify_packagecloud_match_release(){
+    echo "INFO: Fetching console log from $logs_url"
+    wget -q -P /tmp "${logs_url}/"console.log.gz
+    echo "INFO: Searching for uploaded step, package name $PACKAGE_NAME and version $VERSION in job log"
+    if  zgrep -E "Pushing.*$PACKAGE_NAME.*$VERSION.*success\!" /tmp/console.log.gz; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+
+verify_pypi_match_release(){
+    echo "INFO: Fetching console log from $logs_url"
+    wget -q -P /tmp "${logs_url}/"console.log.gz
+    echo "INFO: Searching for uploaded step, project $PYPI_PROJECT and version $VERSION in job log"
+    # pypi-upload.sh generates success message with file list
+    if zgrep -i "uploaded" /tmp/console.log.gz | grep "$PYPI_PROJECT" | grep "$VERSION" ; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+
 verify_schema(){
     echo "INFO: Verifying $release_file against schema $release_schema"
     lftools schema verify "$release_file" "$release_schema"
 }
 
 verify_version(){
-
     # Override the regex for projects that do not follow https://semver.org
     OVERRIDE_SEMVER_REGEX="${OVERRIDE_SEMVER_REGEX:-None}"
     if [[ $OVERRIDE_SEMVER_REGEX == "None" ]]; then
@@ -260,31 +312,10 @@ verify_version_match_release(){
     fi
 }
 
-# check prerequisites to detect mistakes in the release YAML file
-verify_pypi_match_release(){
-    echo "INFO: Fetching console log from $logs_url"
-    wget -q -P /tmp "${logs_url}/"console.log.gz
-    echo "INFO: Searching for uploaded step, project $PYPI_PROJECT and version $VERSION in job log"
-    # pypi-upload.sh generates success message with file list
-    if zgrep -i "uploaded" /tmp/console.log.gz | grep "$PYPI_PROJECT" | grep "$VERSION" ; then
-        echo "INFO: found expected strings in job log"
-    else
-        echo "ERROR: failed to find expected strings in job log"
-        exit 1
-    fi
-}
 
-verify_packagecloud_match_release(){
-    echo "INFO: Fetching console log from $logs_url"
-    wget -q -P /tmp "${logs_url}/"console.log.gz
-    echo "INFO: Searching for uploaded step, package name $PACKAGE_NAME and version $VERSION in job log"
-    if  zgrep -E "Pushing.*$PACKAGE_NAME.*$VERSION.*success\!" /tmp/console.log.gz; then
-        echo "INFO: found expected strings in job log"
-    else
-        echo "ERROR: failed to find expected strings in job log"
-        exit 1
-    fi
-}
+#####################
+# Tag Repo Functions.
+#####################
 
 # sigul is only available on Centos
 # TODO: write tag-github-repo function
@@ -336,29 +367,40 @@ tag-git-repo(){
     fi
 }
 
-nexus_release(){
-    echo "INFO: Processing nexus release"
-    for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
-        # extract the domain name from URL
-        NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
-        echo "INFO: NEXUS_URL: $NEXUS_URL"
-        # extract the staging repo from URL
-        STAGING_REPO=${staging_url#*repositories/}
-        echo "INFO: Running Nexus Verify"
-        lftools nexus release -v --server https://"$NEXUS_URL" "$STAGING_REPO"
-        echo "INFO: Merge will run:"
-        echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO"
-    done
+###############################
+# Release Processing Functions.
+###############################
 
-    #Run the loop twice, to catch errors on either nexus repo
-    if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then
-        for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
-          NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
-          STAGING_REPO=${staging_url#*repositories/}
-          echo "INFO: Promoting $STAGING_REPO on $NEXUS_URL."
-          lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO"
-        done
-    fi
+artifact_release_file(){
+    echo "INFO: Processing artifact release"
+    mkdir artifacts
+    ORG=$(echo "$NEXUS_URL" | awk -F'.' '{print $2}')
+
+    for namequoted in $(yq '.artifacts[].name' $release_file); do
+        pathquoted=$(yq ".artifacts[] |select(.name==$namequoted) |.path" $release_file)
+
+        #Remove extra yaml quotes
+        name="${namequoted#\"}"
+        name="${name%\"}"
+        path="${pathquoted#\"}"
+        path="${path%\"}"
+
+        echo "$name"
+        echo "$path"
+        echo "INFO: Merge will post artifact: $name"
+        # Attempt to pull from releases to see if the artifact has been released.
+        if "${NEXUS_URL}"/content/repositories/releases/org/"${ORG}"/"${VERSION}"/"$name"; then
+            echo "INFO: $name is already released as version:$VERSION, Continuing..."
+        else
+            echo "INFO: $name not found in releases, release will be prepared. Continuing..."
+            wget "${path}"/"${name}" -o artifacts/"${name}"
+            if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then
+                #lftools sign sigul artifacts
+                curl -v -u <NEXUSUSER>:<NEXUSPASS> --upload-file "${NEXUS_URL}"/content/repositories/releases/org/"${ORG}"/"${VERSION}"/"${name}" \;
+            fi
+            echo "#########################"
+        fi
+    done
 }
 
 container_release_file(){
@@ -419,6 +461,61 @@ maven_release_file(){
     tag-git-repo
 }
 
+nexus_release(){
+    echo "INFO: Processing nexus release"
+    for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
+        # extract the domain name from URL
+        NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
+        echo "INFO: NEXUS_URL: $NEXUS_URL"
+        # extract the staging repo from URL
+        STAGING_REPO=${staging_url#*repositories/}
+        echo "INFO: Running Nexus Verify"
+        lftools nexus release -v --server https://"$NEXUS_URL" "$STAGING_REPO"
+        echo "INFO: Merge will run:"
+        echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO"
+    done
+
+    #Run the loop twice, to catch errors on either nexus repo
+    if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then
+        for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do
+          NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
+          STAGING_REPO=${staging_url#*repositories/}
+          echo "INFO: Promoting $STAGING_REPO on $NEXUS_URL."
+          lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO"
+        done
+    fi
+}
+
+packagecloud_promote(){
+    echo "INFO: Preparing to promote $1..."
+    promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
+        --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+        | yq -r .[].promote_url)"
+    echo "INFO: Promoting $1 from staging to release"
+    curl --netrc-file ~/packagecloud_api -X POST -F \
+        destination="$2/release" "$promote_url" \
+        | echo "INFO: Promoted package location: \
+        https://packagecloud.io$(yq -r .package_html_url)"
+    git checkout "$REF"
+    tag-git-repo
+}
+
+packagecloud_verify(){
+    echo "INFO: Verifying $1 exists in staging..."
+    if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
+        https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+        | yq -r .[].filename) ]]; then
+        echo "INFO: $1 exists in staging!"
+        echo "INFO: Existing package location: https://packagecloud.io$(curl \
+            --netrc-file ~/packagecloud_api --silent \
+            https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
+            | yq -r .[].package_html_url)"
+    else
+        echo "ERROR: $1 does not exist in staging"
+        exit 1
+    fi
+}
+
 # calls pip to download binary and source distributions from the specified index,
 # which requires a recent-in-2019 version.  Uploads the files it received.
 pypi_release_file(){
@@ -464,59 +561,30 @@ pypi_release_file(){
     tag-git-repo
 }
 
-packagecloud_verify(){
-    echo "INFO: Verifying $1 exists in staging..."
-    if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
-        https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
-        | yq -r .[].filename) ]]; then
-        echo "INFO: $1 exists in staging!"
-        echo "INFO: Existing package location: https://packagecloud.io$(curl \
-            --netrc-file ~/packagecloud_api --silent \
-            https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
-            | yq -r .[].package_html_url)"
-    else
-        echo "ERROR: $1 does not exist in staging"
-        exit 1
-    fi
-}
-
-packagecloud_promote(){
-    echo "INFO: Preparing to promote $1..."
-    promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
-        --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
-        | yq -r .[].promote_url)"
-    echo "INFO: Promoting $1 from staging to release"
-    curl --netrc-file ~/packagecloud_api -X POST -F \
-        destination="$2/release" "$promote_url" \
-        | echo "INFO: Promoted package location: \
-        https://packagecloud.io$(yq -r .package_html_url)"
-    git checkout "$REF"
-    tag-git-repo
-}
-
 ##############################  End Function Declarations  ################################
 
 # Set common environment variables
 set_variables_common
 
 # Determine the type of release:
+#   - artifact, release-artifact-schema.yaml
 #   - container, release-container-schema.yaml
 #   - maven, release-schema.yaml
+#   - packagecloud, release-packagecloud-schema.yaml
 #   - pypi,  release-pypi-schema.yaml
 
 case $DISTRIBUTION_TYPE in
 
-    maven)
+    artifact)
         if $USE_RELEASE_FILE ; then
-            release_schema="release-schema.yaml"
+            release_schema="release-artifact-schema.yaml"
             echo "INFO: Fetching schema $release_schema"
-            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
             verify_schema
         fi
-        set_variables_maven
+        set_variables_artifact
         verify_version
-        verify_version_match_release
-        maven_release_file
+        artifact_release_file
         ;;
 
     container)
@@ -531,17 +599,17 @@ case $DISTRIBUTION_TYPE in
         container_release_file
         ;;
 
-    pypi)
+    maven)
         if $USE_RELEASE_FILE ; then
-            release_schema="release-pypi-schema.yaml"
+            release_schema="release-schema.yaml"
             echo "INFO: Fetching schema $release_schema"
-            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema
             verify_schema
         fi
-        set_variables_pypi
+        set_variables_maven
         verify_version
-        verify_pypi_match_release
-        pypi_release_file
+        verify_version_match_release
+        maven_release_file
         ;;
 
     packagecloud)
@@ -563,6 +631,19 @@ case $DISTRIBUTION_TYPE in
         done
         ;;
 
+    pypi)
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-pypi-schema.yaml"
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_pypi
+        verify_version
+        verify_pypi_match_release
+        pypi_release_file
+        ;;
+
     *)
         echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
         exit 1
diff --git a/shell/rtd-verify.sh b/shell/rtd-verify.sh
index 29a991b5..e91dd71c 100644
--- a/shell/rtd-verify.sh
+++ b/shell/rtd-verify.sh
@@ -28,7 +28,7 @@ if [ "$GERRIT_PROJECT" != "$PROJECT" ]; then
 fi
 
 git fetch origin "$GERRIT_REFSPEC" && git checkout FETCH_HEAD
-git submodule update
+git submodule update --init --recursive
 for submod in $(git config -f .gitmodules --get-regexp '^submodule\.(.+)\.url' | awk -e '{print $1}'); do
     git config --get "$submod"
 done
diff --git a/shell/tox-install.sh b/shell/tox-install.sh
index 78716b05..02db0c27 100644
--- a/shell/tox-install.sh
+++ b/shell/tox-install.sh
@@ -14,23 +14,7 @@ echo "---> tox-install.sh"
 # Use -x to show value of $PYTHON in output
 set -eux -o pipefail
 
-#Python 3.5 and python2 tox in Ubuntu 16.04 workaround
-done="False"
-if [[ -f /etc/lsb-release ]]; then
-   # shellcheck disable=SC1091
-   source /etc/lsb-release
-   if [[ $DISTRIB_RELEASE == "16.04" ]]; then
-       echo "WARNING: Python projects should move to Ubuntu 18.04 to continue receiving support"
-       python2 -m pip install --user --quiet --upgrade tox tox-pyenv virtualenv more-itertools~=5.0.0
-       python3 -m pip install --user --quiet --upgrade tox tox-pyenv virtualenv zipp==1.1.0
-       done="True"
-   fi
-fi
-
-if [[ $done != "True" ]]; then
-        python -m pip install --user --quiet --upgrade tox tox-pyenv virtualenv zipp more-itertools~=5.0.0
-        python3 -m pip install --user --quiet --upgrade tox tox-pyenv virtualenv
-fi
+python3 -m pip install --user --quiet --upgrade tox tox-pyenv virtualenv
 
 # installs are silent, show version details in log
 $PYTHON --version
diff --git a/shell/tox-run.sh b/shell/tox-run.sh
index 91e040ad..68a5af94 100644
--- a/shell/tox-run.sh
+++ b/shell/tox-run.sh
@@ -13,6 +13,9 @@ echo "---> tox-run.sh"
 # do not use -o pipefail
 set -eux
 
+#Ensure that tox from tox-install.sh takes precedence.
+PATH=$HOME/.local/bin:$PATH
+
 ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox"
 ARCHIVE_DOC_DIR="$WORKSPACE/archives/docs"
 mkdir -p "$ARCHIVE_TOX_DIR"
@@ -27,6 +30,9 @@ if [[ -d /opt/pyenv ]]; then
     export TOX_TESTENV_PASSENV=PYTHONPATH
 fi
 
+#Useful debug
+tox --version
+
 PARALLEL="${PARALLEL:-true}"
 if [[ ${PARALLEL,,} = true ]]; then
     if [[ -n ${TOX_ENVS:-} ]]; then
diff --git a/tox.ini b/tox.ini
index 3dfd4411..0da55839 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,7 +52,7 @@ commands = lftools license check-dir -r '.+' shell
 [testenv:pre-commit]
 basepython = python3
 deps =
-    pre-commit==2.9.3
+    pre-commit
 commands =
     pre-commit run --all-files --show-diff-on-failure
     pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG
@@ -76,5 +76,5 @@ deps = python-openstackclient{env:OPENSTACK_VERSION_OP:~=}{env:OPENSTACK_VERSION
 commands = openstack {posargs:--help}
 
 [testenv:reno]
-deps = reno{env:RENO_VERSION_OP:~=}{env:RENO_VERSION:2.10.0}
+deps = reno{env:RENO_VERSION_OP:~=}{env:RENO_VERSION:2.11.2}
 commands = reno {posargs:--help}