From: Anil Belur <abelur@linuxfoundation.org>
Date: Wed, 10 Jul 2019 08:50:17 +0000 (+0000)
Subject: Merge "Add workaround for javadoc verify jobs"
X-Git-Tag: v0.40.0~19
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=e7d3f4f7f7c439bf2b1c09192b41256064224d8e;hp=3cd7e017758476461fab96164720dcb56f1e55ee;p=releng%2Fglobal-jjb.git

Merge "Add workaround for javadoc verify jobs"
---

diff --git a/docs/jjb/lf-info-vote.rst b/docs/jjb/lf-info-vote.rst
index 76c6d3de..64151f1f 100644
--- a/docs/jjb/lf-info-vote.rst
+++ b/docs/jjb/lf-info-vote.rst
@@ -7,7 +7,7 @@ INFO VOTE JOB
 Job counts the votes from the committers against a change
 to the INFO.yaml file
 
-If needed, will also check for a majority of tsc voters
+If needed, will also check for a majority of TSC voters
 (not yet implemented)
 
 Auto-merges the change on a majority vote.
diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst
index ab70a9fa..6772eeef 100644
--- a/docs/jjb/lf-macros.rst
+++ b/docs/jjb/lf-macros.rst
@@ -250,7 +250,7 @@ Calls shell script to validate votes on a change to an INFO.yaml
 lf-releases
 -----------
 
-realises lftools.ini (required)
+releases lftools.ini (required)
 needed to push to nexus.
 
 [nexus]
diff --git a/docs/jjb/lf-maven-jobs.rst b/docs/jjb/lf-maven-jobs.rst
index 85bebcc7..ad59153a 100644
--- a/docs/jjb/lf-maven-jobs.rst
+++ b/docs/jjb/lf-maven-jobs.rst
@@ -466,6 +466,8 @@ interest in that kind of support.
     :java-version: Version of Java to use for the build. (default: openjdk8)
     :mvn-global-settings: The name of the Maven global settings to use for
         Maven configuration. (default: global-settings)
+    :mvn-goals: The maven goals to perform for the build.
+        (default: clean install)
     :mvn-opts: Sets MAVEN_OPTS. (default: '')
     :mvn-params: Additional mvn parameters to pass to the cli. (default: '')
     :mvn-version: Version of maven to use. (default: mvn35)
diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index b3a78e00..85652356 100644
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -28,7 +28,7 @@ Macros
 ======
 
 lf-releases
-----------------------
+-----------
 
 Release verify and merge jobs are the same except for their scm, trigger, and
 builders definition. This anchor is the common template.
diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index 2545a6be..a70e8938 100644
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -1291,6 +1291,7 @@
     openstack-image-cleanup: true
     openstack-image-cleanup-age: 30
     openstack-image-protect: true
+    openstack-port-cleanup: true
     openstack-server-cleanup: true
     openstack-stack-cleanup: true
     openstack-volume-cleanup: true
@@ -1355,6 +1356,12 @@
           condition-expression: '{openstack-server-cleanup}'
           steps:
             - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-servers.sh
+      # Ports
+      - conditional-step:
+          condition-kind: boolean-expression
+          condition-expression: '{openstack-port-cleanup}'
+          steps:
+            - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-ports.sh
       # Volumes
       - conditional-step:
           condition-kind: boolean-expression
diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml
index f8982c06..e42dcdef 100644
--- a/jjb/lf-maven-jobs.yaml
+++ b/jjb/lf-maven-jobs.yaml
@@ -1106,6 +1106,7 @@
     github-url: 'https://github.com'
     java-version: openjdk8
     mvn-global-settings: global-settings
+    mvn-goals: clean install
     mvn-opts: ''
     mvn-params: ''
     mvn-version: mvn35
@@ -1177,6 +1178,7 @@
             - shell: echo 'Using SonarCloud'
             - lf-infra-maven-sonarcloud:
                 java-version: '{java-version}'
+                mvn-goals: '{mvn-goals}'
                 mvn-settings: '{mvn-settings}'
                 mvn-version: '{mvn-version}'
                 sonarcloud-project-key: '{sonarcloud-project-key}'
@@ -1192,6 +1194,7 @@
             - shell: echo 'Using SonarQube'
             - lf-infra-maven-sonar:
                 java-version: '{java-version}'
+                mvn-goals: '{mvn-goals}'
                 mvn-settings: '{mvn-settings}'
                 mvn-version: '{mvn-version}'
 
@@ -1210,7 +1213,9 @@
           java-version: '{java-version}'
       - inject:
           # Switch this to the sonar wrapper when JJB 2.0 is available
-          properties-content: SONAR_HOST_URL=$SONAR_URL
+          properties-content: |
+            SONAR_HOST_URL=$SONAR_URL
+            MAVEN_GOALS={mvn-goals}
       - lf-provide-maven-settings:
           global-settings-file: global-settings
           settings-file: '{mvn-settings}'
@@ -1233,6 +1238,7 @@
             SONAR_HOST_URL=https://sonarcloud.io
             PROJECT_KEY={sonarcloud-project-key}
             PROJECT_ORGANIZATION={sonarcloud-project-organization}
+            MAVEN_GOALS={mvn-goals}
             API_TOKEN={sonarcloud-api-token}
       - lf-provide-maven-settings:
           global-settings-file: global-settings
diff --git a/releasenotes/notes/configure-maven-sonar-goals-1b0018c932b2f9a1.yaml b/releasenotes/notes/configure-maven-sonar-goals-1b0018c932b2f9a1.yaml
new file mode 100644
index 00000000..642e3559
--- /dev/null
+++ b/releasenotes/notes/configure-maven-sonar-goals-1b0018c932b2f9a1.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Allow maven goals to be configured in maven-sonar.sh.
+    Set to "clean install" by default.
diff --git a/releasenotes/notes/openstack-cleanup-orphaned-ports-2619454ce4d083e3.yaml b/releasenotes/notes/openstack-cleanup-orphaned-ports-2619454ce4d083e3.yaml
new file mode 100644
index 00000000..77c24c22
--- /dev/null
+++ b/releasenotes/notes/openstack-cleanup-orphaned-ports-2619454ce4d083e3.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Remove orphaned ports from the openstack cloud environment. These orphaned
+    ports are residue of CSIT jobs that needs to be purged, as a part of the
+    openstack cron job. A large number of stale jobs could cause IP address
+    allocation failures.
diff --git a/releasenotes/notes/openstack-stack-del-with-force-b50cadbacc49d516.yaml b/releasenotes/notes/openstack-stack-del-with-force-b50cadbacc49d516.yaml
new file mode 100644
index 00000000..31a973b1
--- /dev/null
+++ b/releasenotes/notes/openstack-stack-del-with-force-b50cadbacc49d516.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Delete stacks with the ``--force`` option to ensure that any delete
+    failures does not stop the openstack-cron jobs from continuing.
diff --git a/releasenotes/notes/standardize-nexus-proxy-release-job-9818485ddbfca867.yaml b/releasenotes/notes/standardize-nexus-proxy-release-job-9818485ddbfca867.yaml
new file mode 100644
index 00000000..ef9321fe
--- /dev/null
+++ b/releasenotes/notes/standardize-nexus-proxy-release-job-9818485ddbfca867.yaml
@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    Projects using lf-release-jobs need to make sure they have the global
+    variable NEXUSPROXY added in Jenkins production and Jenkins sandbox
+    servers. The value of this variable should be the URL to the project's
+    Nexus server.
+fixes:
+  - |
+    Avoid the usage of project specific variables. Do not use ODLNEXUSPROXY
+    var, but instead use a generalized variable.
diff --git a/shell/check-info-votes.sh b/shell/check-info-votes.sh
index 1e02a386..51ca172f 100644
--- a/shell/check-info-votes.sh
+++ b/shell/check-info-votes.sh
@@ -16,17 +16,17 @@ pip="pip3"
 
 # For OPNFV
 if [[ $NODE_NAME =~ "lf-build" ]]; then
-  pip=pip
+    pip=pip
 fi
 
 if [ -d "/opt/pyenv" ]; then
-  echo "---> Setting up pyenv"
-  export PYENV_ROOT="/opt/pyenv"
-  export PATH="$PYENV_ROOT/bin:$PATH"
-  PYTHONPATH=$(pwd)
-  export PYTHONPATH
-  pyenv local 3.6.4
-  export PYENV_VERSION="3.6.4"
+    echo "---> Setting up pyenv"
+    export PYENV_ROOT="/opt/pyenv"
+    export PATH="$PYENV_ROOT/bin:$PATH"
+    PYTHONPATH=$(pwd)
+    export PYTHONPATH
+    pyenv local 3.6.4
+    export PYENV_VERSION="3.6.4"
 fi
 
 $pip install --user niet
@@ -39,12 +39,12 @@ lftools infofile check-votes INFO.yaml "$GERRIT_URL" "$ref" > gerrit_comment.txt
 exit_status="$?"
 
 if [[ "$exit_status" -ne 0 ]]; then
-  echo "Vote not yet complete"
-  cat gerrit_comment.txt
-  exit "$exit_status"
+    echo "Vote not yet complete"
+    cat gerrit_comment.txt
+    exit "$exit_status"
 else
-  echo "Vote completed submitting review"
-  ssh -p "$GERRIT_PORT" "$USER"@"$GERRIT_HOST" gerrit review "$GERRIT_PATCHSET_REVISION" --verified 1
-  sleep 5
-  ssh -p "$GERRIT_PORT" "$USER"@"$GERRIT_HOST" gerrit review "$GERRIT_PATCHSET_REVISION" --submit
+    echo "Vote completed submitting review"
+    ssh -p "$GERRIT_PORT" "$USER"@"$GERRIT_HOST" gerrit review "$GERRIT_PATCHSET_REVISION" --verified 1
+    sleep 5
+    ssh -p "$GERRIT_PORT" "$USER"@"$GERRIT_HOST" gerrit review "$GERRIT_PATCHSET_REVISION" --submit
 fi
diff --git a/shell/maven-sonar.sh b/shell/maven-sonar.sh
index 706697b1..e78d156f 100644
--- a/shell/maven-sonar.sh
+++ b/shell/maven-sonar.sh
@@ -25,7 +25,7 @@ params+=("--settings $SETTINGS_FILE")
 
 # Disable SC2086 because we want to allow word splitting for $MAVEN_* parameters.
 # shellcheck disable=SC2086
-$MVN clean install \
+$MVN $MAVEN_GOALS \
     -e -Dsonar \
     ${params[*]} \
     $MAVEN_OPTIONS $MAVEN_PARAMS
diff --git a/shell/openstack-cleanup-orphaned-ports.sh b/shell/openstack-cleanup-orphaned-ports.sh
new file mode 100644
index 00000000..fac8d202
--- /dev/null
+++ b/shell/openstack-cleanup-orphaned-ports.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# Scans OpenStack for orphaned ports
+echo "---> Orphaned ports"
+
+os_cloud="${OS_CLOUD:-vex}"
+
+set -eux -o pipefail
+
+mapfile -t os_ports < <(openstack --os-cloud "$os_cloud" port list -f value -c ID -c status | egrep DOWN | awk '{print $1}')
+
+if [ ${#os_ports[@]} -eq 0 ]; then
+    echo "No orphaned ports found."
+else
+    for port in "${os_ports[@]}"; do
+        echo "Removing orphaned port $port"
+        openstack --os-cloud "$os_cloud" port delete "$port"
+    done
+fi
diff --git a/shell/openstack-cleanup-orphaned-stacks.sh b/shell/openstack-cleanup-orphaned-stacks.sh
index 8140adea..ccc29969 100644
--- a/shell/openstack-cleanup-orphaned-stacks.sh
+++ b/shell/openstack-cleanup-orphaned-stacks.sh
@@ -87,6 +87,6 @@ for STACK_NAME in "${OS_STACKS[@]}"; do
         continue
     else
         echo "Deleting orphaned stack: $STACK_NAME"
-        lftools openstack --os-cloud "$os_cloud" stack delete "$STACK_NAME"
+        lftools openstack --os-cloud "$os_cloud" stack delete --force "$STACK_NAME"
     fi
 done
diff --git a/shell/release-job.sh b/shell/release-job.sh
index a5ab1169..88b285c4 100644
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -8,12 +8,14 @@
 # which accompanies this distribution, and is available at
 # http://www.eclipse.org/legal/epl-v10.html
 ##############################################################################
+echo "---> release-job.sh"
+
 set -eu -o pipefail
 
 if [ -d "/opt/pyenv" ]; then
-  echo "---> Setting up pyenv"
-  export PYENV_ROOT="/opt/pyenv"
-  export PATH="$PYENV_ROOT/bin:$PATH"
+    echo "---> Setting up pyenv"
+    export PYENV_ROOT="/opt/pyenv"
+    export PATH="$PYENV_ROOT/bin:$PATH"
 fi
 PYTHONPATH=$(pwd)
 export PYTHONPATH
@@ -29,80 +31,80 @@ MAVEN_CENTRAL_URL="${MAVEN_CENTRAL_URL:-None}"
 
 #OPTIONAL
 if grep -q "\.maven_central_url" "$release_file"; then
-  MAVEN_CENTRAL_URL="$(niet ".maven_central_url" "$release_file")"
+    MAVEN_CENTRAL_URL="$(niet ".maven_central_url" "$release_file")"
 fi
 
 if [ "${LOGS_SERVER}" == 'None' ]; then
-  echo "FAILED: log server not found"
-  exit 1
+    echo "FAILED: log server not found"
+    exit 1
 fi
 
-NEXUS_URL="${ODLNEXUSPROXY:-$NEXUS_URL}"
+NEXUS_URL="${NEXUSPROXY:-$NEXUS_URL}"
 
 release_files=$(git diff HEAD^1 --name-only -- "releases/")
 echo "RELEASE FILES ARE AS FOLLOWS: $release_files"
 
 for release_file in $release_files; do
-  echo "This is the release file: $release_file"
-  echo "--> Verifying $release_file Schema."
-  echo "DUMMY CODE:"
-  #Make sure the schema check catches a missing trailing / on log_dir
-  #lftools schema is written, but not the schema file (yet)
-  echo "lftools schema verify [OPTIONS] $release_file $SCHEMAFILE"
-
-  VERSION="$(niet ".version" "$release_file")"
-  PROJECT="$(niet ".project" "$release_file")"
-  LOG_DIR="$(niet ".log_dir" "$release_file")"
-
-
-  NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
-  LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
-  PATCH_DIR="$(mktemp -d)"
-
-  pushd "$PATCH_DIR"
-    wget --quiet "${LOGS_URL}"staging-repo.txt.gz
-    STAGING_REPO="$(zcat staging-repo.txt)"
-
-    #INFO
-    echo "INFO:"
-    echo "RELEASE_FILE: $release_file"
-    echo "LOGS_SERVER: $LOGS_SERVER"
-    echo "NEXUS_URL: $NEXUS_URL"
-    echo "NEXUS_PATH: $NEXUS_PATH"
-    echo "ODLNEXUSPROXY: $ODLNEXUSPROXY"
-    echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME"
-    echo "SILO: $SILO"
-    echo "PROJECT: $PROJECT"
-    echo "STAGING_REPO: $STAGING_REPO"
-    echo "VERSION: $VERSION"
-    echo "PROJECT: $PROJECT"
-    echo "LOG DIR: $LOG_DIR"
-
-    wget --quiet  "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz}
-    gunzip taglist.log.gz
-    cat "$PATCH_DIR"/taglist.log
-  popd
-
-  git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")"
-  git fetch "$PATCH_DIR/$PROJECT.bundle"
-  git merge --ff-only FETCH_HEAD
-  git tag -am "$PROJECT $VERSION" "v$VERSION"
-  sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" v"$VERSION" < "$SIGUL_PASSWORD"
-  echo "Showing latest signature for $PROJECT:"
-  git log --show-signature -n1
-
-
-  ########## Merge Part ##############
-  if [[ "$JOB_NAME" =~ "merge" ]]; then
-    echo "Running merge"
-    git push origin "v$VERSION"
-    lftools nexus release --server "$NEXUS_URL" "$STAGING_REPO"
-    if [ "${MAVEN_CENTRAL_URL}" == 'None' ]; then
-      echo "No Maven central url specified, not pushing to maven central"
-    else
-      lftools nexus release --server "$MAVEN_CENTRAL_URL" "$STAGING_REPO"
+    echo "This is the release file: $release_file"
+    echo "--> Verifying $release_file Schema."
+    echo "DUMMY CODE:"
+    #Make sure the schema check catches a missing trailing / on log_dir
+    #lftools schema is written, but not the schema file (yet)
+    echo "lftools schema verify [OPTIONS] $release_file $SCHEMAFILE"
+
+    VERSION="$(niet ".version" "$release_file")"
+    PROJECT="$(niet ".project" "$release_file")"
+    LOG_DIR="$(niet ".log_dir" "$release_file")"
+
+
+    NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
+    LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+    PATCH_DIR="$(mktemp -d)"
+
+    pushd "$PATCH_DIR"
+        wget --quiet "${LOGS_URL}"staging-repo.txt.gz
+        STAGING_REPO="$(zcat staging-repo.txt)"
+
+        #INFO
+        echo "INFO:"
+        echo "RELEASE_FILE: $release_file"
+        echo "LOGS_SERVER: $LOGS_SERVER"
+        echo "NEXUS_URL: $NEXUS_URL"
+        echo "NEXUS_PATH: $NEXUS_PATH"
+        echo "ODLNEXUSPROXY: $ODLNEXUSPROXY"
+        echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME"
+        echo "SILO: $SILO"
+        echo "PROJECT: $PROJECT"
+        echo "STAGING_REPO: $STAGING_REPO"
+        echo "VERSION: $VERSION"
+        echo "PROJECT: $PROJECT"
+        echo "LOG DIR: $LOG_DIR"
+
+        wget --quiet  "${LOGS_URL}"/patches/{"${PROJECT}".bundle,taglist.log.gz}
+        gunzip taglist.log.gz
+        cat "$PATCH_DIR"/taglist.log
+    popd
+
+    git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")"
+    git fetch "$PATCH_DIR/$PROJECT.bundle"
+    git merge --ff-only FETCH_HEAD
+    git tag -am "$PROJECT $VERSION" "v$VERSION"
+    sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" v"$VERSION" < "$SIGUL_PASSWORD"
+    echo "Showing latest signature for $PROJECT:"
+    git log --show-signature -n1
+
+
+    ########## Merge Part ##############
+    if [[ "$JOB_NAME" =~ "merge" ]]; then
+        echo "Running merge"
+        git push origin "v$VERSION"
+        lftools nexus release --server "$NEXUS_URL" "$STAGING_REPO"
+        if [ "${MAVEN_CENTRAL_URL}" == 'None' ]; then
+            echo "No Maven central url specified, not pushing to maven central"
+        else
+            lftools nexus release --server "$MAVEN_CENTRAL_URL" "$STAGING_REPO"
+        fi
     fi
-  fi
 
 done
 echo "########### End Script release-job.sh ###################################"