Merge "Upgrade packer version 1.6.6"

diff --git a/.jjb-test/defaults.yaml b/.jjb-test/defaults.yaml
index 34027eb..781090c 100644 (file)
--- a/.jjb-test/defaults.yaml
+++ b/.jjb-test/defaults.yaml
@@ -5,6 +5,9 @@
     # General
     jenkins-ssh-credential: test-credential
     jenkins-ssh-release-credential: test-release-credential
+    submodule-recursive: true
+    submodule-timeout: 10
+    submodule-disable: false
 
     # Gerrit Infra
     gerrit-server-name: test-server
@@ -13,6 +16,7 @@
     git-url: https://github.com
     git-clone-url: "git@github.com:"
     github-org: example-org
+    github-url: https://github.com/project
     github_pr_whitelist:
       - jpwku
       - tykeal

diff --git a/.jjb-test/lf-release-jobs.yaml b/.jjb-test/lf-release-jobs.yaml
index 6dc72bc..2a53f86 100644 (file)
--- a/.jjb-test/lf-release-jobs.yaml
+++ b/.jjb-test/lf-release-jobs.yaml
@@ -8,3 +8,13 @@
     project-name: self-release-project
     branch: master
     mvn-settings: gerrit-maven-project-settings
+
+- project:
+    name: "{project-name}-release-github"
+    jobs:
+      - "{project-name}-github-release-jobs"
+
+    project: self-release/project-github
+    project-name: self-release-project-github
+    branch: master
+    mvn-settings: github-maven-project-settings

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index dd8b811..f872808 100644 (file)
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -253,6 +253,15 @@ The JSON schema for a PackageCloud release file appears below.
    :language: yaml
 
 
+Job Groups
+==========
+
+Below is a list of Release job groups:
+
+.. literalinclude:: ../../jjb/lf-release-job-groups.yaml
+   :language: yaml
+
+
 Jenkins Jobs
 ------------
 

diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml
index e65b048..14fb9fc 100644 (file)
--- a/jjb/lf-ci-jobs.yaml
+++ b/jjb/lf-ci-jobs.yaml
@@ -1572,6 +1572,14 @@
                   pattern: 'packer\/templates\/{templates}\.json'
                 - compare-type: REG_EXP
                   pattern: 'packer\/vars\/{platforms}\.json'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/provision\/{templates}\.yaml'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/provision\/local-{templates}\.yaml'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/templates\/{templates}\.json'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/vars\/{platforms}\.json'
 
 - job-template:
     name: "{project-name}-packer-merge-{platforms}-{templates}"
@@ -1629,6 +1637,11 @@
             - 'packer\/provision\/local-{templates}\.yaml'
             - 'packer\/templates\/{templates}\.json'
             - 'packer\/vars\/{platforms}\.json'
+            - 'packer\/common-packer\/provision\/{templates}\.yaml'
+            - 'packer\/common-packer\/provision\/local-{templates}\.yaml'
+            - 'packer\/common-packer\/templates\/{templates}\.json'
+            - 'packer\/common-packer\/vars\/{platforms}\.json'
+
           white-list-target-branches:
             - "{branch}"
 
@@ -1689,6 +1702,14 @@
                   pattern: 'packer\/templates\/{templates}\.json'
                 - compare-type: REG_EXP
                   pattern: 'packer\/vars\/{platforms}\.json'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/provision\/{templates}\.yaml'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/provision\/local-{templates}\.yaml'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/templates\/{templates}\.json'
+                - compare-type: REG_EXP
+                  pattern: 'packer\/common-packer\/vars\/{platforms}\.json'
 
 - job-template:
     name: "{project-name}-packer-verify-build-{platforms}-{templates}"
@@ -1737,6 +1758,10 @@
             - 'packer\/provision\/local-{templates}\.yaml'
             - 'packer\/templates\/{templates}\.json'
             - 'packer\/vars\/{platforms}\.json'
+            - 'packer\/common-packer\/vars\/{platforms}\.json'
+            - 'packer\/common-packer\/provision\/{templates}\.yaml'
+            - 'packer\/common-packer\/provision\/local-{templates}\.yaml'
+            - 'packer\/common-packer\/templates\/{templates}\.json'
 
 #################
 # Packer Verify #

diff --git a/jjb/lf-release-job-groups.yaml b/jjb/lf-release-job-groups.yaml
index a1451b0..b2d09d6 100644 (file)
--- a/jjb/lf-release-job-groups.yaml
+++ b/jjb/lf-release-job-groups.yaml
@@ -8,3 +8,13 @@
     jobs:
       - gerrit-release-verify
       - gerrit-release-merge
+
+- job-group:
+    name: "{project-name}-github-release-jobs"
+
+    # This job group contains all the recommended jobs that should be deployed
+    # for any project using self releases.
+
+    jobs:
+      - github-release-verify
+      - github-release-merge

diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 3cc37e9..e1bf068 100644 (file)
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -56,6 +56,9 @@
           # yamllint disable-line rule:line-length
           comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
 
+    github_included_regions:
+      - '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+
     #####################
     # Job Configuration #
     #####################
@@ -138,6 +141,46 @@
             unstable: "{gerrit-skip-vote}"
             notbuilt: "{gerrit-skip-vote}"
 
+- job-template:
+    name: "{project-name}-release-verify"
+    id: github-release-verify
+    <<: *lf_release_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_release_verify
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+      - github:
+          url: "{github-url}/{github-org}/{project}"
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: "{build-timeout}"
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    scm:
+      - lf-infra-github-scm:
+          url: "{git-clone-url}{github-org}/{project}"
+          refspec: "+refs/pull/*:refs/remotes/origin/pr/*"
+          branch: "$sha1"
+          submodule-recursive: "{submodule-recursive}"
+          submodule-timeout: "{submodule-timeout}"
+          submodule-disable: "{submodule-disable}"
+          choosing-strategy: default
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    triggers:
+      - github-pull-request:
+          trigger-phrase: ^(recheck|reverify)$
+          only-trigger-phrase: false
+          status-context: "Github Release Verify"
+          permit-all: true
+          github-hooks: true
+          white-list-target-branches:
+            - "{branch}"
+          included-regions: "{obj:github_included_regions}"
+
 #################
 # RELEASE MERGE #
 #################
@@ -160,6 +203,9 @@
       - comment-added-contains-event:
           comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
 
+    github_included_regions:
+      - '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+
     #####################
     # Job Configuration #
     #####################
@@ -248,6 +294,46 @@
                 - compare-type: REG_EXP
                   pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
 
+- job-template:
+    name: "{project-name}-release-merge"
+    id: github-release-merge
+    <<: *lf_release_common
+    # yamllint disable-line rule:key-duplicates
+    <<: *lf_release_merge
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: "{build-days-to-keep}"
+      - github:
+          url: "{github-url}/{github-org}/{project}"
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: "{build-timeout}"
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    scm:
+      - lf-infra-github-scm:
+          url: "{git-clone-url}{github-org}/{project}"
+          refspec: "+refs/pull/*:refs/remotes/origin/pr/*"
+          branch: "$sha1"
+          submodule-recursive: "{submodule-recursive}"
+          submodule-timeout: "{submodule-timeout}"
+          submodule-disable: "{submodule-disable}"
+          choosing-strategy: default
+          jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+    triggers:
+      - github-pull-request:
+          trigger-phrase: "^remerge$"
+          only-trigger-phrase: true
+          status-context: "Github Release Merge"
+          permit-all: true
+          github-hooks: true
+          white-list-target-branches:
+            - "{branch}"
+          included-regions: "{obj:github_included_regions}"
+
 ################
 # PyPI RELEASE #
 ################

diff --git a/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml b/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml
new file mode 100644 (file)
index 0000000..b9fbf17
--- /dev/null
+++ b/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Add regex to trigger packer jobs when common-packer templates are updated.

diff --git a/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml b/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml
new file mode 100644 (file)
index 0000000..552a63c
--- /dev/null
+++ b/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Add self release verify and merge jobs for GitHub based projects.

diff --git a/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml b/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml
new file mode 100644 (file)
index 0000000..6af989b
--- /dev/null
+++ b/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes an bug with 'job-cost.sh' that would cause builds to be marked
+    as unstable if not run on AWS or OpenStack.

diff --git a/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml b/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml
new file mode 100644 (file)
index 0000000..dfb1b71
--- /dev/null
+++ b/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fix the release job script to handle LOG_DIR unbound variable
+    and condition to check if the LOGS_SERVER or CDN_URL is being used.

diff --git a/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml b/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml
new file mode 100644 (file)
index 0000000..75ff50d
--- /dev/null
+++ b/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml
@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Remove python 2.7 support
+
+    As per the deprecation notice python 2.7 is not long supported.
+    This causing job failures since the dependencies install
+    are not maintained.
+
+    DEPRECATION: Python 2.7 reached the end of its life on January 1st,
+    2020. Please upgrade your Python as Python 2.7 is no longer
+    maintained. pip 21.0 will drop support for Python 2.7 in January
+    2021. More details about Python 2 support in pip can be found at
+    https://pip.pypa.io/en/latest/development/release-process/#python-2-support
+    pip 21.0 will remove support for this functionality.

diff --git a/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml b/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml
new file mode 100644 (file)
index 0000000..1a67dec
--- /dev/null
+++ b/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml
@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Updates the 'sudo-logs.sh' script to set ownerhsip to current build
+    user and user's login group, instead of the explicit
+    'jenkins:jenkins'. This will allow sudoer log ownership to work on
+    builders not using 'jenkins' as their build username.

diff --git a/shell/job-cost.sh b/shell/job-cost.sh
index 595b342..dde3129 100644 (file)
--- a/shell/job-cost.sh
+++ b/shell/job-cost.sh
@@ -15,6 +15,13 @@ set -euf -o pipefail
 # shellcheck disable=SC1090
 source ~/lf-env.sh
 
+if [[ ! -f /run/cloud-init/result.json && ! -f stack-cost ]]; then
+    # Don't attempt to calculate job cost as build is not running in a
+    # cloud environment
+    echo "INFO: Skipping job cost calculation"
+    exit 0
+fi
+
 # AWS job cost not supported, exit
 cloudtype="$(jq -r .v1.datasource /run/cloud-init/result.json)"
 if [[ $cloudtype == "DataSourceEc2Local" ]]; then

diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh
index bf9b12e..062beb5 100644 (file)
--- a/shell/python-tools-install.sh
+++ b/shell/python-tools-install.sh
@@ -50,14 +50,6 @@ EOF
        fi
     fi
 
-    #Project may still want to use system py27
-    #pip and setuptools must be updated for py27 to work.
-    python -m pip install --user --quiet --upgrade pip
-    python -m pip install --user --quiet --no-warn-script-location --upgrade setuptools
-    python --version
-    python -m pip --version
-    python -m pip freeze
-
     python3 -m pip install --user --quiet --upgrade pip
     python3 -m pip install --user --quiet --no-warn-script-location --upgrade setuptools
     python3 -m pip install --user --quiet --no-warn-script-location --upgrade --upgrade-strategy eager -r "$requirements_file"

diff --git a/shell/release-job.sh b/shell/release-job.sh
index 7ac6fc5..9ff6799 100644 (file)
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -25,7 +25,7 @@ python -m pip freeze
 
 set_variables_common(){
     echo "INFO: Setting common variables"
-    if [[ -z ${LOGS_SERVER:-} ]] || [[ -z ${CDN_URL:-} ]]; then
+    if [[ -z ${LOGS_SERVER:-} ]] && [[ -z ${CDN_URL:-} ]]; then
         echo "ERROR: LOGS_SERVER or CDN_URL not defined"
         exit 1
     fi
@@ -48,6 +48,9 @@ set_variables_common(){
         release_file="None"
     fi
 
+    if [[ -z ${LOG_DIR:-} ]]; then
+        LOG_DIR=$(yq -r ".log_dir" "$release_file")
+    fi
     if [[ -n ${LOGS_SERVER:-} ]]; then
         logs_url="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     elif [[ -n ${CDN_URL:-} ]]; then

diff --git a/shell/sudo-logs.sh b/shell/sudo-logs.sh
index 29e54df..257ee23 100755 (executable)
--- a/shell/sudo-logs.sh
+++ b/shell/sudo-logs.sh
@@ -31,7 +31,7 @@ function copy_log()
             ;;
     esac
     sudo_log=$(basename "$sudo_log")
-    sudo chown jenkins:jenkins "/tmp/$sudo_log"
+    sudo chown "$(id -nu)": "/tmp/$sudo_log"
     chmod 0644 "/tmp/$sudo_log"
     mkdir -p "$WORKSPACE/archives/sudo"
     mv "/tmp/$sudo_log" "$WORKSPACE/archives/sudo/$sudo_log"

diff --git a/tox.ini b/tox.ini
index b9d28d0..3dfd441 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -52,7 +52,7 @@ commands = lftools license check-dir -r '.+' shell
 [testenv:pre-commit]
 basepython = python3
 deps =
-    pre-commit
+    pre-commit==2.9.3
 commands =
     pre-commit run --all-files --show-diff-on-failure
     pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG