From: Anil Belur Date: Thu, 28 Jan 2021 12:28:09 +0000 (+0000) Subject: Merge "Upgrade packer version 1.6.6" X-Git-Tag: v0.61.0~6 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=3b887734bb2bc6cf09b29cd39e6d99706ef57531;hp=350a6099cc560eaf1984db4d56154d9cec440577;p=releng%2Fglobal-jjb.git Merge "Upgrade packer version 1.6.6" --- diff --git a/.jjb-test/defaults.yaml b/.jjb-test/defaults.yaml index 34027eb9..781090cd 100644 --- a/.jjb-test/defaults.yaml +++ b/.jjb-test/defaults.yaml @@ -5,6 +5,9 @@ # General jenkins-ssh-credential: test-credential jenkins-ssh-release-credential: test-release-credential + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false # Gerrit Infra gerrit-server-name: test-server @@ -13,6 +16,7 @@ git-url: https://github.com git-clone-url: "git@github.com:" github-org: example-org + github-url: https://github.com/project github_pr_whitelist: - jpwku - tykeal diff --git a/.jjb-test/lf-release-jobs.yaml b/.jjb-test/lf-release-jobs.yaml index 6dc72bca..2a53f869 100644 --- a/.jjb-test/lf-release-jobs.yaml +++ b/.jjb-test/lf-release-jobs.yaml @@ -8,3 +8,13 @@ project-name: self-release-project branch: master mvn-settings: gerrit-maven-project-settings + +- project: + name: "{project-name}-release-github" + jobs: + - "{project-name}-github-release-jobs" + + project: self-release/project-github + project-name: self-release-project-github + branch: master + mvn-settings: github-maven-project-settings diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index dd8b811d..f872808b 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -253,6 +253,15 @@ The JSON schema for a PackageCloud release file appears below. :language: yaml +Job Groups +========== + +Below is a list of Release job groups: + +.. literalinclude:: ../../jjb/lf-release-job-groups.yaml + :language: yaml + + Jenkins Jobs ------------ diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index e65b0487..14fb9fc0 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1572,6 +1572,14 @@ pattern: 'packer\/templates\/{templates}\.json' - compare-type: REG_EXP pattern: 'packer\/vars\/{platforms}\.json' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/provision\/{templates}\.yaml' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/provision\/local-{templates}\.yaml' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/templates\/{templates}\.json' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/vars\/{platforms}\.json' - job-template: name: "{project-name}-packer-merge-{platforms}-{templates}" @@ -1629,6 +1637,11 @@ - 'packer\/provision\/local-{templates}\.yaml' - 'packer\/templates\/{templates}\.json' - 'packer\/vars\/{platforms}\.json' + - 'packer\/common-packer\/provision\/{templates}\.yaml' + - 'packer\/common-packer\/provision\/local-{templates}\.yaml' + - 'packer\/common-packer\/templates\/{templates}\.json' + - 'packer\/common-packer\/vars\/{platforms}\.json' + white-list-target-branches: - "{branch}" @@ -1689,6 +1702,14 @@ pattern: 'packer\/templates\/{templates}\.json' - compare-type: REG_EXP pattern: 'packer\/vars\/{platforms}\.json' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/provision\/{templates}\.yaml' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/provision\/local-{templates}\.yaml' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/templates\/{templates}\.json' + - compare-type: REG_EXP + pattern: 'packer\/common-packer\/vars\/{platforms}\.json' - job-template: name: "{project-name}-packer-verify-build-{platforms}-{templates}" @@ -1737,6 +1758,10 @@ - 'packer\/provision\/local-{templates}\.yaml' - 'packer\/templates\/{templates}\.json' - 'packer\/vars\/{platforms}\.json' + - 'packer\/common-packer\/vars\/{platforms}\.json' + - 'packer\/common-packer\/provision\/{templates}\.yaml' + - 'packer\/common-packer\/provision\/local-{templates}\.yaml' + - 'packer\/common-packer\/templates\/{templates}\.json' ################# # Packer Verify # diff --git a/jjb/lf-release-job-groups.yaml b/jjb/lf-release-job-groups.yaml index a1451b0d..b2d09d6e 100644 --- a/jjb/lf-release-job-groups.yaml +++ b/jjb/lf-release-job-groups.yaml @@ -8,3 +8,13 @@ jobs: - gerrit-release-verify - gerrit-release-merge + +- job-group: + name: "{project-name}-github-release-jobs" + + # This job group contains all the recommended jobs that should be deployed + # for any project using self releases. + + jobs: + - github-release-verify + - github-release-merge diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml index 3cc37e9b..e1bf0684 100644 --- a/jjb/lf-release-jobs.yaml +++ b/jjb/lf-release-jobs.yaml @@ -56,6 +56,9 @@ # yamllint disable-line rule:line-length comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + github_included_regions: + - '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + ##################### # Job Configuration # ##################### @@ -138,6 +141,46 @@ unstable: "{gerrit-skip-vote}" notbuilt: "{gerrit-skip-vote}" +- job-template: + name: "{project-name}-release-verify" + id: github-release-verify + <<: *lf_release_common + # yamllint disable-line rule:key-duplicates + <<: *lf_release_verify + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: ^(recheck|reverify)$ + only-trigger-phrase: false + status-context: "Github Release Verify" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + ################# # RELEASE MERGE # ################# @@ -160,6 +203,9 @@ - comment-added-contains-event: comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' + github_included_regions: + - '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + ##################### # Job Configuration # ##################### @@ -248,6 +294,46 @@ - compare-type: REG_EXP pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' +- job-template: + name: "{project-name}-release-merge" + id: github-release-merge + <<: *lf_release_common + # yamllint disable-line rule:key-duplicates + <<: *lf_release_merge + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^remerge$" + only-trigger-phrase: true + status-context: "Github Release Merge" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + ################ # PyPI RELEASE # ################ diff --git a/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml b/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml new file mode 100644 index 00000000..b9fbf172 --- /dev/null +++ b/releasenotes/notes/add-regex-to-trigger-common-packer-updates-6ca049d422a00f67.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Add regex to trigger packer jobs when common-packer templates are updated. diff --git a/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml b/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml new file mode 100644 index 00000000..552a63cc --- /dev/null +++ b/releasenotes/notes/add-self-release-github-jobs-263bd060310209ea.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Add self release verify and merge jobs for GitHub based projects. diff --git a/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml b/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml new file mode 100644 index 00000000..6af989be --- /dev/null +++ b/releasenotes/notes/fix-job-cost-18fd129b9713c5dd.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an bug with 'job-cost.sh' that would cause builds to be marked + as unstable if not run on AWS or OpenStack. diff --git a/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml b/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml new file mode 100644 index 00000000..dfb1b71c --- /dev/null +++ b/releasenotes/notes/fix-release-job-script-edeb0473e2a9c7cb.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix the release job script to handle LOG_DIR unbound variable + and condition to check if the LOGS_SERVER or CDN_URL is being used. diff --git a/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml b/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml new file mode 100644 index 00000000..75ff50d3 --- /dev/null +++ b/releasenotes/notes/remove-python-27-support-1e109eab35756c0b.yaml @@ -0,0 +1,15 @@ +--- +fixes: + - | + Remove python 2.7 support + + As per the deprecation notice python 2.7 is not long supported. + This causing job failures since the dependencies install + are not maintained. + + DEPRECATION: Python 2.7 reached the end of its life on January 1st, + 2020. Please upgrade your Python as Python 2.7 is no longer + maintained. pip 21.0 will drop support for Python 2.7 in January + 2021. More details about Python 2 support in pip can be found at + https://pip.pypa.io/en/latest/development/release-process/#python-2-support + pip 21.0 will remove support for this functionality. diff --git a/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml b/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml new file mode 100644 index 00000000..1a67dec9 --- /dev/null +++ b/releasenotes/notes/sudo-log-owner-03e7cf690e867482.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Updates the 'sudo-logs.sh' script to set ownerhsip to current build + user and user's login group, instead of the explicit + 'jenkins:jenkins'. This will allow sudoer log ownership to work on + builders not using 'jenkins' as their build username. diff --git a/shell/job-cost.sh b/shell/job-cost.sh index 595b342e..dde31298 100644 --- a/shell/job-cost.sh +++ b/shell/job-cost.sh @@ -15,6 +15,13 @@ set -euf -o pipefail # shellcheck disable=SC1090 source ~/lf-env.sh +if [[ ! -f /run/cloud-init/result.json && ! -f stack-cost ]]; then + # Don't attempt to calculate job cost as build is not running in a + # cloud environment + echo "INFO: Skipping job cost calculation" + exit 0 +fi + # AWS job cost not supported, exit cloudtype="$(jq -r .v1.datasource /run/cloud-init/result.json)" if [[ $cloudtype == "DataSourceEc2Local" ]]; then diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index bf9b12ee..062beb56 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -50,14 +50,6 @@ EOF fi fi - #Project may still want to use system py27 - #pip and setuptools must be updated for py27 to work. - python -m pip install --user --quiet --upgrade pip - python -m pip install --user --quiet --no-warn-script-location --upgrade setuptools - python --version - python -m pip --version - python -m pip freeze - python3 -m pip install --user --quiet --upgrade pip python3 -m pip install --user --quiet --no-warn-script-location --upgrade setuptools python3 -m pip install --user --quiet --no-warn-script-location --upgrade --upgrade-strategy eager -r "$requirements_file" diff --git a/shell/release-job.sh b/shell/release-job.sh index 7ac6fc58..9ff67994 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -25,7 +25,7 @@ python -m pip freeze set_variables_common(){ echo "INFO: Setting common variables" - if [[ -z ${LOGS_SERVER:-} ]] || [[ -z ${CDN_URL:-} ]]; then + if [[ -z ${LOGS_SERVER:-} ]] && [[ -z ${CDN_URL:-} ]]; then echo "ERROR: LOGS_SERVER or CDN_URL not defined" exit 1 fi @@ -48,6 +48,9 @@ set_variables_common(){ release_file="None" fi + if [[ -z ${LOG_DIR:-} ]]; then + LOG_DIR=$(yq -r ".log_dir" "$release_file") + fi if [[ -n ${LOGS_SERVER:-} ]]; then logs_url="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" elif [[ -n ${CDN_URL:-} ]]; then diff --git a/shell/sudo-logs.sh b/shell/sudo-logs.sh index 29e54df7..257ee23a 100755 --- a/shell/sudo-logs.sh +++ b/shell/sudo-logs.sh @@ -31,7 +31,7 @@ function copy_log() ;; esac sudo_log=$(basename "$sudo_log") - sudo chown jenkins:jenkins "/tmp/$sudo_log" + sudo chown "$(id -nu)": "/tmp/$sudo_log" chmod 0644 "/tmp/$sudo_log" mkdir -p "$WORKSPACE/archives/sudo" mv "/tmp/$sudo_log" "$WORKSPACE/archives/sudo/$sudo_log" diff --git a/tox.ini b/tox.ini index b9d28d05..3dfd4411 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ commands = lftools license check-dir -r '.+' shell [testenv:pre-commit] basepython = python3 deps = - pre-commit + pre-commit==2.9.3 commands = pre-commit run --all-files --show-diff-on-failure pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG