From: Bengt Thuree Date: Thu, 14 Mar 2019 02:12:38 +0000 (+0000) Subject: Merge "Optimize the jjb-deploy trigger regex" X-Git-Tag: v0.34.0~8 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=2b811ee74f223040ff6071969566de72aed71b2a;hp=c3128e6979246c6dadb83be4a14b5179b5d53721;p=releng%2Fglobal-jjb.git Merge "Optimize the jjb-deploy trigger regex" --- diff --git a/.jjb-test/lf-macros/lf-maven-central-minimal.yaml b/.jjb-test/lf-macros/lf-maven-central-minimal.yaml index 5fb58b0e..2a2f6917 100644 --- a/.jjb-test/lf-macros/lf-maven-central-minimal.yaml +++ b/.jjb-test/lf-macros/lf-maven-central-minimal.yaml @@ -7,6 +7,8 @@ ##################### mvn-central: true + mvn-global-settings: '' + mvn-settings: '' ossrh-profile-id: '' ##################### @@ -16,4 +18,6 @@ builders: - lf-maven-central: mvn-central: '{mvn-central}' + mvn-global-settings: '{mvn-global-settings}' + mvn-settings: '{mvn-settings}' ossrh-profile-id: '{ossrh-profile-id}' diff --git a/docs/jjb/lf-c-cpp-jobs.rst b/docs/jjb/lf-c-cpp-jobs.rst index 17d45814..0e014916 100644 --- a/docs/jjb/lf-c-cpp-jobs.rst +++ b/docs/jjb/lf-c-cpp-jobs.rst @@ -5,6 +5,61 @@ C/C++ Jobs Job Templates ============= +CMake Sonar +----------- + +Sonar job which runs cmake && make then publishes to Sonar. + +This job purposely runs on the master branch as there are configuration needed +to support multi-branch. + +:Template Names: + + - {project-name}-cmake-sonar + - gerrit-cmake-sonar + - github-cmake-sonar + +:Comment Trigger: run-sonar + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Configure in + defaults.yaml) + :sonar-scanner-version: Version of sonar-scanner to install. + :sonarcloud-project-key: SonarCloud project key. + :sonarcloud-project-organization: SonarCloud project organization. + :sonarcloud-api-token: SonarCloud API Token. + +:Optional parameters: + + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 60) + :cmake-opts: Parameters to pass to cmake. (default: '') + :cron: Cron schedule when to trigger the job. This parameter also + supports multiline input via YAML pipe | character in cases where + one may want to provide more than 1 cron timer. (default: '@daily') + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :install-prefix: CMAKE_INSTALL_PREFIX to use for install. + (default: $BUILD_DIR/output) + + .. code-block:: bash + :caption: Example + + install-prefix: | + #!/bin/bash + echo "Hello World." + + :make-opts: Parameters to pass to make. (default: '') + :pre-build: Shell script to run before performing build. Useful for + setting up dependencies. (default: '') + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + + :gerrit_sonar_triggers: Override Gerrit Triggers. + CMake Stage ----------- diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 4d2d4f5f..94fcb1c5 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -180,6 +180,10 @@ repository which is to upload to OSSRH. :Required parameters: :mvn-central: Whether or not to upload to mvn-central. (true|false) + :mvn-global-settings: The name of the Maven global settings to use for + Maven configuration. (default: global-settings) + :mvn-settings: The name of settings file containing credentials for the + project. :ossrh-profile-id: Nexus staging profile ID as provided by OSSRH. .. literalinclude:: ../../.jjb-test/lf-macros/lf-maven-central-minimal.yaml @@ -240,6 +244,7 @@ Requires ``SIGUL_BRIDGE_IP`` configured as a global envvar. :Required Parameters: :sign-artifacts: Whether or not to sign artifacts with Sigul. :sign-dir: Directory to sign. + :sign-mode: serial|parallel lf-infra-provide-docker-cleanup ------------------------------- diff --git a/jenkins-init-scripts/create-jenkins-user.sh b/jenkins-init-scripts/create-jenkins-user.sh index 00052442..0983129f 100755 --- a/jenkins-init-scripts/create-jenkins-user.sh +++ b/jenkins-init-scripts/create-jenkins-user.sh @@ -29,10 +29,14 @@ if grep -q mock /etc/group; then usermod -a -G mock jenkins fi -mkdir /home/jenkins/.ssh /w +mkdir /home/jenkins/.ssh cp -r "/home/${OS}/.ssh/authorized_keys" /home/jenkins/.ssh/authorized_keys # Generate ssh key for use by Robot jobs echo -e 'y\n' | ssh-keygen -N "" -f /home/jenkins/.ssh/id_rsa -t rsa -chown -R jenkins:jenkins /home/jenkins/.ssh /w +chown -R jenkins:jenkins /home/jenkins/.ssh chmod 700 /home/jenkins/.ssh + +# The '/w' volume may already be part of image +[[ ! -d '/w' ]] && mkdir /w +chown -R jenkins:jenkins /w diff --git a/jenkins-init-scripts/create-swap-file.sh b/jenkins-init-scripts/create-swap-file.sh index cf8d97f1..6f077388 100755 --- a/jenkins-init-scripts/create-swap-file.sh +++ b/jenkins-init-scripts/create-swap-file.sh @@ -9,7 +9,22 @@ # http://www.eclipse.org/legal/epl-v10.html ############################################################################## -dd if=/dev/zero of=/swap count=1024 bs=1MiB +# Get the blockCount from the 'SWAP_SIZE' environmental variable +blockCount=${SWAP_SIZE-''} + +# Validate SWAP_SIZE +# Empty: Set blockCount 1 +# Zero: No Swap +# Integer: Set blockCount +# Else: No Swap +case $blockCount in + '') blockCount=1 ;; + [0-9]*) blockCount=$blockCount ;; + *) exit ;; +esac +[[ $blockCount == 0 ]] && exit + +dd if=/dev/zero of=/swap count="${blockCount}k" bs=1MiB chmod 600 /swap mkswap /swap swapon /swap diff --git a/jjb/lf-c-cpp-jobs.yaml b/jjb/lf-c-cpp-jobs.yaml index 07e5ec4f..2eb394ee 100644 --- a/jjb/lf-c-cpp-jobs.yaml +++ b/jjb/lf-c-cpp-jobs.yaml @@ -1,4 +1,136 @@ --- +############### +# CMAKE Sonar # +############### + +- lf_cmake_sonar: &lf_cmake_sonar + name: lf-cmake-sonar + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-dir: '$WORKSPACE/target' + build-timeout: 15 + cmake-opts: '' + cron: '@daily' + disable-job: false + git-url: '$GIT_URL/$PROJECT' + github-url: 'https://github.com' + install-prefix: '$BUILD_DIR/output' + make-opts: '' + pre-build: '' + sonar-scanner-version: 3.3.0.1492 + sonarcloud-api-token: '' + sonarcloud-organization: '' + sonarcloud-project-key: '' + submodule-recursive: true + submodule-timeout: 10 + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + disabled: '{disable-job}' + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + lftools-version: '{lftools-version}' + - lf-cmake-parameters: + build-dir: '{build-dir}' + cmake-opts: '{cmake-opts}' + install-prefix: '{install-prefix}' + make-opts: '{make-opts}' + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + builders: + - lf-infra-pre-build + - shell: '{pre-build}' + - inject: + # Switch this to the sonar wrapper when JJB 2.0 is available + properties-content: | + SONAR_HOST_URL=https://sonarcloud.io + SONAR_SCANNER_VERSION={sonar-scanner-version} + PROJECT_KEY={sonarcloud-project-key} + PROJECT_ORGANIZATION={sonarcloud-organization} + API_TOKEN={sonarcloud-api-token} + - shell: !include-raw-escape: ../shell/cmake-sonar.sh + - lf-provide-maven-settings-cleanup + + publishers: + - lf-infra-publish + +- job-template: + name: '{project-name}-cmake-sonar' + id: gerrit-cmake-sonar + concurrent: false + <<: *lf_cmake_sonar + + scm: + - lf-infra-gerrit-scm: + branch: '$GERRIT_BRANCH' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: gerrit + + triggers: + - timed: '{obj:cron}' + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - comment-added-contains-event: + comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(stage-release)$' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + +- job-template: + name: '{project-name}-cmake-sonar' + id: github-cmake-sonar + concurrent: false + <<: *lf_cmake_sonar + + properties: + - lf-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + - github: + url: '{github-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '$sha1' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - timed: '{obj:cron}' + + ############### # CMAKE STAGE # ############### diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 2b6cf10f..6b3faa85 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -275,9 +275,10 @@ - shell: !include-raw-escape: ../shell/sigul-configuration.sh - shell: !include-raw-escape: ../shell/sigul-install.sh - inject: - properties-content: SIGN_DIR={sign-dir} + properties-content: | + SIGN_DIR={sign-dir} + SIGN_MODE={sign-mode} - shell: !include-raw-escape: - - ../shell/common-variables.sh - ../shell/sigul-sign-dir.sh - shell: !include-raw-escape: ../shell/sigul-configuration-cleanup.sh diff --git a/jjb/lf-maven-jobs.yaml b/jjb/lf-maven-jobs.yaml index f82d6c6a..0c5abf69 100644 --- a/jjb/lf-maven-jobs.yaml +++ b/jjb/lf-maven-jobs.yaml @@ -679,6 +679,7 @@ mvn-version: mvn35 ossrh-profile-id: '' sign-artifacts: false + sign-mode: serial stream: master submodule-recursive: true submodule-timeout: 10 @@ -728,6 +729,7 @@ - lf-sigul-sign-dir: sign-artifacts: '{sign-artifacts}' sign-dir: '$WORKSPACE/m2repo' + sign-mode: '{sign-mode}' - lf-maven-stage: mvn-global-settings: '{mvn-global-settings}' mvn-settings: '{mvn-settings}' diff --git a/releasenotes/notes/custom-swap-size-864fc8ca8343d4af.yaml b/releasenotes/notes/custom-swap-size-864fc8ca8343d4af.yaml new file mode 100644 index 00000000..ab95bf92 --- /dev/null +++ b/releasenotes/notes/custom-swap-size-864fc8ca8343d4af.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + **jenkins-init-scripts** If the environmental variable 'SWAP_SIZE' is + set when the 'init.sh' script is called, then a 'SWAP_SIZE' GB swap + space will be configured. If 'SWAP_SIZE' is '0' or is not a valid + integer, then no swap space is configured. If it is unset then 1GB + of swap will be configured. Previously the swap size was fixed at + 1GB. diff --git a/releasenotes/notes/fix-lf-maven-central-macro-docs-ad43756dcd551c30.yaml b/releasenotes/notes/fix-lf-maven-central-macro-docs-ad43756dcd551c30.yaml new file mode 100644 index 00000000..5d3794fc --- /dev/null +++ b/releasenotes/notes/fix-lf-maven-central-macro-docs-ad43756dcd551c30.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Update the lf-maven-cental macro documentation and example templates with + the missing requireed params. diff --git a/releasenotes/notes/pre-existing-work-volume-05af4e8546facbea.yaml b/releasenotes/notes/pre-existing-work-volume-05af4e8546facbea.yaml new file mode 100644 index 00000000..df6ff344 --- /dev/null +++ b/releasenotes/notes/pre-existing-work-volume-05af4e8546facbea.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + **jenkins-init-scripts** If the work directory or volume (/w) aleady + exists, the ownership will be recursivly set to 'jenkins:jenkins'. + Previously only the top directory /w was owned by 'jenkins:jenkins' diff --git a/releasenotes/notes/sign-mode-2e46e7c5e0b93135.yaml b/releasenotes/notes/sign-mode-2e46e7c5e0b93135.yaml new file mode 100644 index 00000000..5d4fffd9 --- /dev/null +++ b/releasenotes/notes/sign-mode-2e46e7c5e0b93135.yaml @@ -0,0 +1,24 @@ +--- +features: + - | + **lf-sigul-sign-dir** macros now supports a ``sign-mode`` parameter which + allows jobs to choose to sign artifacts using either *parallel* mode + or *serial* mode (default). +upgrade: + - | + **lf-sigul-sign-dir** users need to add a new parameter ``sign-mode`` to + their job-templates setting either *parallel* or *serial* as the value, we + recommend setting serial mode for this setting. + + **{project-name}-maven-stage-{stream}**'s Sigul signer now defaults to + *serial* mode instead of the previous *parallel* behaviour. To change + this back to the previous behaviour pass the "sign-mode" parameter to the + job template: + + .. code-block:: yaml + + - project: + name: parallel-sign + jobs: + - gerrit-maven-stage: + sign-mode: parallel diff --git a/shell/cmake-sonar.sh b/shell/cmake-sonar.sh new file mode 100644 index 00000000..cd7ee715 --- /dev/null +++ b/shell/cmake-sonar.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> cmake-sonar.sh" + +build_dir="${BUILD_DIR:-$WORKSPACE/target}" +cmake_opts="${CMAKE_OPTS:-}" +make_opts="${MAKE_OPTS:-}" + +################ +# Script start # +################ + +set -ex -o pipefail + +cd /tmp || exit 1 +wget -O /tmp/sonar-scan.zip \ + "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip" +unzip sonar-scan.zip +sudo mv sonar-scanner-* /opt/sonar-scanner + +wget -O /tmp/bw.zip \ + "https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip" +unzip bw.zip +sudo mv build-wrapper-* /opt/build-wrapper + +mkdir -p "$build_dir" +cd "$build_dir" || exit 1 +# $cmake_opts needs to wordsplit to pass options. +# shellcheck disable=SC2086 +eval cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" $cmake_opts .. + +/opt/build-wrapper/build-wrapper-linux-x86-64 --out-dir "$WORKSPACE/bw-output" \ + make $make_opts + +/opt/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=${PROJECT_KEY} \ + -Dsonar.organization=${PROJECT_ORGANIZATION} \ + -Dsonar.sources=. \ + -Dsonar.cfamily.build-wrapper-output="$WORKSPACE/bw-output" \ + -Dsonar.host.url=${SONAR_HOST_URL} \ + -Dsonar.login=${API_TOKEN} diff --git a/shell/openstack-protect-in-use-images.sh b/shell/openstack-protect-in-use-images.sh index fe0bc0f1..9152348e 100644 --- a/shell/openstack-protect-in-use-images.sh +++ b/shell/openstack-protect-in-use-images.sh @@ -21,18 +21,17 @@ os_cloud="${OS_CLOUD:-vex}" set -eu -o pipefail -declare -a images -images+=("$(grep -r IMAGE_NAME --include \*.cfg jenkins-config \ - | awk -F'=' '{print $2}' | sort -u)") -set +o pipefail # Not all projects have images in YAML files and grep returns non-zero on 0 results -# Ignore SC2179 since we do not want () to result in an empty array item. -#shellcheck disable=SC2179 -images+="$(grep -r 'ZZCI - ' --include \*.yaml jjb \ - | awk -F": " '{print $3}' | sed "s:'::;s:'$::;/^$/d" | sort -u)" -set -o pipefail -readarray -t images <<< "$(for i in "${images[@]}"; do echo "$i"; done | sort)" +conf_images=("$(grep -r IMAGE_NAME --include \*.cfg jenkins-config \ + | awk -F'=' '{print $2}' | sort -u)") +# If there are no yaml files the 'grep' will fail, which is OK +yaml_images=("$(grep -r 'ZZCI - ' --include \*.yaml jjb \ + | awk -F": " '{print $3}' | sed "s:'::;s:'$::;/^$/d" \ + | sort -u)") || true +readarray -t images <<< "$(for i in "${conf_images[@}" "${yaml_images[@]}"; do \ + echo "$i"; done | sort)" for image in "${images[@]}"; do + [[ -z $image ]] && continue os_image_protected=$(openstack --os-cloud "$os_cloud" \ image show "$image" -f value -c protected) echo "Protected setting for $image: $os_image_protected" diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index 9fda0be4..6b963a48 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -18,7 +18,7 @@ REQUIREMENTS_FILE=$(mktemp /tmp/requirements-XXXX.txt) # git+https://github.com/lfit/releng-lftools.git#egg=lftools[openstack] cat << EOF > "$REQUIREMENTS_FILE" -lftools[openstack]~=0.20.0 +lftools[openstack]~=0.21.0 python-heatclient~=1.16.1 python-openstackclient~=3.16.0 dogpile.cache~=0.6.8 # Version 0.7.[01] seems to break openstackclient diff --git a/shell/sigul-sign-dir.sh b/shell/sigul-sign-dir.sh index 8bb7beac..2bcf5894 100644 --- a/shell/sigul-sign-dir.sh +++ b/shell/sigul-sign-dir.sh @@ -13,5 +13,4 @@ echo "---> sigul-sign-dir.sh" # Ensure we fail the job if any steps fail. set -e -o pipefail -lftools_activate -lftools sign sigul "${SIGN_DIR}" +lftools sign sigul -m "${SIGN_MODE}" "${SIGN_DIR}"