- java-builder
- mininet
+ update-cloud-images: false
+
- project:
name: throttle-ci-jobs
jobs:
--- /dev/null
+---
+- project:
+ name: openstack-update-cloud-images-full-test
+ jobs:
+ - "gerrit-openstack-update-cloud-image"
+
+ project: ciman
+ project-name: ciman-full
+ build-timeout: 10
+ branch: master
+ archive-artifacts: "**/*.log"
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+ gerrit-user: "jenkins-user"
+ gerrit-host: "git.example.org"
+ gerrit-topic: "update-cloud-image"
+ reviewers-email: "jenkins-user@example.org"
--- /dev/null
+---
+- project:
+ name: openstack-update-cloud-images-minimal-test
+ jobs:
+ - "gerrit-openstack-update-cloud-image"
+
+ project-name: ciman-minimal
+ gerrit-user: "jenkins-user"
+ gerrit-host: "git.example.org"
+ gerrit-topic: "update-cloud-image"
+ reviewers-email: "jenkins-user@example.org"
.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-full.yaml
+.. _gjjb-openstack-update-cloud-image:
+
+OpenStack Update Cloud Image
+----------------------------
+
+This job finds and updates OpenStack cloud images on the ci-management source
+repository.
+
+The job is triggered in two ways:
+
+1. When packer merge job completes, the new image name created is passed
+ down to the job.
+2. When the job is triggered manually to update all new images.
+
+When the job is triggered through an upstream packer merge job, this only
+generates a change request for the new image built.
+
+When the job is triggered manually, this job finds the latest images on
+OpenStack cloud and compares them with the images currently used in the source
+ci-management source repository. If the compared images have newer
+time stamps are **all** updated through a change request.
+
+This job requires a Jenkins configuration merge and verify job setup and
+working on Jenkins.
+
+:Template Names:
+ - {project-name}-openstack-update-cloud-image
+ - gerrit-openstack-update-cloud-image
+ - github-openstack-update-cloud-image
+
+:Required parameters:
+
+ :build-node: The node to run build on.
+ :jenkins-ssh-credential: Credential to use for SSH. (Generally should
+ be configured in defaults.yaml)
+ :new-image-name: Name of new image name passed from packer merge job or
+ set to 'all' to update all images. (default: all)
+
+:Optional parameters:
+
+ :branch: Git branch to fetch for the build. (default: master)
+ :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
+ :build-timeout: Timeout in minutes before aborting build. (default: 90)
+ :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+ :openstack-cloud: OS_CLOUD setting to pass to openstack client.
+ (default: vex)
+ :stream: Keyword that can be used to represent a release code-name.
+ Often the same as the branch. (default: master)
+ :submodule-recursive: Whether to checkout submodules recursively.
+ (default: true)
+ :submodule-timeout: Timeout (in minutes) for checkout operation.
+ (default: 10)
+ :submodule-disable: Disable submodule checkout operation.
+ (default: false)
+ :update-cloud-image: Submit a change request to update new built cloud
+ image to Jenkins. (default: false)
+
+Minimal Example:
+
+.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml
+
+Full Example:
+
+.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml
+
.. _gjjb-packer-merge:
(default: false)
:gerrit_verify_triggers: Override Gerrit Triggers.
+ :update-cloud-image: Submit a change request to update new built cloud
+ image to Jenkins. (default: false)
+
Test an in-progress patch
^^^^^^^^^^^^^^^^^^^^^^^^^
:platform: Build platform as found in the vars directory.
:template: Packer template to build as found in the templates directory.
+:Optional parameters:
+
+ :update-cloud-image: Submit a change request to update new built cloud
+ image to Jenkins.
+
lf-infra-packer-validate
------------------------
Retrieves system stats.
+lf-infra-update-packer-images
+-----------------------------
+
+Find and update the new built cloud image{s} in the ci-management source
+repository.
+
+
lf-jacoco-nojava-workaround
---------------------------
``RELEASE_USERNAME = YOUR_RELEASE_USERNAME``
``RELEASE_EMAIL = YOUR_RELEASE_EMAIL``
+
+.. note::
+
+ These also need to be added to your global-vars-$SILO.sh
+ or they will be overwritten.
+
Jenkins configure -> Managed Files -> Add a New Config -> Custom File
id: signing-pubkey
jobs:
- github-packer-merge
- github-packer-verify
+
+- job-group:
+ name: "{project-name}-openstack-jobs"
+
+ jobs:
+ - gerrit-openstack-update-cloud-image
+ - gerrit-openstack-cron
+
+- job-group:
+ name: "{project-name}-github-openstack-jobs"
+
+ jobs:
+ - github-openstack-update-cloud-image
+ - github-openstack-cron
openstack: true
openstack-cloud: vex
+ update-cloud-image: false
#####################
# Job Configuration #
branch: "{branch}"
- lf-packer-parameters:
packer-version: "{packer-version}"
+ - bool:
+ name: UPDATE_CLOUD_IMAGE
+ default: "{update-cloud-image}"
+ description: "Update new built image on the cloud."
builders:
- lf-infra-packer-build:
packer-version: "{packer-version}"
platform: "{platforms}"
template: "{templates}"
+ update-cloud-image: "{update-cloud-image}"
- description-setter:
regexp: '(\s+.*)(ZZCI\s+.*\d+-\d+\.\d+)'
description: 'Image: \2'
+ # - trigger-builds:
+ # - project: '{project-name}-openstack-update-cloud-image'
+ # block: false
+ # predefined-parameters: |
+ # GERRIT_BRANCH=$GERRIT_BRANCH
+ # GERRIT_PROJECT=$GERRIT_PROJECT
+ # GERRIT_REFSPEC=$GERRIT_REFSPEC
+ # NEW_IMAGE_NAME=$NEW_IMAGE_NAME
+ # property-file: variables.jenkins-trigger
+ # property-file-fail-on-missing: true
+
+ publishers:
+ - lf-infra-publish
+ - trigger-parameterized-builds:
+ - project: "{project-name}-openstack-update-cloud-image"
+ condition: UNSTABLE_OR_BETTER
+ predefined-parameters: |
+ NEW_IMAGE_NAME=$NEW_IMAGE_NAME
+ property-file: variables.jenkins-trigger
+ fail-on-missing: true
- job-template:
name: "{project-name}-packer-merge-{platforms}-{templates}"
white-list-target-branches:
- "{branch}"
+################################
+# Openstack Update Cloud Image #
+################################
+
+- lf_openstack_cron: &lf_openstack_update_cloud_image
+ name: lf-openstack-update-cloud-image
+
+ ######################
+ # Default parameters #
+ ######################
+
+ branch: master
+ build-days-to-keep: 7
+ build-timeout: 10
+ cron: "@monthly"
+ disable-job: false
+ git-url: "$GIT_URL/$PROJECT"
+ github-url: "https://github.com"
+ new-image-name: "all"
+ openstack-cloud: vex
+ stream: master
+ submodule-timeout: 10
+ submodule-disable: false
+ update-cloud-image: false
+
+ #####################
+ # Job Configuration #
+ #####################
+
+ project-type: freestyle
+ node: "{build-node}"
+ concurrent: false
+ disabled: "{disable-job}"
+
+ properties:
+ - lf-infra-properties:
+ build-days-to-keep: "{build-days-to-keep}"
+
+ parameters:
+ - lf-infra-parameters:
+ project: "{project}"
+ stream: "{stream}"
+ branch: "{branch}"
+ - string:
+ name: NEW_IMAGE_NAME
+ default: "{new-image-name}"
+ description: "Name of cloud image to update in Jenkins"
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: "{build-timeout}"
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+ # Listed after to override openstack-infra-wrappers clouds.yaml definition
+ - config-file-provider:
+ files:
+ - file-id: clouds-yaml
+ target: "$HOME/.config/openstack/clouds.yaml"
+ - file-id: npmrc
+ target: "$HOME/.npmrc"
+ - file-id: pipconf
+ target: "$HOME/.config/pip/pip.conf"
+
+ builders:
+ - lf-infra-pre-build
+ - inject:
+ properties-content: |
+ OS_CLOUD={openstack-cloud}
+ - shell: |
+ #!/bin/bash
+ echo "Extract the image type for commit message"
+ # echo IMAGE_TYPE=${{NEW_IMAGE_NAME% -*}} >> image-type.txt
+ IMAGE_TYPE=$(echo ${{NEW_IMAGE_NAME}} | tr -d "\'\"\ " | awk -F- '{{ print $2 " " $3 " " $4 }}')
+ echo IMAGE_TYPE=${{IMAGE_TYPE}} >> image-type.txt
+ cat image-type.txt
+ - inject:
+ properties-file: "image-type.txt"
+ - lf-infra-update-packer-images
+ - lf-infra-push-gerrit-patch:
+ project: "{project}"
+ gerrit-user: "{gerrit-user}"
+ gerrit-host: "{gerrit-host}"
+ gerrit-topic: "{gerrit-topic}"
+ gerrit-commit-message: "Update cloud image $IMAGE_TYPE"
+ reviewers-email: "{reviewers-email}"
+
+ publishers:
+ - lf-infra-publish
+
+- job-template:
+ name: "{project-name}-openstack-update-cloud-image"
+ id: gerrit-openstack-update-cloud-image
+ <<: *lf_openstack_update_cloud_image
+
+ ######################
+ # Default parameters #
+ ######################
+
+ git-url: "$GIT_URL/$GERRIT_PROJECT"
+
+ #####################
+ # Job Configuration #
+ #####################
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: "{git-url}"
+ refspec: "$GERRIT_REFSPEC"
+ branch: "$GERRIT_BRANCH"
+ submodule-recursive: false
+ submodule-timeout: "{submodule-timeout}"
+ submodule-disable: "{submodule-disable}"
+ choosing-strategy: gerrit
+ jenkins-ssh-credential: "{jenkins-ssh-credential}"
+
+ triggers:
+ - timed: "{obj:cron}"
+
#################
# Puppet Verify #
#################
PACKER_PLATFORM={platform}
PACKER_TEMPLATE={template}
PACKER_VERSION={packer-version}
+ UPDATE_CLOUD_IMAGE={update-cloud-image}
- shell: !include-raw-escape:
- ../shell/packer-install.sh
- ../shell/packer-build.sh
- shell: !include-raw:
- ../shell/packer-clear-credentials.sh
+- builder:
+ name: lf-infra-update-packer-images
+ builders:
+ - shell: !include-raw: ../shell/update-cloud-images.sh
+
- builder:
name: lf-infra-push-gerrit-patch
builders:
branch: master
build-days-to-keep: 30
build-timeout: 60
- cron: "@daily"
+ cron: "H H * * 7"
disable-job: false
git-url: "$GIT_URL/$PROJECT"
github-url: "https://github.com"
--- /dev/null
+---
+features:
+ - |
+ Add template to update OpenStack cloud images.
+ - |
+ This job finds and updates OpenStack cloud images on the ci-management
+ source repository.
+ - |
+ The job is triggered in two ways:
+ - |
+ 1. When a packer merge job completes, the new image name created is passed
+ down to the job.
+ 2. Manually trigger the job to update all images.
+ - |
+ When the job is triggered through an upstream packer merge job, this only
+ generates a change request for the new image built.
+ - |
+ When the job is triggered manually, this job finds the latest images on
+ OpenStack cloud and compares them with the images currently used in
+ the source ci-management source repository. If the compared images have
+ newer time stamps are **all** updated through a change request.
+ - |
+ This job requires a Jenkins configuration merge and verify job setup and
+ working on Jenkins.
+upgrade:
+ - |
+ Packer merge jobs have a new build parameter when checked also updates the
+ cloud image.
+ - |
+ **lf-infra-packer-build** macro now requires 1 new variables to be passed.
+ - |
+ #. **update-cloud-image:** Set to true when images need to be updated on
+ Jenkins.
--- /dev/null
+---
+fix:
+ - |
+ Builders may have diffrent pyenv versions installed.
+ Programically pick the latest pyenv version.
+ Since we change pyenv version when building images, we do not know which
+ pyenv version are avaliable.
--- /dev/null
+---
+fixes:
+ - |
+ Run WhiteSource scan jobs weekly on Sunday.
--- /dev/null
+---
+fixes:
+ - |
+ Pip install pyenv from python2 should force more-itertools to 5.0.0
+ In a fresh python2.7 venv "pip install pyenv" correctly pulls down
+ more-itertools [required: Any, installed: 5.0.0]
+ If for some reason a higher version is already installed this will downgrade
+ more-itertools to a py2 compatible version
export PATH="$PYENV_ROOT/bin:$PATH"
PYTHONPATH=$(pwd)
export PYTHONPATH
- pyenv local 3.6.4
+
+ latest_version=$(pyenv versions \
+ | sed s,*,,g \
+ | awk '/[0-9]+/{ print $1 }' \
+ | sort --version-sort \
+ | awk '/./{line=$0} END{print line}')
+
+ pyenv local "$latest_version"
export PYENV_VERSION="3.6.4"
fi
pip install --quiet --upgrade git-review
set -u
# End git-review workaround
-
# Remove any leading or trailing quotes surrounding the strings
# which can cause parse errors when passed as CLI options to commands
PROJECT="$(echo "$PROJECT" | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")"
mkdir -p "$PACKER_LOGS_DIR"
export PATH="${WORKSPACE}/bin:$PATH"
-cd packer || exit
+cd packer
# Prioritize the project's own version of vars if available
platform_file="common-packer/vars/$PACKER_PLATFORM.json"
-if [ -f "vars/$PACKER_PLATFORM.json" ]; then
+if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then
platform_file="vars/$PACKER_PLATFORM.json"
fi
-var-file="$platform_file" \
"templates/$PACKER_TEMPLATE.json"
+# Extract image name from log and store value in the downstream job
+if [[ ${UPDATE_CLOUD_IMAGE} ]]; then
+
+ NEW_IMAGE_NAME=$(grep -P '(\s+.*image: )(ZZCI\s+.*\d+-\d+\.\d+)' "$PACKER_BUILD_LOG" \
+ | awk -F': ' '{print $4}')\")
+
+ echo NEW_IMAGE_NAME="$NEW_IMAGE_NAME" >> "$WORKSPACE/variables.prop"
+ echo "NEW_IMAGE_NAME: ${NEW_IMAGE_NAME}"
+
+ # Copy variables.prop to variables.jenkins-trigger so that the end of build
+ # trigger can pick up the file as input for triggering downstream jobs.
+ # Dont tigger downstream job when UPDATE_CLOUD_IMAGE is set to 'false'
+ cp $WORKSPACE/variables.prop $WORKSPACE/variables.jenkins-trigger
+fi
+
# Retrive the list of cloud providers
mapfile -t clouds < <(jq -r '.builders[].name' "templates/$PACKER_TEMPLATE.json")
fi
PYTHONPATH=$(pwd)
export PYTHONPATH
-pyenv local 3.6.4
-export PYENV_VERSION="3.6.4"
+
+latest_version=$(pyenv versions \
+ | sed s,*,,g \
+ | awk '/[0-9]+/{ print $1 }' \
+ | sort --version-sort \
+ | awk '/./{line=$0} END{print line}')
+
+pyenv local "$latest_version"
+
+export PYENV_VERSION="$latest_version"
pip install --user lftools[nexus] jsonschema niet yq
#Functions.
set -eux -o pipefail
# Tox version is pulled in through detox to mitigate version conflict
-$PYTHON -m pip install --user --quiet --upgrade tox-pyenv
+
+
+if [[ $PYTHON == "python2" ]]; then
+ $PYTHON -m pip install --user --quiet --upgrade tox-pyenv more-itertools~=5.0.0
+else
+ $PYTHON -m pip install --user --quiet --upgrade tox-pyenv
+fi
+
$PYTHON -m pip freeze
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# Auto-update packer image{s} when the job is started manually or a single
+# image passed by upstream packer merge job:
+# 1. Get a list of image{s} from the releng/builder repository
+# 2. Search openstack cloud for the latest image{s} available or use the image
+# name passed down from the upstream job.
+# 3. Compare the time stamps of the new image{s} with the image in use
+# 4. Update the image{s} in the config files and yaml files
+# 5. Push the change to Gerrit
+
+virtualenv "/tmp/v/openstack"
+# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091
+source "/tmp/v/openstack/bin/activate"
+pip install --upgrade --quiet "pip<10.0.0" setuptools
+pip install --upgrade --quiet python-openstackclient
+pip freeze
+
+set -e
+
+mkdir -p "$WORKSPACE/archives"
+echo "List of images used on the source repository:"
+grep -Er '(_system_image:|IMAGE_NAME)' \
+--exclude-dir="global-jjb" --exclude-dir="common-packer" \
+| grep ZZCI | awk -F: -e '{print $3}' \
+| grep '\S' | tr -d \'\" | sort -n | uniq \
+| tee "$WORKSPACE/archives/used_image_list.txt"
+
+while read -r line ; do
+ image_in_use="${line}"
+
+ # get image type - ex: builder, docker, gbp etc
+ image_type="${line% -*}"
+ # Get the latest images available on the cloud, when $NEW_IMAGE_NAME env
+ # var is unset and update all images on Jenkins to the latest.
+ if [[ ${NEW_IMAGE_NAME} != all ]]; then
+ new_image=${NEW_IMAGE_NAME}
+ else
+ new_image=$(openstack image list --long -f value -c Name -c Protected \
+ | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//')
+ fi
+ [[ -z ${new_image} ]] && continue
+
+ # strip the timestamp from the image name amd compare
+ new_image_isotime=${new_image##*- }
+ image_in_use_isotime=${image_in_use##*- }
+ # compare timestamps
+ if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then
+ # generate a patch to be submited to Gerrit
+ echo "Update old image: ${image_in_use} with new image: ${new_image}"
+ grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/"
+ # When the script is triggered by upstream packer-merge job
+ # update only the requested image and break the loop
+ [[ ${NEW_IMAGE_NAME} != all ]] && break
+ else
+ echo "No new image to update: ${new_image}"
+ fi
+done < "$WORKSPACE/archives/used_image_list.txt"
+
+git remote -v
+git status
+git diff > "$WORKSPACE/archives/new-images-patchset.diff"
+git add -u
Code Review / releng / global-jjb.git / commitdiff