From 4028ad2bc487460837736f040d885c3674b39031 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Mon, 16 Sep 2019 22:48:02 +0530 Subject: [PATCH] Add template to update OpenStack cloud images This job finds and updates OpenStack cloud images on the ci-management source repository. The job is triggered in two ways: 1. When a packer merge job completes, the new image name created is passed down to the job. 2. Manually trigger the job to update all images. When the job is triggered through an upstream packer merge job, this only generates a change request for the new image built. When the job is triggered manually, this job finds the latest images on OpenStack cloud and compares them with the images currently used in the source ci-management source repository. If the compared images have newer time stamps are **all** updated through a change request. This job requires a Jenkins configuration merge and verify job setup and working on Jenkins. Issue: RELENG-2352 Change-Id: I2c75407bbe9d6866ded492310b8d34b47c3a1341 Signed-off-by: Anil Belur --- .jjb-test/lf-ci-jobs.yaml | 2 + .../openstack-update-cloud-image-full.yaml | 16 +++ .../openstack-update-cloud-image-minimal.yaml | 11 ++ docs/jjb/lf-ci-jobs.rst | 68 ++++++++++ docs/jjb/lf-macros.rst | 12 ++ jjb/lf-ci-job-groups.yaml | 14 ++ jjb/lf-ci-jobs.yaml | 143 +++++++++++++++++++++ jjb/lf-macros.yaml | 6 + .../auto-update-cloud-images-19a2241aca85abe3.yaml | 33 +++++ shell/gerrit-push-patch.sh | 1 - shell/packer-build.sh | 19 ++- shell/update-cloud-images.sh | 72 +++++++++++ 12 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 .jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml create mode 100644 .jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml create mode 100644 releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml create mode 100644 shell/update-cloud-images.sh diff --git a/.jjb-test/lf-ci-jobs.yaml b/.jjb-test/lf-ci-jobs.yaml index ed0ef468..baeced5b 100644 --- a/.jjb-test/lf-ci-jobs.yaml +++ b/.jjb-test/lf-ci-jobs.yaml @@ -66,6 +66,8 @@ - java-builder - mininet + update-cloud-images: false + - project: name: throttle-ci-jobs jobs: diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml new file mode 100644 index 00000000..fd6eee8a --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml @@ -0,0 +1,16 @@ +--- +- project: + name: openstack-update-cloud-images-full-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project: ciman + project-name: ciman-full + build-timeout: 10 + branch: master + archive-artifacts: "**/*.log" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml new file mode 100644 index 00000000..9f536002 --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml @@ -0,0 +1,11 @@ +--- +- project: + name: openstack-update-cloud-images-minimal-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project-name: ciman-minimal + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 9883dd7a..a266142e 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -650,6 +650,71 @@ Full Example: .. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-full.yaml +.. _gjjb-openstack-update-cloud-image: + +OpenStack Update Cloud Image +---------------------------- + +This job finds and updates OpenStack cloud images on the ci-management source +repository. + +The job is triggered in two ways: + +1. When packer merge job completes, the new image name created is passed + down to the job. +2. When the job is triggered manually to update all new images. + +When the job is triggered through an upstream packer merge job, this only +generates a change request for the new image built. + +When the job is triggered manually, this job finds the latest images on +OpenStack cloud and compares them with the images currently used in the source +ci-management source repository. If the compared images have newer +time stamps are **all** updated through a change request. + +This job requires a Jenkins configuration merge and verify job setup and +working on Jenkins. + +:Template Names: + - {project-name}-openstack-update-cloud-image + - gerrit-openstack-update-cloud-image + - github-openstack-update-cloud-image + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally should + be configured in defaults.yaml) + :new-image-name: Name of new image name passed from packer merge job or + set to 'all' to update all images. (default: all) + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 90) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :openstack-cloud: OS_CLOUD setting to pass to openstack client. + (default: vex) + :stream: Keyword that can be used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + +Minimal Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml + +Full Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml + .. _gjjb-packer-merge: @@ -700,6 +765,9 @@ Packer Merge job runs `packer build` to build system images in the cloud. (default: false) :gerrit_verify_triggers: Override Gerrit Triggers. + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + Test an in-progress patch ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index a653bc55..e628179d 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -126,6 +126,11 @@ Run `packer build` to build system images. :platform: Build platform as found in the vars directory. :template: Packer template to build as found in the templates directory. +:Optional parameters: + + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. + lf-infra-packer-validate ------------------------ @@ -167,6 +172,13 @@ lf-infra-sysstat Retrieves system stats. +lf-infra-update-packer-images +----------------------------- + +Find and update the new built cloud image{s} in the ci-management source +repository. + + lf-jacoco-nojava-workaround --------------------------- diff --git a/jjb/lf-ci-job-groups.yaml b/jjb/lf-ci-job-groups.yaml index 7b32f4da..81e6b0b9 100644 --- a/jjb/lf-ci-job-groups.yaml +++ b/jjb/lf-ci-job-groups.yaml @@ -46,3 +46,17 @@ jobs: - github-packer-merge - github-packer-verify + +- job-group: + name: "{project-name}-openstack-jobs" + + jobs: + - gerrit-openstack-update-cloud-image + - gerrit-openstack-cron + +- job-group: + name: "{project-name}-github-openstack-jobs" + + jobs: + - github-openstack-update-cloud-image + - github-openstack-cron diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 06a10d2b..f69e8d71 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1418,6 +1418,7 @@ openstack: true openstack-cloud: vex + update-cloud-image: false ##################### # Job Configuration # @@ -1430,6 +1431,10 @@ branch: "{branch}" - lf-packer-parameters: packer-version: "{packer-version}" + - bool: + name: UPDATE_CLOUD_IMAGE + default: "{update-cloud-image}" + description: "Update new built image on the cloud." builders: - lf-infra-packer-build: @@ -1439,9 +1444,30 @@ packer-version: "{packer-version}" platform: "{platforms}" template: "{templates}" + update-cloud-image: "{update-cloud-image}" - description-setter: regexp: '(\s+.*)(ZZCI\s+.*\d+-\d+\.\d+)' description: 'Image: \2' + # - trigger-builds: + # - project: '{project-name}-openstack-update-cloud-image' + # block: false + # predefined-parameters: | + # GERRIT_BRANCH=$GERRIT_BRANCH + # GERRIT_PROJECT=$GERRIT_PROJECT + # GERRIT_REFSPEC=$GERRIT_REFSPEC + # NEW_IMAGE_NAME=$NEW_IMAGE_NAME + # property-file: variables.jenkins-trigger + # property-file-fail-on-missing: true + + publishers: + - lf-infra-publish + - trigger-parameterized-builds: + - project: "{project-name}-openstack-update-cloud-image" + condition: UNSTABLE_OR_BETTER + predefined-parameters: | + NEW_IMAGE_NAME=$NEW_IMAGE_NAME + property-file: variables.jenkins-trigger + fail-on-missing: true - job-template: name: "{project-name}-packer-merge-{platforms}-{templates}" @@ -1687,6 +1713,123 @@ white-list-target-branches: - "{branch}" +################################ +# Openstack Update Cloud Image # +################################ + +- lf_openstack_cron: &lf_openstack_update_cloud_image + name: lf-openstack-update-cloud-image + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 10 + cron: "@monthly" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + new-image-name: "all" + openstack-cloud: vex + stream: master + submodule-timeout: 10 + submodule-disable: false + update-cloud-image: false + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + concurrent: false + disabled: "{disable-job}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - lf-infra-parameters: + project: "{project}" + stream: "{stream}" + branch: "{branch}" + - string: + name: NEW_IMAGE_NAME + default: "{new-image-name}" + description: "Name of cloud image to update in Jenkins" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + # Listed after to override openstack-infra-wrappers clouds.yaml definition + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - file-id: npmrc + target: "$HOME/.npmrc" + - file-id: pipconf + target: "$HOME/.config/pip/pip.conf" + + builders: + - lf-infra-pre-build + - inject: + properties-content: | + OS_CLOUD={openstack-cloud} + - shell: | + #!/bin/bash + echo "Extract the image type for commit message" + # echo IMAGE_TYPE=${{NEW_IMAGE_NAME% -*}} >> image-type.txt + IMAGE_TYPE=$(echo ${{NEW_IMAGE_NAME}} | tr -d "\'\"\ " | awk -F- '{{ print $2 " " $3 " " $4 }}') + echo IMAGE_TYPE=${{IMAGE_TYPE}} >> image-type.txt + cat image-type.txt + - inject: + properties-file: "image-type.txt" + - lf-infra-update-packer-images + - lf-infra-push-gerrit-patch: + project: "{project}" + gerrit-user: "{gerrit-user}" + gerrit-host: "{gerrit-host}" + gerrit-topic: "{gerrit-topic}" + gerrit-commit-message: "Update cloud image $IMAGE_TYPE" + reviewers-email: "{reviewers-email}" + + publishers: + - lf-infra-publish + +- job-template: + name: "{project-name}-openstack-update-cloud-image" + id: gerrit-openstack-update-cloud-image + <<: *lf_openstack_update_cloud_image + + ###################### + # Default parameters # + ###################### + + git-url: "$GIT_URL/$GERRIT_PROJECT" + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: false + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - timed: "{obj:cron}" + ################# # Puppet Verify # ################# diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 91fd16b5..fa012028 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -111,6 +111,7 @@ PACKER_PLATFORM={platform} PACKER_TEMPLATE={template} PACKER_VERSION={packer-version} + UPDATE_CLOUD_IMAGE={update-cloud-image} - shell: !include-raw-escape: - ../shell/packer-install.sh - ../shell/packer-build.sh @@ -143,6 +144,11 @@ - shell: !include-raw: - ../shell/packer-clear-credentials.sh +- builder: + name: lf-infra-update-packer-images + builders: + - shell: !include-raw: ../shell/update-cloud-images.sh + - builder: name: lf-infra-push-gerrit-patch builders: diff --git a/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml new file mode 100644 index 00000000..41327100 --- /dev/null +++ b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml @@ -0,0 +1,33 @@ +--- +features: + - | + Add template to update OpenStack cloud images. + - | + This job finds and updates OpenStack cloud images on the ci-management + source repository. + - | + The job is triggered in two ways: + - | + 1. When a packer merge job completes, the new image name created is passed + down to the job. + 2. Manually trigger the job to update all images. + - | + When the job is triggered through an upstream packer merge job, this only + generates a change request for the new image built. + - | + When the job is triggered manually, this job finds the latest images on + OpenStack cloud and compares them with the images currently used in + the source ci-management source repository. If the compared images have + newer time stamps are **all** updated through a change request. + - | + This job requires a Jenkins configuration merge and verify job setup and + working on Jenkins. +upgrade: + - | + Packer merge jobs have a new build parameter when checked also updates the + cloud image. + - | + **lf-infra-packer-build** macro now requires 1 new variables to be passed. + - | + #. **update-cloud-image:** Set to true when images need to be updated on + Jenkins. diff --git a/shell/gerrit-push-patch.sh b/shell/gerrit-push-patch.sh index b7065130..62f8e9be 100644 --- a/shell/gerrit-push-patch.sh +++ b/shell/gerrit-push-patch.sh @@ -42,7 +42,6 @@ pip install --quiet --upgrade "pip==9.0.3" setuptools pip install --quiet --upgrade git-review set -u # End git-review workaround - # Remove any leading or trailing quotes surrounding the strings # which can cause parse errors when passed as CLI options to commands PROJECT="$(echo "$PROJECT" | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")" diff --git a/shell/packer-build.sh b/shell/packer-build.sh index 1caca8e3..dfacdeb7 100644 --- a/shell/packer-build.sh +++ b/shell/packer-build.sh @@ -22,11 +22,11 @@ PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" mkdir -p "$PACKER_LOGS_DIR" export PATH="${WORKSPACE}/bin:$PATH" -cd packer || exit +cd packer # Prioritize the project's own version of vars if available platform_file="common-packer/vars/$PACKER_PLATFORM.json" -if [ -f "vars/$PACKER_PLATFORM.json" ]; then +if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then platform_file="vars/$PACKER_PLATFORM.json" fi @@ -41,6 +41,21 @@ packer.io build -color=false \ -var-file="$platform_file" \ "templates/$PACKER_TEMPLATE.json" +# Extract image name from log and store value in the downstream job +if [[ ${UPDATE_CLOUD_IMAGE} ]]; then + + NEW_IMAGE_NAME=$(grep -P '(\s+.*image: )(ZZCI\s+.*\d+-\d+\.\d+)' "$PACKER_BUILD_LOG" \ + | awk -F': ' '{print $4}')\") + + echo NEW_IMAGE_NAME="$NEW_IMAGE_NAME" >> "$WORKSPACE/variables.prop" + echo "NEW_IMAGE_NAME: ${NEW_IMAGE_NAME}" + + # Copy variables.prop to variables.jenkins-trigger so that the end of build + # trigger can pick up the file as input for triggering downstream jobs. + # Dont tigger downstream job when UPDATE_CLOUD_IMAGE is set to 'false' + cp $WORKSPACE/variables.prop $WORKSPACE/variables.jenkins-trigger +fi + # Retrive the list of cloud providers mapfile -t clouds < <(jq -r '.builders[].name' "templates/$PACKER_TEMPLATE.json") diff --git a/shell/update-cloud-images.sh b/shell/update-cloud-images.sh new file mode 100644 index 00000000..6b39c7cc --- /dev/null +++ b/shell/update-cloud-images.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Auto-update packer image{s} when the job is started manually or a single +# image passed by upstream packer merge job: +# 1. Get a list of image{s} from the releng/builder repository +# 2. Search openstack cloud for the latest image{s} available or use the image +# name passed down from the upstream job. +# 3. Compare the time stamps of the new image{s} with the image in use +# 4. Update the image{s} in the config files and yaml files +# 5. Push the change to Gerrit + +virtualenv "/tmp/v/openstack" +# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091 +source "/tmp/v/openstack/bin/activate" +pip install --upgrade --quiet "pip<10.0.0" setuptools +pip install --upgrade --quiet python-openstackclient +pip freeze + +set -e + +mkdir -p "$WORKSPACE/archives" +echo "List of images used on the source repository:" +grep -Er '(_system_image:|IMAGE_NAME)' \ +--exclude-dir="global-jjb" --exclude-dir="common-packer" \ +| grep ZZCI | awk -F: -e '{print $3}' \ +| grep '\S' | tr -d \'\" | sort -n | uniq \ +| tee "$WORKSPACE/archives/used_image_list.txt" + +while read -r line ; do + image_in_use="${line}" + + # get image type - ex: builder, docker, gbp etc + image_type="${line% -*}" + # Get the latest images available on the cloud, when $NEW_IMAGE_NAME env + # var is unset and update all images on Jenkins to the latest. + if [[ ${NEW_IMAGE_NAME} != all ]]; then + new_image=${NEW_IMAGE_NAME} + else + new_image=$(openstack image list --long -f value -c Name -c Protected \ + | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//') + fi + [[ -z ${new_image} ]] && continue + + # strip the timestamp from the image name amd compare + new_image_isotime=${new_image##*- } + image_in_use_isotime=${image_in_use##*- } + # compare timestamps + if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then + # generate a patch to be submited to Gerrit + echo "Update old image: ${image_in_use} with new image: ${new_image}" + grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/" + # When the script is triggered by upstream packer-merge job + # update only the requested image and break the loop + [[ ${NEW_IMAGE_NAME} != all ]] && break + else + echo "No new image to update: ${new_image}" + fi +done < "$WORKSPACE/archives/used_image_list.txt" + +git remote -v +git status +git diff > "$WORKSPACE/archives/new-images-patchset.diff" +git add -u -- 2.16.6