From 1d1aaf33f079ee4f51dfce68e8f5592a649947ca Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Thu, 18 Jan 2018 19:24:56 -0500 Subject: [PATCH] Create job to manage Jenkins Global Variables Parses a configuration file containing variable listings in the form "KEY=value" one per line and creates the appropriate groovy script to push these environment variables to Jenkins Global Config. Configuration file is stored in ci-management/jenkins-config/global-vars-SILO.sh (Where silo is production | sandbox) While JJB provided us a way to store job configuration in a git repo we have never had a good way to manage global Jenkins configuration. This patch enables us to manage one aspect, Global Environment variables in the ci-management repository. Issue: RELENG-753 Change-Id: I7d134950b25d469686cace2f79fd8c297727166c Signed-off-by: Thanh Ha --- .jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml | 8 ++ .../lf-ci-jobs/jenkins-cfg-merge-minimal.yaml | 7 ++ docs/jjb/lf-ci-jobs.rst | 63 +++++++++++++++ jenkins-admin/set_global_properties.groovy | 29 +++++++ jjb/lf-ci-jobs.yaml | 93 ++++++++++++++++++++++ shell/jenkins-configure-global-vars.sh | 69 ++++++++++++++++ 6 files changed, 269 insertions(+) create mode 100644 .jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml create mode 100644 .jjb-test/lf-ci-jobs/jenkins-cfg-merge-minimal.yaml create mode 100644 jenkins-admin/set_global_properties.groovy create mode 100644 shell/jenkins-configure-global-vars.sh diff --git a/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml b/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml new file mode 100644 index 00000000..7123693c --- /dev/null +++ b/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml @@ -0,0 +1,8 @@ +--- +- project: + name: jenkins-cfg-merge-full-test + jobs: + - 'gerrit-jenkins-cfg-merge' + + project-name: builder + jenkins-silos: releng sandbox diff --git a/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-minimal.yaml b/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-minimal.yaml new file mode 100644 index 00000000..5b7173e8 --- /dev/null +++ b/.jjb-test/lf-ci-jobs/jenkins-cfg-merge-minimal.yaml @@ -0,0 +1,7 @@ +--- +- project: + name: jenkins-cfg-merge-minimal-test + jobs: + - 'gerrit-jenkins-cfg-merge' + + project-name: ci-management diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 49fb4678..3ddec148 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -12,6 +12,7 @@ Recommended jobs that should be deployed for CI using Gerrit. :Includes: + - gerrit-jenkins-cfg-merge - gerrit-jjb-deploy-job - gerrit-jjb-merge - gerrit-jjb-verify @@ -23,6 +24,7 @@ Recommended jobs that should be deployed CI using GitHub. :Includes: + - github-jenkins-cfg-merge - github-jjb-deploy-job - github-jjb-merge - github-jjb-verify @@ -50,6 +52,18 @@ Jobs related to Packer builds for CI using GitHub. Macros ====== +lf-jenkins-cfg-global-vars +-------------------------- + +Manages the Global Jenkins variables. This macro will clear all exist macros +in Jenkins and replaces them with the ones defined by the +ci-management/jenkins-config/global-vars-SILO.sh script. + +:Required parameters: + + :jenkins-silos: Space separated list of Jenkins silos to update + configuration for as defined in ~/.config/jenkins_jobs/jenkins_jobs.ini + lf-infra-jjbini --------------- @@ -90,6 +104,55 @@ Job submits a patch to lock or unlock a project's branch. - gerrit-branch-lock +Jenkins Configuration Merge +--------------------------- + +Jenkins job to manage Global Jenkins configuration. + +Global Environment Variables are managed via the +``jenkins-config/global-vars-SILO.sh`` file in ci-management. Replace SILO with +the name of the Jenkins silo the variable configuration is for. + +The format for this file is ``KEY=value`` for example:: + + GERRIT_URL=https://git.opendaylight.org/gerrit + GIT_BASE=git://devvexx.opendaylight.org/mirror/$PROJECT + GIT_URL=git://devvexx.opendaylight.org/mirror + JENKINS_HOSTNAME=vex-yul-odl-jenkins-2 + LOGS_SERVER=https://logs.opendaylight.org + NEXUS_URL=https://nexus.opendaylight.org + ODLNEXUSPROXY=https://nexus.opendaylight.org + SILO=sandbox + SONAR_URL=https://sonar.opendaylight.org + +:Template names: + + - {project-name}-jenkins-cfg-merge + - gerrit-jenkins-cfg-merge + - github-jenkins-cfg-merge + +:Optional parameters: + + :git-url: URL to clone project from. (default: $GIT_URL/$GERRIT_PROJECT) + :jenkins-silos: Space separated list of Jenkins silos to update + configuration for as defined in ~/.config/jenkins_jobs/jenkins_jobs.ini + (default: production sandbox) + +Typically this template is automatically pulled in by the +"{project-name}-ci-jobs" job-group and does not need to be explicitly called if +the job group is being used. + +Miniaml Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/jenkins-cfg-merge-minimal.yaml + :language: yaml + +Full Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/jenkins-cfg-merge-full.yaml + :language: yaml + + JJB Deploy Job -------------- diff --git a/jenkins-admin/set_global_properties.groovy b/jenkins-admin/set_global_properties.groovy new file mode 100644 index 00000000..83008a28 --- /dev/null +++ b/jenkins-admin/set_global_properties.groovy @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: EPL-1.0 + * Copyright (c) 2018 The Linux Foundation and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ + +/** + * Manage Jenkins Global Properties by injecting configuration defined here + * + * In LFCI a Jenkins job script will replace the JENKINS_URL line below and + * inject the managed list of global variables. + */ + +def global_vars = [ + 'JENKINS_URL': 'https://localhost:8080', +] + +def gnode_prop = Jenkins.getInstance().getGlobalNodeProperties() +def properties = new hudson.slaves.EnvironmentVariablesNodeProperty() +gnode_prop.replace(properties) +env_vars = properties.getEnvVars() + +env_vars.clear() +global_vars.each{ k, v -> env_vars.put(k, v) } +instance.save() diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 2bff633c..e3dd10ba 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -3,6 +3,7 @@ name: '{project-name}-ci-jobs' jobs: + - gerrit-jenkins-cfg-merge - gerrit-jjb-deploy-job - gerrit-jjb-merge - gerrit-jjb-verify @@ -11,6 +12,7 @@ name: '{project-name}-github-ci-jobs' jobs: + - github-jenkins-cfg-merge - github-jjb-deploy-job - github-jjb-merge - github-jjb-verify @@ -248,6 +250,97 @@ publishers: - lf-infra-publish +########################################## +# Jenkins Configuration Management Merge # +########################################## + +- builder: + name: lf-jenkins-cfg-global-vars + builders: + - lf-infra-jjbini + - inject: + properties-content: 'jenkins_silos={jenkins-silos}' + - shell: !include-raw-escape: ../shell/jenkins-configure-global-vars.sh + - shell: rm "$HOME/.config/jenkins_jobs/jenkins_jobs.ini" + +- lf_jenkins_configuration: &lf_jenkins_cfg_merge + name: lf-jenkins-cfg-merge + + ###################### + # Default parameters # + ###################### + + branch: master + git-url: '$GIT_URL/$GERRIT_PROJECT' + jenkins-silos: production sandbox + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: '{build-node}' + concurrent: false + + properties: + - lf-infra-properties: + project: '{project}' + build-days-to-keep: 1 + + parameters: + - lf-infra-parameters: + project: '{project}' + stream: '' + branch: master + lftools-version: '{lftools-version}' + + wrappers: + - lf-infra-wrappers: + build-timeout: 10 + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + builders: + - shell: !include-raw-escape: + - ../shell/lftools-install.sh + - lf-jenkins-cfg-global-vars: + jenkins-silos: '{jenkins-silos}' + + publishers: + - lf-infra-publish + +- job-template: + name: '{project-name}-jenkins-cfg-merge' + id: gerrit-jenkins-cfg-merge + <<: *lf_jenkins_cfg_merge + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: 'refs/heads/{branch}' + branch: '{branch}' + submodule-recursive: true + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + +- job-template: + name: '{project-name}-jenkins-cfg-merge' + id: github-jenkins-cfg-merge + <<: *lf_jenkins_cfg_merge + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '$sha1' + submodule-recursive: true + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + ################## # JJB DEPLOY JOB # ################## diff --git a/shell/jenkins-configure-global-vars.sh b/shell/jenkins-configure-global-vars.sh new file mode 100644 index 00000000..6e640232 --- /dev/null +++ b/shell/jenkins-configure-global-vars.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# Pulls global variable definitions out of a file. +# +# Configuration is read from $WORKSPACE/jenkins-config/global-vars-$silo.sh +# +# Requirements: lftools must be installed to /tmp/v/lftools +# Parameters: +# jenkins_silos: Space separated list of Jenkins silos to push global-vars +# configuration to. (default: jenkins) +echo "---> jenkins-configure-global-vars.sh" + +GROOVY_SCRIPT_FILE="jjb/global-jjb/jenkins-admin/set_global_properties.groovy" + +# shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091 +source "/tmp/v/lftools/bin/activate" +silos="${jenkins_silos:-jenkins}" + +set -eu -o pipefail + +for silo in $silos; do + set +x # Ensure that no other scripts add `set -x` and print passwords + echo "Configuring $silo" + + JENKINS_URL=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" url) + JENKINS_USER=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" user) + JENKINS_PASSWORD=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" password) + export JENKINS_URL + export JENKINS_USER + export JENKINS_PASSWORD + + global_vars="$WORKSPACE/jenkins-config/global-vars-$silo.sh" + + if [ ! -f "$global_vars" ]; then + echo "ERROR: Configuration file $global_vars not found." + exit 1 + fi + + mapfile -t vars < <(cat $global_vars) + + rm -f insert.txt + for var in "${vars[@]}"; do + # Ignore comments and blank lines + if [[ $var == '#'* ]] || [ -z "$var" ]; then + continue + fi + + key=$(echo $var | cut -d\= -f1) + value=$(echo $var | cut -d\= -f2) + echo " '$key': '$value'," >> insert.txt + done + + # Insert variables and remove first occurrence of JENKINS_URL variable + echo "-----> script.groovy" + sed "/'JENKINS_URL'/r insert.txt" "$GROOVY_SCRIPT_FILE" \ + | sed "0,/'JENKINS_URL'/{/'JENKINS_URL'/d}" \ + > script.groovy + cat script.groovy + + lftools jenkins groovy script.groovy +done -- 2.16.6