shell/jenkins-configure-clouds.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2018 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 # Pulls global variable definitions out of a file.
  12 #
  13 # Configuration is read from $WORKSPACE/jenkins-config/clouds/openstack/$cloud/cloud.cfg
  14 #
  15 # Requirements: lftools must be installed to /tmp/v/lftools
  16 # Parameters:
  17 #     jenkins_silos:  Space separated list of Jenkins silos to push
  18 #                     configuration to. (default: jenkins)
  19 echo "---> jenkins-configure-clouds.sh"
  20
  21 if [ ! -d "$WORKSPACE/jenkins-config/clouds" ]; then
  22     echo "WARN: jenkins-config/clouds does not exist. Skipping cloud management..."
  23     exit 0
  24 fi
  25
  26 #shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091
  27 source "/tmp/v/lftools/bin/activate"
  28
  29 GROOVY_SCRIPT_FILE="jjb/global-jjb/jenkins-admin/manage_clouds.groovy"
  30 OS_CLOUD_DIR="$WORKSPACE/jenkins-config/clouds/openstack"
  31 SCRIPT_DIR="$WORKSPACE/archives/groovy-inserts"
  32 mkdir -p "$SCRIPT_DIR"
  33
  34 silos="${jenkins_silos:-jenkins}"
  35
  36 set -eu -o pipefail
  37
  38 get_cfg() {
  39     if [ -z ${3+x} ]; then
  40         >&2 echo "Usage: get_cfg CFG_FILE SETTING DEFAULT"
  41         exit 1
  42     fi
  43
  44     local cfg_file="$1"
  45     local setting="$2"
  46     local default="$3"
  47
  48     if [ ! -f "$cfg_file" ]; then
  49         >&2 echo "ERROR: Configuration file $cfg_file not found."
  50         exit 1
  51     fi
  52
  53     cfg=$(grep "${setting^^}" "$cfg_file" | tail -1 | awk -F'=' '{print $2}')
  54     cfg=${cfg:-"$default"}
  55     echo "$cfg"
  56 }
  57 export get_cfg
  58
  59 get_cloud_cfg() {
  60     if [ -z $1 ]; then
  61         >&2 echo "Usage: get_cloud_cfg CFG_DIR"
  62         exit 1
  63     fi
  64
  65     local cfg_dir="$1"
  66     local silo="$2"
  67     local cfg_file="$cfg_dir/cloud.cfg"
  68
  69     cloud_name=$(basename "$cfg_dir")
  70     cloud_url=$(get_cfg "$cfg_file" CLOUD_URL "")
  71     cloud_ignore_ssl=$(get_cfg "$cfg_file" CLOUD_IGNORE_SSL "false")
  72     cloud_zone=$(get_cfg "$cfg_file" CLOUD_ZONE "")
  73     cloud_credential_id=$(get_cfg "$cfg_file" CLOUD_CREDENTIAL_ID "os-cloud")
  74
  75     echo "default_options = new SlaveOptions("
  76     get_minion_options "$cfg_file" "$silo"
  77     echo ")"
  78
  79     echo "cloud = new JCloudsCloud("
  80     echo "    \"$cloud_name\","
  81     echo "    \"$cloud_url\","
  82     echo "    $cloud_ignore_ssl,"
  83     echo "    \"$cloud_zone\","
  84     echo "    default_options,"
  85     echo "    templates,"
  86     echo "    \"$cloud_credential_id\""
  87     echo ")"
  88 }
  89
  90 get_minion_options() {
  91     if [ -z $1 ]; then
  92         >&2 echo "Usage: get_minion_options CFG_FILE"
  93         exit 1
  94     fi
  95
  96     local cfg_file="$1"
  97     local silo="${2:-}"
  98
  99     # Create a flavor mapping to manage hardware_id until OpenStack Cloud
 100     # plugin supports using names
 101     declare -A flavors
 102     flavors["v1-standard-1"]="bbcb7eb5-5c8d-498f-9d7e-307c575d3566"
 103     flavors["v1-standard-2"]="ca2a6e9c-2236-4107-8905-7ae9427132ff"
 104     flavors["v1-standard-4"]="5cf64088-893b-46b5-9bb1-ee020277635d"
 105     flavors["v1-standard-8"]="6eec77b4-2286-4e3b-b3f0-cac67aa2c727"
 106     flavors["v1-standard-16"]="2f8730dd-7688-4b72-a512-99fb9a482414"
 107     flavors["v1-standard-32"]="0da688af-bb0c-4116-a158-cbf37240a8b1"
 108     flavors["v1-standard-48"]="69471d69-61fb-40dd-bdf3-e6b7f4e6daa3"
 109     flavors["v1-standard-64"]="0c1d9008-f546-4608-9e8f-f8bdaec8dddd"
 110     flavors["v1-standard-96"]="5741c775-92a4-4488-bd77-dd7b08e2be81"
 111     flavors["v1-standard-128"]="e82d0a5b-8031-4526-9a5d-a15f7b4d48ff"
 112     flavors["v2-standard-1"]="52a01f6b-e660-48b5-8c06-5fb2a0fab0ec"
 113     flavors["v2-standard-2"]="ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949"
 114     flavors["v2-standard-4"]="d9115351-defe-4fac-986b-1a1187e2c31c"
 115     flavors["v2-standard-8"]="e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb"
 116     flavors["v2-standard-16"]="9e4b01cd-6744-4120-aafe-1b5e17584919"
 117     flavors["v2-standard-360"]="f0d27f44-a410-4f0f-9781-d722f5b5489e"
 118     flavors["v2-highcpu-1"]="c04abb7a-2b61-4ed3-8ce8-6c40ad9df750"
 119     flavors["v2-highcpu-2"]="03bdf34e-8905-46bc-a4b9-8dbf94b6e06d"
 120     flavors["v2-highcpu-4"]="3b72e578-7875-4e0e-91b7-71ed292f3ca2"
 121     flavors["v2-highcpu-8"]="221de281-95ec-414f-8e42-c86c9e0b318d"
 122     flavors["v2-highcpu-16"]="ddd6863a-ef4f-475c-9aee-61d46898651d"
 123     flavors["v2-highcpu-32"]="21dfb8a3-c472-4a2c-a8e1-4da8de415ff8"
 124
 125     image_name=$(get_cfg "$cfg_file" IMAGE_NAME "")
 126     volume_size=$(get_cfg "$cfg_file" VOLUME_SIZE "")
 127     hardware_id=$(get_cfg "$cfg_file" HARDWARE_ID "")
 128     network_id=$(get_cfg "$cfg_file" NETWORK_ID "")
 129     user_data_id=$(get_cfg "$cfg_file" USER_DATA_ID "jenkins-init-script")
 130
 131     # Handle Sandbox systems that might have a different cap.
 132     instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null")
 133     if [ "$silo" == "sandbox" ]; then
 134         instance_cap=$(get_cfg "$cfg_file" SANDBOX_CAP "null")
 135     fi
 136
 137     floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "")
 138     security_groups=$(get_cfg "$cfg_file" SECURITY_GROUPS "default")
 139     availability_zone=$(get_cfg "$cfg_file" AVAILABILITY_ZONE "")
 140     start_timeout=$(get_cfg "$cfg_file" START_TIMEOUT "600000")
 141     key_pair_name=$(get_cfg "$cfg_file" KEY_PAIR_NAME "jenkins")
 142     num_executors=$(get_cfg "$cfg_file" NUM_EXECUTORS "1")
 143     jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "")
 144     fs_root=$(get_cfg "$cfg_file" FS_ROOT "/w")
 145     retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0")
 146
 147     if [ ! -z "$volume_size" ]; then
 148         echo "    new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
 149     else
 150         echo "    new BootSource.Image(\"$image_name\"),"
 151     fi
 152     echo "    \"${flavors[${hardware_id}]}\","
 153     echo "    \"$network_id\","
 154     echo "    \"$user_data_id\","
 155     echo "    $instance_cap,"
 156     echo "    \"$floating_ip_pool\","
 157     echo "    \"$security_groups\","
 158     echo "    \"$availability_zone\","
 159     echo "    $start_timeout,"
 160     echo "    \"$key_pair_name\","
 161     echo "    $num_executors,"
 162     echo "    \"$jvm_options\","
 163     echo "    \"$fs_root\","
 164     echo "    new LauncherFactory.SSH(\"jenkins\", \"\"),"
 165     echo "    $retention_time"
 166 }
 167
 168 get_template_cfg() {
 169     if [ -z $1 ]; then
 170         >&2 echo "Usage: get_template_cfg CFG_FILE"
 171         exit 1
 172     fi
 173
 174     local cfg_file="$1"
 175     local minion_prefix="${2:-}"
 176
 177     template_name=$(basename $cfg_file .cfg)
 178     labels=$(get_cfg "$cfg_file" LABELS "")
 179
 180     echo "minion_options = new SlaveOptions("
 181     get_minion_options "$cfg_file"
 182     echo ")"
 183
 184     echo "template = new JCloudsSlaveTemplate("
 185     # TODO: Figure out how to insert the "prd / snd" prefix into template name.
 186     echo "    \"${minion_prefix}${template_name}\","
 187     echo "    \"$template_name $labels\","
 188     echo "    minion_options,"
 189     echo ")"
 190 }
 191
 192 mapfile -t clouds < <(ls -d1 $OS_CLOUD_DIR/*/)
 193
 194 for silo in $silos; do
 195
 196     script_file="$SCRIPT_DIR/${silo}-cloud-cfg.groovy"
 197     cp "$GROOVY_SCRIPT_FILE" "$script_file"
 198
 199     # Linux Foundation Jenkins systems use "prd-" and "snd-" to mark
 200     # production and sandbox servers.
 201     if [ "$silo" == "releng" ] || [ "$silo" == "production" ]; then
 202         node_prefix="prd-"
 203     elif [ "$silo" == "sandbox" ]; then
 204         node_prefix="snd-"
 205     else
 206         node_prefix="${silo}-"
 207     fi
 208
 209     echo "-----> Groovy script $script_file"
 210     for cloud in "${clouds[@]}"; do
 211         cfg_dir="${cloud}"
 212         echo "Processing $cfg_dir"
 213         insert_file="$SCRIPT_DIR/$silo/$(basename $cloud)/cloud-cfg.txt"
 214         mkdir -p "$(dirname $insert_file)"
 215         rm -f "$insert_file"
 216
 217         echo "" >> "$insert_file"
 218         echo "//////////////////////////////////////////////////" >> "$insert_file"
 219         echo "// Cloud config for $(basename $cloud)" >> "$insert_file"
 220         echo "//////////////////////////////////////////////////" >> "$insert_file"
 221         echo "" >> "$insert_file"
 222
 223         echo "templates = []" >> $insert_file
 224         mapfile -t templates < <(find $cfg_dir -maxdepth 1 -not -type d -not -name "cloud.cfg")
 225         for template in "${templates[@]}"; do
 226             get_template_cfg "$template" "$node_prefix" >> "$insert_file"
 227             echo "templates.add(template)" >> "$insert_file"
 228         done
 229
 230         get_cloud_cfg "$cfg_dir" "$silo" >> "$insert_file"
 231         echo "clouds.add(cloud)" >> "$insert_file"
 232
 233         cat "$insert_file" >> "$script_file"
 234     done
 235
 236     set +x  # Disable `set -x` to prevent printing passwords
 237     echo "Configuring $silo"
 238     JENKINS_URL=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" url)
 239     JENKINS_USER=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" user)
 240     JENKINS_PASSWORD=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" password)
 241     export JENKINS_URL
 242     export JENKINS_USER
 243     export JENKINS_PASSWORD
 244     lftools jenkins groovy "$script_file"
 245 done