2 # SPDX-License-Identifier: EPL-1.0
3 ##############################################################################
4 # Copyright (c) 2018 The Linux Foundation and others.
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Eclipse Public License v1.0
8 # which accompanies this distribution, and is available at
9 # http://www.eclipse.org/legal/epl-v10.html
10 ##############################################################################
11 # Pulls global variable definitions out of a file.
13 # Configuration is read from $WORKSPACE/jenkins-config/clouds/openstack/$cloud/cloud.cfg
15 # Requirements: lftools must be installed to /tmp/v/lftools
19 # WORKSPACE: The path to the local ci-management repository.
20 # jenkins_silos: Space separated list of Jenkins silos to push
21 # configuration to. This must match a configuration section
22 # in the config file located at
23 # ~/.config/jenkins_jobs/jenkins_jobs.ini config file.
26 # Local testing can be performed by exporting the parameters "WORKSPACE" and
27 # "jenkins_silos" as environment variables. For example:
29 # export WORKSPACE=/tmp/ci-management
30 # export jenkins_silos=sandbox
31 # bash ./jjb/global-jjb/shell/jenkins-configure-clouds.sh
32 echo "---> jenkins-configure-clouds.sh"
34 if [ ! -d "$WORKSPACE/jenkins-config/clouds" ]; then
35 echo "WARN: jenkins-config/clouds does not exist. Skipping cloud management..."
39 #shellcheck source=/tmp/v/lftools/bin/activate disable=SC1091
40 source "/tmp/v/lftools/bin/activate"
42 GROOVY_SCRIPT_FILE="jjb/global-jjb/jenkins-admin/manage_clouds.groovy"
43 OS_CLOUD_DIR="$WORKSPACE/jenkins-config/clouds/openstack"
44 SCRIPT_DIR="$WORKSPACE/archives/groovy-inserts"
45 mkdir -p "$SCRIPT_DIR"
47 silos="${jenkins_silos:-jenkins}"
51 version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
54 if [ -z ${3+x} ]; then
55 >&2 echo "Usage: get_cfg CFG_FILE SETTING DEFAULT"
63 if [ ! -f "$cfg_file" ]; then
64 >&2 echo "ERROR: Configuration file $cfg_file not found."
68 cfg=$(grep "^${setting^^}=" "$cfg_file" | tail -1 | awk -F'=' '{print $2}')
69 cfg=${cfg:-"$default"}
76 >&2 echo "Usage: get_cloud_cfg CFG_DIR"
82 local cfg_file="$cfg_dir/cloud.cfg"
84 cloud_name=$(basename "$cfg_dir")
85 cloud_url=$(get_cfg "$cfg_file" CLOUD_URL "")
86 cloud_ignore_ssl=$(get_cfg "$cfg_file" CLOUD_IGNORE_SSL "false")
87 cloud_zone=$(get_cfg "$cfg_file" CLOUD_ZONE "")
88 cloud_credential_id=$(get_cfg "$cfg_file" CLOUD_CREDENTIAL_ID "os-cloud")
90 echo "default_options = new SlaveOptions("
91 get_minion_options "$cfg_file" "$silo"
94 echo "cloud = new JCloudsCloud("
95 echo " \"$cloud_name\","
96 echo " \"$cloud_url\","
97 echo " $cloud_ignore_ssl,"
98 echo " \"$cloud_zone\","
99 echo " default_options,"
101 echo " \"$cloud_credential_id\""
105 get_launcher_factory() {
107 >&2 echo "Usage: get_launcher_factory JNLP|SSH"
111 local connection_type="$1"
113 if [ "$connection_type" == "JNLP" ]; then
114 echo "new LauncherFactory.JNLP()"
115 elif [ "$connection_type" == "SSH" ]; then
116 echo "new LauncherFactory.SSH(\"$key_pair_name\", \"\")"
118 >&2 echo "Unknown connection type $connection_type"
123 get_minion_options() {
125 >&2 echo "Usage: get_minion_options CFG_FILE"
132 # Create a flavor mapping to manage hardware_id until OpenStack Cloud
133 # plugin supports using names
135 flavors["v1-standard-1"]="bbcb7eb5-5c8d-498f-9d7e-307c575d3566"
136 flavors["v1-standard-2"]="ca2a6e9c-2236-4107-8905-7ae9427132ff"
137 flavors["v1-standard-4"]="5cf64088-893b-46b5-9bb1-ee020277635d"
138 flavors["v1-standard-8"]="6eec77b4-2286-4e3b-b3f0-cac67aa2c727"
139 flavors["v1-standard-16"]="2f8730dd-7688-4b72-a512-99fb9a482414"
140 flavors["v1-standard-32"]="0da688af-bb0c-4116-a158-cbf37240a8b1"
141 flavors["v1-standard-48"]="69471d69-61fb-40dd-bdf3-e6b7f4e6daa3"
142 flavors["v1-standard-64"]="0c1d9008-f546-4608-9e8f-f8bdaec8dddd"
143 flavors["v1-standard-96"]="5741c775-92a4-4488-bd77-dd7b08e2be81"
144 flavors["v1-standard-128"]="e82d0a5b-8031-4526-9a5d-a15f7b4d48ff"
145 flavors["v2-standard-1"]="52a01f6b-e660-48b5-8c06-5fb2a0fab0ec"
146 flavors["v2-standard-2"]="ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949"
147 flavors["v2-standard-4"]="d9115351-defe-4fac-986b-1a1187e2c31c"
148 flavors["v2-standard-8"]="e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb"
149 flavors["v2-standard-16"]="9e4b01cd-6744-4120-aafe-1b5e17584919"
150 flavors["v2-standard-360"]="f0d27f44-a410-4f0f-9781-d722f5b5489e"
151 flavors["v2-highcpu-1"]="c04abb7a-2b61-4ed3-8ce8-6c40ad9df750"
152 flavors["v2-highcpu-2"]="03bdf34e-8905-46bc-a4b9-8dbf94b6e06d"
153 flavors["v2-highcpu-4"]="3b72e578-7875-4e0e-91b7-71ed292f3ca2"
154 flavors["v2-highcpu-8"]="221de281-95ec-414f-8e42-c86c9e0b318d"
155 flavors["v2-highcpu-16"]="ddd6863a-ef4f-475c-9aee-61d46898651d"
156 flavors["v2-highcpu-32"]="21dfb8a3-c472-4a2c-a8e1-4da8de415ff8"
157 flavors["odl-highcpu-2"]="def1b86f-b7f8-4943-b430-4a0599170006"
158 flavors["odl-highcpu-4"]="0c8ec795-2ff8-4623-98cf-b4c1d92bb37c"
159 flavors["odl-highcpu-8"]="458d6499-e2c8-4580-aa88-a4a04a33ee25"
160 flavors["odl-standard-1"]="35800a3f-0c69-428d-b5cb-136d17d46c48"
161 flavors["odl-standard-2"]="8ead227a-acfe-4290-be70-fbab92e6dd2f"
162 flavors["odl-standard-4"]="f76fb18d-d5fb-4175-95c1-b29d8039d102"
163 flavors["odl-standard-8"]="ba38b1af-4f87-4e4e-860e-94e8329d0d78"
165 image_name=$(get_cfg "$cfg_file" IMAGE_NAME "")
166 volume_size=$(get_cfg "$cfg_file" VOLUME_SIZE "")
167 hardware_id=$(get_cfg "$cfg_file" HARDWARE_ID "")
168 network_id=$(get_cfg "$cfg_file" NETWORK_ID "")
170 udi_default="$(get_cfg "$(dirname $cfg_file)/cloud.cfg" USER_DATA_ID "jenkins-init-script")"
171 user_data_id=$(get_cfg "$cfg_file" USER_DATA_ID "$udi_default")
173 # Handle Sandbox systems that might have a different cap.
174 instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null")
175 if [ "$silo" == "sandbox" ]; then
176 instance_cap=$(get_cfg "$cfg_file" SANDBOX_CAP "null")
178 min_instance_cap=$(get_cfg "$cfg_file" MIN_INSTANCE_CAP "null")
180 floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "")
181 security_groups=$(get_cfg "$cfg_file" SECURITY_GROUPS "default")
182 availability_zone=$(get_cfg "$cfg_file" AVAILABILITY_ZONE "")
183 start_timeout=$(get_cfg "$cfg_file" START_TIMEOUT "600000")
185 kpn_default="$(get_cfg "$(dirname $cfg_file)/cloud.cfg" KEY_PAIR_NAME "jenkins-ssh")"
186 key_pair_name=$(get_cfg "$cfg_file" KEY_PAIR_NAME "$kpn_default")
188 num_executors=$(get_cfg "$cfg_file" NUM_EXECUTORS "1")
189 jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "")
190 fs_root=$(get_cfg "$cfg_file" FS_ROOT "/w")
191 retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0")
192 connection_type=$(get_cfg "$cfg_file" CONNECTION_TYPE "SSH")
193 launcher_factory=$(get_launcher_factory "$connection_type")
195 OS_PLUGIN_VER="$(lftools jenkins plugins list \
196 | grep -i 'OpenStack Cloud Plugin' | awk -F':' '{print $2}')"
197 if version_ge "$OS_PLUGIN_VER" "2.35"; then
198 if [ ! -z "$volume_size" ]; then
199 echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
201 echo " new BootSource.Image(\"$image_name\"),"
203 echo " \"${flavors[${hardware_id}]}\","
204 echo " \"$network_id\","
205 echo " \"$user_data_id\","
206 echo " $instance_cap,"
207 echo " $min_instance_cap,"
208 echo " \"$floating_ip_pool\","
209 echo " \"$security_groups\","
210 echo " \"$availability_zone\","
211 echo " $start_timeout,"
212 echo " \"$key_pair_name\","
213 echo " $num_executors,"
214 echo " \"$jvm_options\","
215 echo " \"$fs_root\","
216 echo " $launcher_factory,"
217 echo " $retention_time"
219 else # SlaveOptions() structure for versions <= 2.34
220 if [ ! -z "$volume_size" ]; then
221 echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
223 echo " new BootSource.Image(\"$image_name\"),"
225 echo " \"${flavors[${hardware_id}]}\","
226 echo " \"$network_id\","
227 echo " \"$user_data_id\","
228 echo " $instance_cap,"
229 echo " \"$floating_ip_pool\","
230 echo " \"$security_groups\","
231 echo " \"$availability_zone\","
232 echo " $start_timeout,"
233 echo " \"$key_pair_name\","
234 echo " $num_executors,"
235 echo " \"$jvm_options\","
236 echo " \"$fs_root\","
237 echo " $launcher_factory,"
238 echo " $retention_time"
244 >&2 echo "Usage: get_template_cfg CFG_FILE SILO [MINION_PREFIX]"
250 local minion_prefix="${3:-}"
253 template_name=$(basename $cfg_file .cfg)
254 labels=$(get_cfg "$cfg_file" LABELS "")
256 echo "minion_options = new SlaveOptions("
257 get_minion_options "$cfg_file" "$silo"
260 echo "template = new JCloudsSlaveTemplate("
261 # TODO: Figure out how to insert the "prd / snd" prefix into template name.
262 echo " \"${minion_prefix}${template_name}\","
263 echo " \"$template_name $labels\","
264 echo " minion_options,"
268 mapfile -t clouds < <(ls -d1 $OS_CLOUD_DIR/*/)
270 for silo in $silos; do
272 script_file="$SCRIPT_DIR/${silo}-cloud-cfg.groovy"
273 cp "$GROOVY_SCRIPT_FILE" "$script_file"
275 # Linux Foundation Jenkins systems use "prd-" and "snd-" to mark
276 # production and sandbox servers.
277 if [ "$silo" == "releng" ] || [ "$silo" == "production" ]; then
279 elif [ "$silo" == "sandbox" ]; then
282 node_prefix="${silo}-"
285 set +x # Disable `set -x` to prevent printing passwords
286 echo "Configuring $silo"
287 JENKINS_URL=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" url)
288 JENKINS_USER=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" user)
289 JENKINS_PASSWORD=$(crudini --get "$HOME"/.config/jenkins_jobs/jenkins_jobs.ini "$silo" password)
292 export JENKINS_PASSWORD
294 echo "-----> Groovy script $script_file"
295 for cloud in "${clouds[@]}"; do
297 echo "Processing $cfg_dir"
298 insert_file="$SCRIPT_DIR/$silo/$(basename $cloud)/cloud-cfg.txt"
299 mkdir -p "$(dirname $insert_file)"
302 echo "" >> "$insert_file"
303 echo "//////////////////////////////////////////////////" >> "$insert_file"
304 echo "// Cloud config for $(basename $cloud)" >> "$insert_file"
305 echo "//////////////////////////////////////////////////" >> "$insert_file"
306 echo "" >> "$insert_file"
308 echo "templates = []" >> $insert_file
309 mapfile -t templates < <(find $cfg_dir -maxdepth 1 -not -type d -not -name "cloud.cfg")
310 for template in "${templates[@]}"; do
311 get_template_cfg "$template" "$silo" "$node_prefix" >> "$insert_file"
312 echo "templates.add(template)" >> "$insert_file"
315 get_cloud_cfg "$cfg_dir" "$silo" >> "$insert_file"
316 echo "clouds.add(cloud)" >> "$insert_file"
318 cat "$insert_file" >> "$script_file"
321 lftools jenkins groovy "$script_file"