--- /dev/null
+#############################
+OpenStack Magnum (Kubernetes)
+#############################
+
+This section contains a series of macros for projects that need to spin up
+kubernetes clusters using JJB.
+
+Job Setup
+=========
+
+The two macros :ref:`lf-kubernetes-create` & :ref:`lf-kubernetes-delete` are
+companion macros and used together when constructing a job template that needs
+a kubernetes cluster.
+
+Example Usage:
+
+.. code-block:: yaml
+
+ - job-template:
+ name: k8s-test
+
+ #####################
+ # Default variables #
+ #####################
+
+ base-image: Fedora Atomic 29 [2019-08-20]
+ boot-volume-size: 10
+ cluster-settle-time: 1m
+ docker-volume-size: 10
+ fixed-network: ecompci
+ fixed-subnet: ecompci-subnet1
+ keypair: jenkins
+ kubernetes-version: v1.16.0
+ master-count: 1
+ master-flavor: v2-standard-1
+ node-count: 2
+ node-flavor: v2-highcpu-8
+ openstack-cloud: vex
+
+ #####################
+ # Job configuration #
+ #####################
+
+ builders:
+ - lf-infra-pre-build
+ - lf-kubernetes-create:
+ openstack-cloud: "{openstack-cloud}"
+ base-image: "{base-image}"
+ boot-volume-size: "{boot-volume-size}"
+ cluster-settle-time: "{cluster-settle-time}"
+ docker-volume-size: "{docker-volume-size}"
+ fixed-network: "{fixed-network}"
+ fixed-subnet: "{fixed-subnet}"
+ keypair: "{keypair}"
+ kubernetes-version: "{kubernetes-version}"
+ master-count: "{master-count}"
+ master-flavor: "{master-flavor}"
+ node-count: "{node-count}"
+ node-flavor: "{node-flavor}"
+ publishers:
+ - lf-kubernetes-delete
+
+
+Macros
+======
+
+.. _lf-kubernetes-create:
+
+lf-kubernetes-create
+--------------------
+
+Creates an OpenStack Kubernetes cluster as configured by the job. Name pattern
+of stack is ``$SILO-$JOB_NAME-$BUILD_NUMBER``.
+
+Requires ``lf-infra-pre-build`` macro to run first to install the
+``openstack`` and ``lftools`` packages.
+
+Requires a Config File Provider configuration for clouds.yaml named
+``clouds-yaml``.
+
+:Required Parameters:
+
+ :openstack-cloud: The ``OS_CLOUD`` variable to pass to OpenStack client.
+ (Docs: https://docs.openstack.org/python-openstackclient)
+ :base-image: The base image to use for building the cluster. LF is
+ using the Fedora Atomic images.
+ :boot-volume-size: The size of the operating system disk for each node.
+ :cluster-settle-time: A parameter that controls the buffer time after
+ cluster creation before we start querying the API for status.
+ :docker-volume-size: The size of the Docker volume.
+ :fixed-network: The private network to build the cluster on.
+ :fixed-subnet: The subnet to use from the above private network
+ :keypair: The ssh keypair to inject into the nodes for access.
+ :kubernetes-version: The version of kubernetes to use for the cluster.
+ Available versions are v1.14, v1.15, and v1.16
+ :master-count: The number of masters for the cluster (configuring more than
+ one master automatically triggers the creation of a load-balancer).
+ :master-flavor: The flavor (size) of the master node.
+ :node-count: The number of kubernetes nodes for the cluster.
+ :node-flavor: The flavor (size) of the worker nodes.
+
+
+.. _lf-kubernetes-delete:
+
+lf-kubernetes-delete
+--------------------
+
+Deletes the stack associated with this job. Name pattern of stack is
+``$SILO-$JOB_NAME-$BUILD_NUMBER``.
+
+Requires ``lf-infra-pre-build`` macro to run first to install the
+``openstack`` and ``lftools`` packages.
+
+Requires a Config File Provider configuration for clouds.yaml named
+``clouds-yaml``.
--- /dev/null
+---
+- parameter:
+ name: lf-kubernetes-create
+ parameters:
+ - string:
+ name: BASE_IMAGE
+ default: "{base-image}"
+ - string:
+ name: BOOT_VOLUME_SIZE
+ default: "{boot-volume-size}"
+ - string:
+ name: CLUSTER_SETTLE_TIME
+ default: "{cluster-settle-time}"
+ - string:
+ name: DOCKER_VOLUME_SIZE
+ default: "{docker-volume-size}"
+ - string:
+ name: FIXED_NETWORK
+ default: "{fixed-network}"
+ - string:
+ name: FIXED_SUBNET
+ default: "{fixed-subnet}"
+ - string:
+ name: KEYPAIR
+ default: "{keypair}"
+ - string:
+ name: KUBERNETES_VERSION
+ default: "{kubernetes-version}"
+ - string:
+ name: MASTER_COUNT
+ default: "{master-count}"
+ - string:
+ name: MASTER_FLAVOR
+ default: "{master-flavor}"
+ - string:
+ name: NODE_COUNT
+ default: "{node-count}"
+ - string:
+ name: NODE_FLAVOR
+ default: "{node-flavor}"
+ - string:
+ name: OS_CLOUD
+ default: "{openstack-cloud}"
+
+- builder:
+ name: lf-kubernetes-create
+ builders:
+ - inject:
+ properties-content: |
+ BASE_IMAGE={base-image}
+ BOOT_VOLUME_SIZE={boot-volume-size}
+ CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER
+ CLUSTER_SETTLE_TIME={cluster-settle-time}
+ CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template
+ DOCKER_VOLUME_SIZE={docker-volume-size}
+ FIXED_NETWORK={fixed-network}
+ FIXED_SUBNET={fixed-subnet}
+ KEYPAIR={keypair}
+ KUBERNETES_VERSION={kubernetes-version}
+ MASTER_COUNT={master-count}
+ MASTER_FLAVOR={master-flavor}
+ NODE_COUNT={node-count}
+ NODE_FLAVOR={node-flavor}
+ OS_CLOUD={openstack-cloud}
+ - config-file-provider:
+ files:
+ - file-id: clouds-yaml
+ target: "$HOME/.config/openstack/clouds.yaml"
+ - shell: !include-raw-escape: ../shell/openstack-kubernetes-create.sh
+
+- publisher:
+ name: lf-kubernetes-delete
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - inject:
+ properties-content: |
+ CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER
+ CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template
+ - config-file-provider:
+ files:
+ - file-id: clouds-yaml
+ target: "$HOME/.config/openstack/clouds.yaml"
+ - shell: |
+ #!/bin/bash -l
+ echo "Deleting $CLUSTER_NAME"
+ set -eux -o pipefail
+ openstack --os-cloud "$OS_CLOUD" coe cluster delete "$CLUSTER_NAME"
+ echo "Cluster $CLUSTER_NAME now deleting, sleeping 5 minutes"
+ sleep 5m
+ echo "Deleting cluster template $CLUSTER_TEMPLATE_NAME"
+ openstack --os-cloud "$OS_CLOUD" coe cluster template delete "$CLUSTER_TEMPLATE_NAME"
+ mark-unstable-if-failed: false
--- /dev/null
+---
+prelude: >
+ This release supports breaking changes in the upstream OpenStack Jenkins
+ plugin. Version 2.47+ of the OpenStack Cloud plugin for Jenkins adds two new
+ SlaveOptions params, node_properties and config_drive. Failure to send
+ these params results in a failed request.
+features:
+ - |
+ Add support for these new parameters, node_properties and config_drive. Add
+ a new function test_version, which will do a numerical (float) comparison
+ between installed plugin versions and user-supplied test version numbers.
+upgrade:
+ - |
+ If you are using the INSTANCE_MIN_CAPMAX paramater in your cloud configs,
+ you will need to change it to INSTANCE_MIN when using v2.47+ of the plugin.
+deprecations:
+ - |
+ Deprecated the config option INSTANCE_MIN_CAPMAX, replaced with INSTANCE_MIN
+ when using v2.47+ of the OpenStack plugin.
--- /dev/null
+---
+features:
+ - |
+ Add the lf-kubernetes-create and lf-kubernetes delete macros which allow ad-hoc
+ kubernetes cluster creation with a selectable disk size for the docker volume.
silos="${jenkins_silos:-jenkins}"
+
set -eu -o pipefail
-version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
+os_plugin_version="$(lftools jenkins plugins list \
+ | grep -i 'OpenStack Cloud Plugin')"
+
+testversion() {
+ local current_val="$1" operator="$2" test_value="$3"
+ awk -vv1="$current_val" -vv2="$test_value" 'BEGIN {
+ split(v1, a, /\:/);
+ if (a[2] == '$test_value') {
+ exit (a[2] == '$test_value') ? 0 : 1
+ }
+ else {
+ exit (a[2] '$operator' '$test_value') ? 0 : 1
+ }
+ }'
+}
get_cfg() {
if [ -z ${3+x} ]; then
}
export get_cfg
+
get_cloud_cfg() {
if [ -z "$1" ]; then
>&2 echo "Usage: get_cloud_cfg CFG_DIR"
echo ")"
}
+
get_launcher_factory() {
if [ -z "$1" ]; then
>&2 echo "Usage: get_launcher_factory JNLP|SSH"
fi
}
+
get_minion_options() {
if [ -z "$1" ]; then
>&2 echo "Usage: get_minion_options CFG_FILE"
flavors["v2-standard-16"]="9e4b01cd-6744-4120-aafe-1b5e17584919"
flavors["v2-standard-360"]="f0d27f44-a410-4f0f-9781-d722f5b5489e"
+
image_name=$(get_cfg "$cfg_file" IMAGE_NAME "")
volume_size=$(get_cfg "$cfg_file" VOLUME_SIZE "")
hardware_id=$(get_cfg "$cfg_file" HARDWARE_ID "")
network_id=$(get_cfg "$cfg_file" NETWORK_ID "")
-
udi_default="$(get_cfg "$(dirname "$cfg_file")/cloud.cfg" USER_DATA_ID "jenkins-init-script")"
user_data_id=$(get_cfg "$cfg_file" USER_DATA_ID "$udi_default")
# Handle Sandbox systems that might have a different cap.
- instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null")
if [ "$silo" == "sandbox" ]; then
instance_cap=$(get_cfg "$cfg_file" SANDBOX_CAP "null")
+ else
+ instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null")
fi
- min_instance_cap=$(get_cfg "$cfg_file" MIN_INSTANCE_CAP "null")
- floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "")
+ floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "null")
security_groups=$(get_cfg "$cfg_file" SECURITY_GROUPS "default")
availability_zone=$(get_cfg "$cfg_file" AVAILABILITY_ZONE "")
start_timeout=$(get_cfg "$cfg_file" START_TIMEOUT "600000")
-
kpn_default="$(get_cfg "$(dirname "$cfg_file")/cloud.cfg" KEY_PAIR_NAME "jenkins-ssh")"
key_pair_name=$(get_cfg "$cfg_file" KEY_PAIR_NAME "$kpn_default")
-
num_executors=$(get_cfg "$cfg_file" NUM_EXECUTORS "1")
- jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "")
+ jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "null")
fs_root=$(get_cfg "$cfg_file" FS_ROOT "/w")
- retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0")
connection_type=$(get_cfg "$cfg_file" CONNECTION_TYPE "SSH")
launcher_factory=$(get_launcher_factory "$connection_type")
+ node_properties=$(get_cfg "$cfg_file" NODE_PROPERTIES, "null")
+ retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0")
+ config_drive=$(get_cfg "$cfg_file" CONFIG_DRIVE, "null")
- OS_PLUGIN_VER="$(lftools jenkins plugins list \
- | grep -i 'OpenStack Cloud Plugin' \
- | awk -F':' '{print $2}' | awk -F' ' '{print $1}')"
- if version_ge "$OS_PLUGIN_VER" "2.35"; then
- if [ -n "$volume_size" ]; then
- echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
- else
- echo " new BootSource.Image(\"$image_name\"),"
- fi
- echo " \"${flavors[${hardware_id}]}\","
- echo " \"$network_id\","
- echo " \"$user_data_id\","
- echo " $instance_cap,"
- echo " $min_instance_cap,"
- echo " \"$floating_ip_pool\","
- echo " \"$security_groups\","
- echo " \"$availability_zone\","
- echo " $start_timeout,"
- echo " \"$key_pair_name\","
- echo " $num_executors,"
- echo " \"$jvm_options\","
- echo " \"$fs_root\","
- echo " $launcher_factory,"
- echo " $retention_time"
-
- else # SlaveOptions() structure for versions <= 2.34
- if [ -n "$volume_size" ]; then
- echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
- else
- echo " new BootSource.Image(\"$image_name\"),"
- fi
- echo " \"${flavors[${hardware_id}]}\","
- echo " \"$network_id\","
- echo " \"$user_data_id\","
- echo " $instance_cap,"
- echo " \"$floating_ip_pool\","
- echo " \"$security_groups\","
- echo " \"$availability_zone\","
- echo " $start_timeout,"
- echo " \"$key_pair_name\","
- echo " $num_executors,"
- echo " \"$jvm_options\","
- echo " \"$fs_root\","
- echo " $launcher_factory,"
- echo " $retention_time"
+
+ if [ -n "$volume_size" ]; then
+ echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size),"
+ else
+ echo " new BootSource.Image(\"$image_name\"),"
+ fi
+
+ echo " \"${flavors[${hardware_id}]}\","
+ echo " \"$network_id\","
+ echo " \"$user_data_id\","
+ echo " $instance_cap,"
+
+ # Handle specifying the minimum instance count across different versions
+ if testversion "$os_plugin_version" '>=' '2.47'
+ then
+ instance_min=$(get_cfg "$cfg_file" INSTANCE_MIN "null")
+ echo " $instance_min,"
+ else
+ instance_min=$(get_cfg "$cfg_file" INSTANCE_MIN_CAPMAX "null")
+ echo " $instance_min,"
fi
+
+ echo " \"$floating_ip_pool\","
+ echo " \"$security_groups\","
+ echo " \"$availability_zone\","
+ echo " $start_timeout,"
+ echo " \"$key_pair_name\","
+ echo " $num_executors,"
+ echo " \"$jvm_options\","
+ echo " \"$fs_root\","
+ echo " $launcher_factory,"
+
+ if testversion "$os_plugin_version" '>=' '2.47'
+ then
+ echo " $node_properties,"
+ echo " $retention_time",
+ echo " $config_drive"
+ else
+ echo " $retention_time"
+ fi
+
+
}
get_template_cfg() {
echo ")"
}
+
mapfile -t clouds < <(ls -d1 "$OS_CLOUD_DIR"/*/)
for silo in $silos; do
--- /dev/null
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# shellcheck disable=SC2153,SC2034
+echo "---> Creating kubernetes cluster"
+
+set -eux -o pipefail
+
+os_cloud="${OS_CLOUD:-vex}"
+fixed_network="${FIXED_NETWORK}"
+fixed_subnet="${FIXED_SUBNET}"
+cluster_template_name="${CLUSTER_TEMPLATE_NAME}"
+cluster_name="${CLUSTER_NAME}"
+base_image="${BASE_IMAGE}"
+keypair="${KEYPAIR}"
+master_flavor="${MASTER_FLAVOR}"
+node_flavor="${NODE_FLAVOR}"
+master_count="${MASTER_COUNT:-1}"
+node_count="${NODE_COUNT:-2}"
+boot_volume_size="${BOOT_VOLUME_SIZE}"
+docker_volume_size="${DOCKER_VOLUME_SIZE}"
+k8s_version="${KUBERNETES_VERSION}"
+cluster_settle_time="${CLUSTER_SETTLE_TIME:-1m}"
+
+
+# Create the template for the cluster first. Returns the cluster ID as $template_uuid
+template_uuid=$(openstack coe cluster template create "$cluster_template_name" \
+ --os-cloud "$os_cloud" \
+ --image "$base_image" \
+ --keypair "$keypair" \
+ --external-network public \
+ --fixed-network "$fixed_network" \
+ --fixed-subnet "$fixed_subnet" \
+ --floating-ip-disabled \
+ --master-flavor "$master_flavor" \
+ --flavor "$node_flavor" \
+ --docker-volume-size "$docker_volume_size" \
+ --network-driver flannel \
+ --master-lb-enabled \
+ --volume-driver cinder \
+ --labels boot_volume_type=ssd,boot_volume_size="${boot_volume_size}",kube_version="${k8s_version}",kube_tag="${k8s_version}" \
+ --coe kubernetes \
+ -f value -c uuid | tail -1)
+
+# Create the kubernetes cluster
+cluster_uuid=$(openstack coe cluster create "$cluster_name" \
+ --os-cloud "$os_cloud" \
+ --master-count "$master_count" \
+ --node-count "$node_count" \
+ --cluster-template "$template_uuid" | awk -F ' ' '{print $5}')
+
+# Sleep for a little, because sometimes OpenStack has to catch up with itself
+sleep 15
+
+while [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_IN_PROGRESS" ]
+do
+ # echo "sleeping $(date)"
+ sleep 2m
+done
+
+if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_FAILED" ]
+then
+ echo "Failed to create cluster: $cluster_uuid $(date)"
+ openstack --os-cloud "$os_cloud" coe cluster delete "$cluster_uuid"
+ sleep 5m
+ openstack --os-cloud "$os_cloud" coe cluster template delete "$template_uuid"
+ exit 1
+fi
+
+if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_COMPLETE" ]
+then
+ echo "Successfully created cluster: $cluster_uuid."
+fi
lftools[openstack]
python-heatclient
python-openstackclient
+python-magnumclient
+kubernetes
niet~=1.4.2
tox>=3.7.0 # Tox 3.7 or greater is necessary for parallel mode support
yq
Code Review / releng / global-jjb.git / commitdiff