Add the ability for jobs to use kubernetes clusters hosted on Vexx.
Issue: RELENG-2528
Signed-off-by: DW Talton <dtalton@contractor.linuxfoundation.org>
Change-Id: I730b0a57849bb0333cda6643c27d49148195bb7e
--- /dev/null
+#############################
+OpenStack Magnum (Kubernetes)
+#############################
+
+This section contains a series of macros for projects that need to spin up
+kubernetes clusters using JJB.
+
+Job Setup
+=========
+
+The two macros :ref:`lf-kubernetes-create` & :ref:`lf-kubernetes-delete` are
+companion macros and used together when constructing a job template that needs
+a kubernetes cluster.
+
+Example Usage:
+
+.. code-block:: yaml
+
+ - job-template:
+ name: k8s-test
+
+ #####################
+ # Default variables #
+ #####################
+
+ base-image: Fedora Atomic 29 [2019-08-20]
+ boot-volume-size: 10
+ cluster-settle-time: 1m
+ docker-volume-size: 10
+ fixed-network: ecompci
+ fixed-subnet: ecompci-subnet1
+ keypair: jenkins
+ kubernetes-version: v1.16.0
+ master-count: 1
+ master-flavor: v2-standard-1
+ node-count: 2
+ node-flavor: v2-highcpu-8
+ openstack-cloud: vex
+
+ #####################
+ # Job configuration #
+ #####################
+
+ builders:
+ - lf-infra-pre-build
+ - lf-kubernetes-create:
+ openstack-cloud: "{openstack-cloud}"
+ base-image: "{base-image}"
+ boot-volume-size: "{boot-volume-size}"
+ cluster-settle-time: "{cluster-settle-time}"
+ docker-volume-size: "{docker-volume-size}"
+ fixed-network: "{fixed-network}"
+ fixed-subnet: "{fixed-subnet}"
+ keypair: "{keypair}"
+ kubernetes-version: "{kubernetes-version}"
+ master-count: "{master-count}"
+ master-flavor: "{master-flavor}"
+ node-count: "{node-count}"
+ node-flavor: "{node-flavor}"
+ publishers:
+ - lf-kubernetes-delete
+
+
+Macros
+======
+
+.. _lf-kubernetes-create:
+
+lf-kubernetes-create
+--------------------
+
+Creates an OpenStack Kubernetes cluster as configured by the job. Name pattern
+of stack is ``$SILO-$JOB_NAME-$BUILD_NUMBER``.
+
+Requires ``lf-infra-pre-build`` macro to run first to install the
+``openstack`` and ``lftools`` packages.
+
+Requires a Config File Provider configuration for clouds.yaml named
+``clouds-yaml``.
+
+:Required Parameters:
+
+ :openstack-cloud: The ``OS_CLOUD`` variable to pass to OpenStack client.
+ (Docs: https://docs.openstack.org/python-openstackclient)
+ :base-image: The base image to use for building the cluster. LF is
+ using the Fedora Atomic images.
+ :boot-volume-size: The size of the operating system disk for each node.
+ :cluster-settle-time: A parameter that controls the buffer time after
+ cluster creation before we start querying the API for status.
+ :docker-volume-size: The size of the Docker volume.
+ :fixed-network: The private network to build the cluster on.
+ :fixed-subnet: The subnet to use from the above private network
+ :keypair: The ssh keypair to inject into the nodes for access.
+ :kubernetes-version: The version of kubernetes to use for the cluster.
+ Available versions are v1.14, v1.15, and v1.16
+ :master-count: The number of masters for the cluster (configuring more than
+ one master automatically triggers the creation of a load-balancer).
+ :master-flavor: The flavor (size) of the master node.
+ :node-count: The number of kubernetes nodes for the cluster.
+ :node-flavor: The flavor (size) of the worker nodes.
+
+
+.. _lf-kubernetes-delete:
+
+lf-kubernetes-delete
+--------------------
+
+Deletes the stack associated with this job. Name pattern of stack is
+``$SILO-$JOB_NAME-$BUILD_NUMBER``.
+
+Requires ``lf-infra-pre-build`` macro to run first to install the
+``openstack`` and ``lftools`` packages.
+
+Requires a Config File Provider configuration for clouds.yaml named
+``clouds-yaml``.
--- /dev/null
+---
+- parameter:
+ name: lf-kubernetes-create
+ parameters:
+ - string:
+ name: BASE_IMAGE
+ default: "{base-image}"
+ - string:
+ name: BOOT_VOLUME_SIZE
+ default: "{boot-volume-size}"
+ - string:
+ name: CLUSTER_SETTLE_TIME
+ default: "{cluster-settle-time}"
+ - string:
+ name: DOCKER_VOLUME_SIZE
+ default: "{docker-volume-size}"
+ - string:
+ name: FIXED_NETWORK
+ default: "{fixed-network}"
+ - string:
+ name: FIXED_SUBNET
+ default: "{fixed-subnet}"
+ - string:
+ name: KEYPAIR
+ default: "{keypair}"
+ - string:
+ name: KUBERNETES_VERSION
+ default: "{kubernetes-version}"
+ - string:
+ name: MASTER_COUNT
+ default: "{master-count}"
+ - string:
+ name: MASTER_FLAVOR
+ default: "{master-flavor}"
+ - string:
+ name: NODE_COUNT
+ default: "{node-count}"
+ - string:
+ name: NODE_FLAVOR
+ default: "{node-flavor}"
+ - string:
+ name: OS_CLOUD
+ default: "{openstack-cloud}"
+
+- builder:
+ name: lf-kubernetes-create
+ builders:
+ - inject:
+ properties-content: |
+ BASE_IMAGE={base-image}
+ BOOT_VOLUME_SIZE={boot-volume-size}
+ CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER
+ CLUSTER_SETTLE_TIME={cluster-settle-time}
+ CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template
+ DOCKER_VOLUME_SIZE={docker-volume-size}
+ FIXED_NETWORK={fixed-network}
+ FIXED_SUBNET={fixed-subnet}
+ KEYPAIR={keypair}
+ KUBERNETES_VERSION={kubernetes-version}
+ MASTER_COUNT={master-count}
+ MASTER_FLAVOR={master-flavor}
+ NODE_COUNT={node-count}
+ NODE_FLAVOR={node-flavor}
+ OS_CLOUD={openstack-cloud}
+ - config-file-provider:
+ files:
+ - file-id: clouds-yaml
+ target: "$HOME/.config/openstack/clouds.yaml"
+ - shell: !include-raw-escape: ../shell/openstack-kubernetes-create.sh
+
+- publisher:
+ name: lf-kubernetes-delete
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - inject:
+ properties-content: |
+ CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER
+ CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template
+ - config-file-provider:
+ files:
+ - file-id: clouds-yaml
+ target: "$HOME/.config/openstack/clouds.yaml"
+ - shell: |
+ #!/bin/bash -l
+ echo "Deleting $CLUSTER_NAME"
+ set -eux -o pipefail
+ openstack --os-cloud "$OS_CLOUD" coe cluster delete "$CLUSTER_NAME"
+ echo "Cluster $CLUSTER_NAME now deleting, sleeping 5 minutes"
+ sleep 5m
+ echo "Deleting cluster template $CLUSTER_TEMPLATE_NAME"
+ openstack --os-cloud "$OS_CLOUD" coe cluster template delete "$CLUSTER_TEMPLATE_NAME"
+ mark-unstable-if-failed: false
--- /dev/null
+---
+features:
+ - |
+ Add the lf-kubernetes-create and lf-kubernetes delete macros which allow ad-hoc
+ kubernetes cluster creation with a selectable disk size for the docker volume.
--- /dev/null
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# shellcheck disable=SC2153,SC2034
+echo "---> Creating kubernetes cluster"
+
+set -eux -o pipefail
+
+os_cloud="${OS_CLOUD:-vex}"
+fixed_network="${FIXED_NETWORK}"
+fixed_subnet="${FIXED_SUBNET}"
+cluster_template_name="${CLUSTER_TEMPLATE_NAME}"
+cluster_name="${CLUSTER_NAME}"
+base_image="${BASE_IMAGE}"
+keypair="${KEYPAIR}"
+master_flavor="${MASTER_FLAVOR}"
+node_flavor="${NODE_FLAVOR}"
+master_count="${MASTER_COUNT:-1}"
+node_count="${NODE_COUNT:-2}"
+boot_volume_size="${BOOT_VOLUME_SIZE}"
+docker_volume_size="${DOCKER_VOLUME_SIZE}"
+k8s_version="${KUBERNETES_VERSION}"
+cluster_settle_time="${CLUSTER_SETTLE_TIME:-1m}"
+
+
+# Create the template for the cluster first. Returns the cluster ID as $template_uuid
+template_uuid=$(openstack coe cluster template create "$cluster_template_name" \
+ --os-cloud "$os_cloud" \
+ --image "$base_image" \
+ --keypair "$keypair" \
+ --external-network public \
+ --fixed-network "$fixed_network" \
+ --fixed-subnet "$fixed_subnet" \
+ --floating-ip-disabled \
+ --master-flavor "$master_flavor" \
+ --flavor "$node_flavor" \
+ --docker-volume-size "$docker_volume_size" \
+ --network-driver flannel \
+ --master-lb-enabled \
+ --volume-driver cinder \
+ --labels boot_volume_type=ssd,boot_volume_size="${boot_volume_size}",kube_version="${k8s_version}",kube_tag="${k8s_version}" \
+ --coe kubernetes \
+ -f value -c uuid | tail -1)
+
+# Create the kubernetes cluster
+cluster_uuid=$(openstack coe cluster create "$cluster_name" \
+ --os-cloud "$os_cloud" \
+ --master-count "$master_count" \
+ --node-count "$node_count" \
+ --cluster-template "$template_uuid" | awk -F ' ' '{print $5}')
+
+# Sleep for a little, because sometimes OpenStack has to catch up with itself
+sleep 15
+
+while [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_IN_PROGRESS" ]
+do
+ # echo "sleeping $(date)"
+ sleep 2m
+done
+
+if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_FAILED" ]
+then
+ echo "Failed to create cluster: $cluster_uuid $(date)"
+ openstack --os-cloud "$os_cloud" coe cluster delete "$cluster_uuid"
+ sleep 5m
+ openstack --os-cloud "$os_cloud" coe cluster template delete "$template_uuid"
+ exit 1
+fi
+
+if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_COMPLETE" ]
+then
+ echo "Successfully created cluster: $cluster_uuid."
+fi
lftools[openstack]
python-heatclient
python-openstackclient
+python-magnumclient
+kubernetes
niet~=1.4.2
tox>=3.7.0 # Tox 3.7 or greater is necessary for parallel mode support
yq
Code Review / releng / global-jjb.git / commitdiff