From: DW Talton Date: Thu, 7 Nov 2019 00:00:01 +0000 (-0700) Subject: Add kubernetes cluster support X-Git-Tag: v0.49.0~23 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=e138a50dbef53b6612527df3c3fed9841c8a3ec2;p=releng%2Fglobal-jjb.git Add kubernetes cluster support Add the ability for jobs to use kubernetes clusters hosted on Vexx. Issue: RELENG-2528 Signed-off-by: DW Talton Change-Id: I730b0a57849bb0333cda6643c27d49148195bb7e --- diff --git a/docs/jjb/lf-openstack-kubernetes.rst b/docs/jjb/lf-openstack-kubernetes.rst new file mode 100644 index 00000000..cc6d7d48 --- /dev/null +++ b/docs/jjb/lf-openstack-kubernetes.rst @@ -0,0 +1,115 @@ +############################# +OpenStack Magnum (Kubernetes) +############################# + +This section contains a series of macros for projects that need to spin up +kubernetes clusters using JJB. + +Job Setup +========= + +The two macros :ref:`lf-kubernetes-create` & :ref:`lf-kubernetes-delete` are +companion macros and used together when constructing a job template that needs +a kubernetes cluster. + +Example Usage: + +.. code-block:: yaml + + - job-template: + name: k8s-test + + ##################### + # Default variables # + ##################### + + base-image: Fedora Atomic 29 [2019-08-20] + boot-volume-size: 10 + cluster-settle-time: 1m + docker-volume-size: 10 + fixed-network: ecompci + fixed-subnet: ecompci-subnet1 + keypair: jenkins + kubernetes-version: v1.16.0 + master-count: 1 + master-flavor: v2-standard-1 + node-count: 2 + node-flavor: v2-highcpu-8 + openstack-cloud: vex + + ##################### + # Job configuration # + ##################### + + builders: + - lf-infra-pre-build + - lf-kubernetes-create: + openstack-cloud: "{openstack-cloud}" + base-image: "{base-image}" + boot-volume-size: "{boot-volume-size}" + cluster-settle-time: "{cluster-settle-time}" + docker-volume-size: "{docker-volume-size}" + fixed-network: "{fixed-network}" + fixed-subnet: "{fixed-subnet}" + keypair: "{keypair}" + kubernetes-version: "{kubernetes-version}" + master-count: "{master-count}" + master-flavor: "{master-flavor}" + node-count: "{node-count}" + node-flavor: "{node-flavor}" + publishers: + - lf-kubernetes-delete + + +Macros +====== + +.. _lf-kubernetes-create: + +lf-kubernetes-create +-------------------- + +Creates an OpenStack Kubernetes cluster as configured by the job. Name pattern +of stack is ``$SILO-$JOB_NAME-$BUILD_NUMBER``. + +Requires ``lf-infra-pre-build`` macro to run first to install the +``openstack`` and ``lftools`` packages. + +Requires a Config File Provider configuration for clouds.yaml named +``clouds-yaml``. + +:Required Parameters: + + :openstack-cloud: The ``OS_CLOUD`` variable to pass to OpenStack client. + (Docs: https://docs.openstack.org/python-openstackclient) + :base-image: The base image to use for building the cluster. LF is + using the Fedora Atomic images. + :boot-volume-size: The size of the operating system disk for each node. + :cluster-settle-time: A parameter that controls the buffer time after + cluster creation before we start querying the API for status. + :docker-volume-size: The size of the Docker volume. + :fixed-network: The private network to build the cluster on. + :fixed-subnet: The subnet to use from the above private network + :keypair: The ssh keypair to inject into the nodes for access. + :kubernetes-version: The version of kubernetes to use for the cluster. + Available versions are v1.14, v1.15, and v1.16 + :master-count: The number of masters for the cluster (configuring more than + one master automatically triggers the creation of a load-balancer). + :master-flavor: The flavor (size) of the master node. + :node-count: The number of kubernetes nodes for the cluster. + :node-flavor: The flavor (size) of the worker nodes. + + +.. _lf-kubernetes-delete: + +lf-kubernetes-delete +-------------------- + +Deletes the stack associated with this job. Name pattern of stack is +``$SILO-$JOB_NAME-$BUILD_NUMBER``. + +Requires ``lf-infra-pre-build`` macro to run first to install the +``openstack`` and ``lftools`` packages. + +Requires a Config File Provider configuration for clouds.yaml named +``clouds-yaml``. diff --git a/jjb/lf-openstack-kubernetes.yaml b/jjb/lf-openstack-kubernetes.yaml new file mode 100644 index 00000000..91966096 --- /dev/null +++ b/jjb/lf-openstack-kubernetes.yaml @@ -0,0 +1,100 @@ +--- +- parameter: + name: lf-kubernetes-create + parameters: + - string: + name: BASE_IMAGE + default: "{base-image}" + - string: + name: BOOT_VOLUME_SIZE + default: "{boot-volume-size}" + - string: + name: CLUSTER_SETTLE_TIME + default: "{cluster-settle-time}" + - string: + name: DOCKER_VOLUME_SIZE + default: "{docker-volume-size}" + - string: + name: FIXED_NETWORK + default: "{fixed-network}" + - string: + name: FIXED_SUBNET + default: "{fixed-subnet}" + - string: + name: KEYPAIR + default: "{keypair}" + - string: + name: KUBERNETES_VERSION + default: "{kubernetes-version}" + - string: + name: MASTER_COUNT + default: "{master-count}" + - string: + name: MASTER_FLAVOR + default: "{master-flavor}" + - string: + name: NODE_COUNT + default: "{node-count}" + - string: + name: NODE_FLAVOR + default: "{node-flavor}" + - string: + name: OS_CLOUD + default: "{openstack-cloud}" + +- builder: + name: lf-kubernetes-create + builders: + - inject: + properties-content: | + BASE_IMAGE={base-image} + BOOT_VOLUME_SIZE={boot-volume-size} + CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER + CLUSTER_SETTLE_TIME={cluster-settle-time} + CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template + DOCKER_VOLUME_SIZE={docker-volume-size} + FIXED_NETWORK={fixed-network} + FIXED_SUBNET={fixed-subnet} + KEYPAIR={keypair} + KUBERNETES_VERSION={kubernetes-version} + MASTER_COUNT={master-count} + MASTER_FLAVOR={master-flavor} + NODE_COUNT={node-count} + NODE_FLAVOR={node-flavor} + OS_CLOUD={openstack-cloud} + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - shell: !include-raw-escape: ../shell/openstack-kubernetes-create.sh + +- publisher: + name: lf-kubernetes-delete + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - SUCCESS + - UNSTABLE + build-steps: + - inject: + properties-content: | + CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER + CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - shell: | + #!/bin/bash -l + echo "Deleting $CLUSTER_NAME" + set -eux -o pipefail + openstack --os-cloud "$OS_CLOUD" coe cluster delete "$CLUSTER_NAME" + echo "Cluster $CLUSTER_NAME now deleting, sleeping 5 minutes" + sleep 5m + echo "Deleting cluster template $CLUSTER_TEMPLATE_NAME" + openstack --os-cloud "$OS_CLOUD" coe cluster template delete "$CLUSTER_TEMPLATE_NAME" + mark-unstable-if-failed: false diff --git a/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml b/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml new file mode 100644 index 00000000..24920ca2 --- /dev/null +++ b/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add the lf-kubernetes-create and lf-kubernetes delete macros which allow ad-hoc + kubernetes cluster creation with a selectable disk size for the docker volume. diff --git a/shell/openstack-kubernetes-create.sh b/shell/openstack-kubernetes-create.sh new file mode 100755 index 00000000..5ee05b09 --- /dev/null +++ b/shell/openstack-kubernetes-create.sh @@ -0,0 +1,80 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# shellcheck disable=SC2153,SC2034 +echo "---> Creating kubernetes cluster" + +set -eux -o pipefail + +os_cloud="${OS_CLOUD:-vex}" +fixed_network="${FIXED_NETWORK}" +fixed_subnet="${FIXED_SUBNET}" +cluster_template_name="${CLUSTER_TEMPLATE_NAME}" +cluster_name="${CLUSTER_NAME}" +base_image="${BASE_IMAGE}" +keypair="${KEYPAIR}" +master_flavor="${MASTER_FLAVOR}" +node_flavor="${NODE_FLAVOR}" +master_count="${MASTER_COUNT:-1}" +node_count="${NODE_COUNT:-2}" +boot_volume_size="${BOOT_VOLUME_SIZE}" +docker_volume_size="${DOCKER_VOLUME_SIZE}" +k8s_version="${KUBERNETES_VERSION}" +cluster_settle_time="${CLUSTER_SETTLE_TIME:-1m}" + + +# Create the template for the cluster first. Returns the cluster ID as $template_uuid +template_uuid=$(openstack coe cluster template create "$cluster_template_name" \ + --os-cloud "$os_cloud" \ + --image "$base_image" \ + --keypair "$keypair" \ + --external-network public \ + --fixed-network "$fixed_network" \ + --fixed-subnet "$fixed_subnet" \ + --floating-ip-disabled \ + --master-flavor "$master_flavor" \ + --flavor "$node_flavor" \ + --docker-volume-size "$docker_volume_size" \ + --network-driver flannel \ + --master-lb-enabled \ + --volume-driver cinder \ + --labels boot_volume_type=ssd,boot_volume_size="${boot_volume_size}",kube_version="${k8s_version}",kube_tag="${k8s_version}" \ + --coe kubernetes \ + -f value -c uuid | tail -1) + +# Create the kubernetes cluster +cluster_uuid=$(openstack coe cluster create "$cluster_name" \ + --os-cloud "$os_cloud" \ + --master-count "$master_count" \ + --node-count "$node_count" \ + --cluster-template "$template_uuid" | awk -F ' ' '{print $5}') + +# Sleep for a little, because sometimes OpenStack has to catch up with itself +sleep 15 + +while [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_IN_PROGRESS" ] +do + # echo "sleeping $(date)" + sleep 2m +done + +if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_FAILED" ] +then + echo "Failed to create cluster: $cluster_uuid $(date)" + openstack --os-cloud "$os_cloud" coe cluster delete "$cluster_uuid" + sleep 5m + openstack --os-cloud "$os_cloud" coe cluster template delete "$template_uuid" + exit 1 +fi + +if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_COMPLETE" ] +then + echo "Successfully created cluster: $cluster_uuid." +fi diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index 660d5f26..b388b0f6 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -47,6 +47,8 @@ else lftools[openstack] python-heatclient python-openstackclient +python-magnumclient +kubernetes niet~=1.4.2 tox>=3.7.0 # Tox 3.7 or greater is necessary for parallel mode support yq