From: Houa Yang Date: Tue, 26 Nov 2019 21:04:11 +0000 (+0000) Subject: Merge "Update lf-release-jobs gerrit trigger" X-Git-Tag: v0.49.0~20 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=e194613aea597ec57b65d169b9ba996eab9e6074;hp=9ce5ef214927e657dbd0d7cf6ed1ad613d642055;p=releng%2Fglobal-jjb.git Merge "Update lf-release-jobs gerrit trigger" --- diff --git a/docs/jjb/lf-openstack-kubernetes.rst b/docs/jjb/lf-openstack-kubernetes.rst new file mode 100644 index 00000000..cc6d7d48 --- /dev/null +++ b/docs/jjb/lf-openstack-kubernetes.rst @@ -0,0 +1,115 @@ +############################# +OpenStack Magnum (Kubernetes) +############################# + +This section contains a series of macros for projects that need to spin up +kubernetes clusters using JJB. + +Job Setup +========= + +The two macros :ref:`lf-kubernetes-create` & :ref:`lf-kubernetes-delete` are +companion macros and used together when constructing a job template that needs +a kubernetes cluster. + +Example Usage: + +.. code-block:: yaml + + - job-template: + name: k8s-test + + ##################### + # Default variables # + ##################### + + base-image: Fedora Atomic 29 [2019-08-20] + boot-volume-size: 10 + cluster-settle-time: 1m + docker-volume-size: 10 + fixed-network: ecompci + fixed-subnet: ecompci-subnet1 + keypair: jenkins + kubernetes-version: v1.16.0 + master-count: 1 + master-flavor: v2-standard-1 + node-count: 2 + node-flavor: v2-highcpu-8 + openstack-cloud: vex + + ##################### + # Job configuration # + ##################### + + builders: + - lf-infra-pre-build + - lf-kubernetes-create: + openstack-cloud: "{openstack-cloud}" + base-image: "{base-image}" + boot-volume-size: "{boot-volume-size}" + cluster-settle-time: "{cluster-settle-time}" + docker-volume-size: "{docker-volume-size}" + fixed-network: "{fixed-network}" + fixed-subnet: "{fixed-subnet}" + keypair: "{keypair}" + kubernetes-version: "{kubernetes-version}" + master-count: "{master-count}" + master-flavor: "{master-flavor}" + node-count: "{node-count}" + node-flavor: "{node-flavor}" + publishers: + - lf-kubernetes-delete + + +Macros +====== + +.. _lf-kubernetes-create: + +lf-kubernetes-create +-------------------- + +Creates an OpenStack Kubernetes cluster as configured by the job. Name pattern +of stack is ``$SILO-$JOB_NAME-$BUILD_NUMBER``. + +Requires ``lf-infra-pre-build`` macro to run first to install the +``openstack`` and ``lftools`` packages. + +Requires a Config File Provider configuration for clouds.yaml named +``clouds-yaml``. + +:Required Parameters: + + :openstack-cloud: The ``OS_CLOUD`` variable to pass to OpenStack client. + (Docs: https://docs.openstack.org/python-openstackclient) + :base-image: The base image to use for building the cluster. LF is + using the Fedora Atomic images. + :boot-volume-size: The size of the operating system disk for each node. + :cluster-settle-time: A parameter that controls the buffer time after + cluster creation before we start querying the API for status. + :docker-volume-size: The size of the Docker volume. + :fixed-network: The private network to build the cluster on. + :fixed-subnet: The subnet to use from the above private network + :keypair: The ssh keypair to inject into the nodes for access. + :kubernetes-version: The version of kubernetes to use for the cluster. + Available versions are v1.14, v1.15, and v1.16 + :master-count: The number of masters for the cluster (configuring more than + one master automatically triggers the creation of a load-balancer). + :master-flavor: The flavor (size) of the master node. + :node-count: The number of kubernetes nodes for the cluster. + :node-flavor: The flavor (size) of the worker nodes. + + +.. _lf-kubernetes-delete: + +lf-kubernetes-delete +-------------------- + +Deletes the stack associated with this job. Name pattern of stack is +``$SILO-$JOB_NAME-$BUILD_NUMBER``. + +Requires ``lf-infra-pre-build`` macro to run first to install the +``openstack`` and ``lftools`` packages. + +Requires a Config File Provider configuration for clouds.yaml named +``clouds-yaml``. diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 4a746157..3ec2b499 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -734,9 +734,7 @@ - inject: properties-content: JJB_WORKERS={jjb-workers} - shell: !include-raw-escape: - - ../shell/jjb-install.sh - ../shell/jjb-merge-job.sh - - ../shell/jjb-cleanup.sh - job-template: name: "{project-name}-jjb-merge" diff --git a/jjb/lf-openstack-kubernetes.yaml b/jjb/lf-openstack-kubernetes.yaml new file mode 100644 index 00000000..91966096 --- /dev/null +++ b/jjb/lf-openstack-kubernetes.yaml @@ -0,0 +1,100 @@ +--- +- parameter: + name: lf-kubernetes-create + parameters: + - string: + name: BASE_IMAGE + default: "{base-image}" + - string: + name: BOOT_VOLUME_SIZE + default: "{boot-volume-size}" + - string: + name: CLUSTER_SETTLE_TIME + default: "{cluster-settle-time}" + - string: + name: DOCKER_VOLUME_SIZE + default: "{docker-volume-size}" + - string: + name: FIXED_NETWORK + default: "{fixed-network}" + - string: + name: FIXED_SUBNET + default: "{fixed-subnet}" + - string: + name: KEYPAIR + default: "{keypair}" + - string: + name: KUBERNETES_VERSION + default: "{kubernetes-version}" + - string: + name: MASTER_COUNT + default: "{master-count}" + - string: + name: MASTER_FLAVOR + default: "{master-flavor}" + - string: + name: NODE_COUNT + default: "{node-count}" + - string: + name: NODE_FLAVOR + default: "{node-flavor}" + - string: + name: OS_CLOUD + default: "{openstack-cloud}" + +- builder: + name: lf-kubernetes-create + builders: + - inject: + properties-content: | + BASE_IMAGE={base-image} + BOOT_VOLUME_SIZE={boot-volume-size} + CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER + CLUSTER_SETTLE_TIME={cluster-settle-time} + CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template + DOCKER_VOLUME_SIZE={docker-volume-size} + FIXED_NETWORK={fixed-network} + FIXED_SUBNET={fixed-subnet} + KEYPAIR={keypair} + KUBERNETES_VERSION={kubernetes-version} + MASTER_COUNT={master-count} + MASTER_FLAVOR={master-flavor} + NODE_COUNT={node-count} + NODE_FLAVOR={node-flavor} + OS_CLOUD={openstack-cloud} + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - shell: !include-raw-escape: ../shell/openstack-kubernetes-create.sh + +- publisher: + name: lf-kubernetes-delete + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - SUCCESS + - UNSTABLE + build-steps: + - inject: + properties-content: | + CLUSTER_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER + CLUSTER_TEMPLATE_NAME=$SILO-$JOB_NAME-$BUILD_NUMBER-template + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - shell: | + #!/bin/bash -l + echo "Deleting $CLUSTER_NAME" + set -eux -o pipefail + openstack --os-cloud "$OS_CLOUD" coe cluster delete "$CLUSTER_NAME" + echo "Cluster $CLUSTER_NAME now deleting, sleeping 5 minutes" + sleep 5m + echo "Deleting cluster template $CLUSTER_TEMPLATE_NAME" + openstack --os-cloud "$OS_CLOUD" coe cluster template delete "$CLUSTER_TEMPLATE_NAME" + mark-unstable-if-failed: false diff --git a/releasenotes/notes/jenkins-configure-clouds-b0893c073c7a95e9.yaml b/releasenotes/notes/jenkins-configure-clouds-b0893c073c7a95e9.yaml new file mode 100644 index 00000000..8551f4c1 --- /dev/null +++ b/releasenotes/notes/jenkins-configure-clouds-b0893c073c7a95e9.yaml @@ -0,0 +1,19 @@ +--- +prelude: > + This release supports breaking changes in the upstream OpenStack Jenkins + plugin. Version 2.47+ of the OpenStack Cloud plugin for Jenkins adds two new + SlaveOptions params, node_properties and config_drive. Failure to send + these params results in a failed request. +features: + - | + Add support for these new parameters, node_properties and config_drive. Add + a new function test_version, which will do a numerical (float) comparison + between installed plugin versions and user-supplied test version numbers. +upgrade: + - | + If you are using the INSTANCE_MIN_CAPMAX paramater in your cloud configs, + you will need to change it to INSTANCE_MIN when using v2.47+ of the plugin. +deprecations: + - | + Deprecated the config option INSTANCE_MIN_CAPMAX, replaced with INSTANCE_MIN + when using v2.47+ of the OpenStack plugin. diff --git a/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml b/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml new file mode 100644 index 00000000..24920ca2 --- /dev/null +++ b/releasenotes/notes/lf-openstack-kubernetes-41bdc8d4bf86b05c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add the lf-kubernetes-create and lf-kubernetes delete macros which allow ad-hoc + kubernetes cluster creation with a selectable disk size for the docker volume. diff --git a/shell/jenkins-configure-clouds.sh b/shell/jenkins-configure-clouds.sh index 5a4e7a06..20092adb 100644 --- a/shell/jenkins-configure-clouds.sh +++ b/shell/jenkins-configure-clouds.sh @@ -43,9 +43,24 @@ mkdir -p "$SCRIPT_DIR" silos="${jenkins_silos:-jenkins}" + set -eu -o pipefail -version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; } +os_plugin_version="$(lftools jenkins plugins list \ + | grep -i 'OpenStack Cloud Plugin')" + +testversion() { + local current_val="$1" operator="$2" test_value="$3" + awk -vv1="$current_val" -vv2="$test_value" 'BEGIN { + split(v1, a, /\:/); + if (a[2] == '$test_value') { + exit (a[2] == '$test_value') ? 0 : 1 + } + else { + exit (a[2] '$operator' '$test_value') ? 0 : 1 + } + }' +} get_cfg() { if [ -z ${3+x} ]; then @@ -68,6 +83,7 @@ get_cfg() { } export get_cfg + get_cloud_cfg() { if [ -z "$1" ]; then >&2 echo "Usage: get_cloud_cfg CFG_DIR" @@ -99,6 +115,7 @@ get_cloud_cfg() { echo ")" } + get_launcher_factory() { if [ -z "$1" ]; then >&2 echo "Usage: get_launcher_factory JNLP|SSH" @@ -117,6 +134,7 @@ get_launcher_factory() { fi } + get_minion_options() { if [ -z "$1" ]; then >&2 echo "Usage: get_minion_options CFG_FILE" @@ -174,82 +192,78 @@ get_minion_options() { flavors["v2-standard-16"]="9e4b01cd-6744-4120-aafe-1b5e17584919" flavors["v2-standard-360"]="f0d27f44-a410-4f0f-9781-d722f5b5489e" + image_name=$(get_cfg "$cfg_file" IMAGE_NAME "") volume_size=$(get_cfg "$cfg_file" VOLUME_SIZE "") hardware_id=$(get_cfg "$cfg_file" HARDWARE_ID "") network_id=$(get_cfg "$cfg_file" NETWORK_ID "") - udi_default="$(get_cfg "$(dirname "$cfg_file")/cloud.cfg" USER_DATA_ID "jenkins-init-script")" user_data_id=$(get_cfg "$cfg_file" USER_DATA_ID "$udi_default") # Handle Sandbox systems that might have a different cap. - instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null") if [ "$silo" == "sandbox" ]; then instance_cap=$(get_cfg "$cfg_file" SANDBOX_CAP "null") + else + instance_cap=$(get_cfg "$cfg_file" INSTANCE_CAP "null") fi - min_instance_cap=$(get_cfg "$cfg_file" MIN_INSTANCE_CAP "null") - floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "") + floating_ip_pool=$(get_cfg "$cfg_file" FLOATING_IP_POOL "null") security_groups=$(get_cfg "$cfg_file" SECURITY_GROUPS "default") availability_zone=$(get_cfg "$cfg_file" AVAILABILITY_ZONE "") start_timeout=$(get_cfg "$cfg_file" START_TIMEOUT "600000") - kpn_default="$(get_cfg "$(dirname "$cfg_file")/cloud.cfg" KEY_PAIR_NAME "jenkins-ssh")" key_pair_name=$(get_cfg "$cfg_file" KEY_PAIR_NAME "$kpn_default") - num_executors=$(get_cfg "$cfg_file" NUM_EXECUTORS "1") - jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "") + jvm_options=$(get_cfg "$cfg_file" JVM_OPTIONS "null") fs_root=$(get_cfg "$cfg_file" FS_ROOT "/w") - retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0") connection_type=$(get_cfg "$cfg_file" CONNECTION_TYPE "SSH") launcher_factory=$(get_launcher_factory "$connection_type") + node_properties=$(get_cfg "$cfg_file" NODE_PROPERTIES, "null") + retention_time=$(get_cfg "$cfg_file" RETENTION_TIME "0") + config_drive=$(get_cfg "$cfg_file" CONFIG_DRIVE, "null") - OS_PLUGIN_VER="$(lftools jenkins plugins list \ - | grep -i 'OpenStack Cloud Plugin' \ - | awk -F':' '{print $2}' | awk -F' ' '{print $1}')" - if version_ge "$OS_PLUGIN_VER" "2.35"; then - if [ -n "$volume_size" ]; then - echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size)," - else - echo " new BootSource.Image(\"$image_name\")," - fi - echo " \"${flavors[${hardware_id}]}\"," - echo " \"$network_id\"," - echo " \"$user_data_id\"," - echo " $instance_cap," - echo " $min_instance_cap," - echo " \"$floating_ip_pool\"," - echo " \"$security_groups\"," - echo " \"$availability_zone\"," - echo " $start_timeout," - echo " \"$key_pair_name\"," - echo " $num_executors," - echo " \"$jvm_options\"," - echo " \"$fs_root\"," - echo " $launcher_factory," - echo " $retention_time" - - else # SlaveOptions() structure for versions <= 2.34 - if [ -n "$volume_size" ]; then - echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size)," - else - echo " new BootSource.Image(\"$image_name\")," - fi - echo " \"${flavors[${hardware_id}]}\"," - echo " \"$network_id\"," - echo " \"$user_data_id\"," - echo " $instance_cap," - echo " \"$floating_ip_pool\"," - echo " \"$security_groups\"," - echo " \"$availability_zone\"," - echo " $start_timeout," - echo " \"$key_pair_name\"," - echo " $num_executors," - echo " \"$jvm_options\"," - echo " \"$fs_root\"," - echo " $launcher_factory," - echo " $retention_time" + + if [ -n "$volume_size" ]; then + echo " new BootSource.VolumeFromImage(\"$image_name\", $volume_size)," + else + echo " new BootSource.Image(\"$image_name\")," + fi + + echo " \"${flavors[${hardware_id}]}\"," + echo " \"$network_id\"," + echo " \"$user_data_id\"," + echo " $instance_cap," + + # Handle specifying the minimum instance count across different versions + if testversion "$os_plugin_version" '>=' '2.47' + then + instance_min=$(get_cfg "$cfg_file" INSTANCE_MIN "null") + echo " $instance_min," + else + instance_min=$(get_cfg "$cfg_file" INSTANCE_MIN_CAPMAX "null") + echo " $instance_min," fi + + echo " \"$floating_ip_pool\"," + echo " \"$security_groups\"," + echo " \"$availability_zone\"," + echo " $start_timeout," + echo " \"$key_pair_name\"," + echo " $num_executors," + echo " \"$jvm_options\"," + echo " \"$fs_root\"," + echo " $launcher_factory," + + if testversion "$os_plugin_version" '>=' '2.47' + then + echo " $node_properties," + echo " $retention_time", + echo " $config_drive" + else + echo " $retention_time" + fi + + } get_template_cfg() { @@ -278,6 +292,7 @@ get_template_cfg() { echo ")" } + mapfile -t clouds < <(ls -d1 "$OS_CLOUD_DIR"/*/) for silo in $silos; do diff --git a/shell/jjb-merge-job.sh b/shell/jjb-merge-job.sh index 0a05c4f0..9a497fd0 100644 --- a/shell/jjb-merge-job.sh +++ b/shell/jjb-merge-job.sh @@ -15,4 +15,9 @@ workers="${JJB_WORKERS:-0}" # Ensure we fail the job if any steps fail. set -eu -o pipefail +# shellcheck disable=SC1090 +source ~/lf-env.sh + +lf-activate-venv jenkins-job-builder + jenkins-jobs update --recursive --delete-old --workers "$workers" jjb/ diff --git a/shell/openstack-kubernetes-create.sh b/shell/openstack-kubernetes-create.sh new file mode 100755 index 00000000..5ee05b09 --- /dev/null +++ b/shell/openstack-kubernetes-create.sh @@ -0,0 +1,80 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +# shellcheck disable=SC2153,SC2034 +echo "---> Creating kubernetes cluster" + +set -eux -o pipefail + +os_cloud="${OS_CLOUD:-vex}" +fixed_network="${FIXED_NETWORK}" +fixed_subnet="${FIXED_SUBNET}" +cluster_template_name="${CLUSTER_TEMPLATE_NAME}" +cluster_name="${CLUSTER_NAME}" +base_image="${BASE_IMAGE}" +keypair="${KEYPAIR}" +master_flavor="${MASTER_FLAVOR}" +node_flavor="${NODE_FLAVOR}" +master_count="${MASTER_COUNT:-1}" +node_count="${NODE_COUNT:-2}" +boot_volume_size="${BOOT_VOLUME_SIZE}" +docker_volume_size="${DOCKER_VOLUME_SIZE}" +k8s_version="${KUBERNETES_VERSION}" +cluster_settle_time="${CLUSTER_SETTLE_TIME:-1m}" + + +# Create the template for the cluster first. Returns the cluster ID as $template_uuid +template_uuid=$(openstack coe cluster template create "$cluster_template_name" \ + --os-cloud "$os_cloud" \ + --image "$base_image" \ + --keypair "$keypair" \ + --external-network public \ + --fixed-network "$fixed_network" \ + --fixed-subnet "$fixed_subnet" \ + --floating-ip-disabled \ + --master-flavor "$master_flavor" \ + --flavor "$node_flavor" \ + --docker-volume-size "$docker_volume_size" \ + --network-driver flannel \ + --master-lb-enabled \ + --volume-driver cinder \ + --labels boot_volume_type=ssd,boot_volume_size="${boot_volume_size}",kube_version="${k8s_version}",kube_tag="${k8s_version}" \ + --coe kubernetes \ + -f value -c uuid | tail -1) + +# Create the kubernetes cluster +cluster_uuid=$(openstack coe cluster create "$cluster_name" \ + --os-cloud "$os_cloud" \ + --master-count "$master_count" \ + --node-count "$node_count" \ + --cluster-template "$template_uuid" | awk -F ' ' '{print $5}') + +# Sleep for a little, because sometimes OpenStack has to catch up with itself +sleep 15 + +while [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_IN_PROGRESS" ] +do + # echo "sleeping $(date)" + sleep 2m +done + +if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_FAILED" ] +then + echo "Failed to create cluster: $cluster_uuid $(date)" + openstack --os-cloud "$os_cloud" coe cluster delete "$cluster_uuid" + sleep 5m + openstack --os-cloud "$os_cloud" coe cluster template delete "$template_uuid" + exit 1 +fi + +if [ "$(openstack --os-cloud "$os_cloud" coe cluster show "$cluster_uuid" -c status -f value)" == "CREATE_COMPLETE" ] +then + echo "Successfully created cluster: $cluster_uuid." +fi diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index 660d5f26..b388b0f6 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -47,6 +47,8 @@ else lftools[openstack] python-heatclient python-openstackclient +python-magnumclient +kubernetes niet~=1.4.2 tox>=3.7.0 # Tox 3.7 or greater is necessary for parallel mode support yq diff --git a/shell/release-job.sh b/shell/release-job.sh index 7c78a870..d852a022 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -411,8 +411,7 @@ elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then package_name=$(yq -r '.package_name' $release_file) username_repo="o-ran-sc/staging" echo "INFO: Fetching schema $RELEASE_SCHEMA" - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/\ - schema/${RELEASE_SCHEMA} + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA} verify_schema for name in $(yq -r '.package_name[].name' $release_file); do package_name=$name