X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=blobdiff_plain;f=jenkins-admin%2Fcreate_jenkins_clouds_openstack_yaml.py;h=a1464a5de7b273a2e6f1ceb0ea7a2f536b7fc7d7;hb=918a6dd6692d7da97dadfaec75702ceb67e64c6e;hp=2d81a2500064e7a017561f14ee4519c7a499f867;hpb=ae9f2f497f6a9c036fc9ffaf4f95d72a0cab6f2e;p=releng%2Fglobal-jjb.git diff --git a/jenkins-admin/create_jenkins_clouds_openstack_yaml.py b/jenkins-admin/create_jenkins_clouds_openstack_yaml.py index 2d81a250..a1464a5d 100755 --- a/jenkins-admin/create_jenkins_clouds_openstack_yaml.py +++ b/jenkins-admin/create_jenkins_clouds_openstack_yaml.py @@ -1,16 +1,23 @@ #!/usr/bin/env python +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2020 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Create JCasC yaml file for the given Openstack cloud config""" import argparse import configparser import glob import os -import sys -from jinja2 import Environment -from jinja2 import FileSystemLoader from jinja2 import Template -#Template section +# Template section lookuptable = { "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86", "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27", @@ -48,7 +55,31 @@ lookuptable = { "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c", "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb", "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919", - "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e" + "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e", + "v3-standard-1": "555dff3a-7ec2-437e-bfc7-5a00113a304d", + "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb", + "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7", + "v3-standard-6": "e145dc6b-7560-4633-ab6e-430028fd877f", + "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc", + "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5", + "v3-standard-20": "6baabd68-258c-4fdc-b0ba-5a77c5b89c21", + "v3-standard-24": "cec3e6ff-667e-431c-9c14-ba7c1d9b4cc2", + "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36", + "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2", + "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7", + "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0", + "v3-starter-1": "4d2a0d31-ebe9-4b99-a6d1-96c075b6c239", + "v3-starter-2": "b542cedb-d3b4-4446-a43f-5416711440ee", + "v3-starter-4": "5f93acce-e8dc-482b-9118-134728a77aa8", + "v3-starter-6": "c5a671a2-2db5-4ffe-b681-ff77ec18bbe5", + "v3-starter-8": "35c0ddb3-4dd8-478c-887c-34620851a66a", + "v3-starter-16": "595dd716-6c7a-4365-9020-2ff10796e29c", + "v3-starter-20": "3e8f788c-50ed-48c5-875e-5dfb3814d1f6", + "v3-starter-24": "eb1af7f9-6b54-4780-a7e6-f76813106227", + "v3-starter-32": "15949005-7952-4e93-be69-ca89dab5b884", + "v3-starter-48": "94eb4cec-3840-4171-ad50-a8bce2757d11", + "v3-starter-64": "4a6e52a2-8f64-4632-adde-72f81616d4f9", + "v3-starter-96": "8e7205fc-3ec7-456c-bff0-e38609e415c1", } maintemplate = """\ --- @@ -58,7 +89,7 @@ jenkins: credentialsId: {{ cloud_credential_id }} endPointUrl: {{ cloud_url }} ignoreSsl: {{ cloud_ignore_ssl }} - name: "cattle" + name: {{ cloud_name }} slaveOptions: availabilityZone: {{ availability_zone }} bootSource: @@ -82,18 +113,19 @@ jenkins: """ machinetemplate = """\ - labels: {{ labels }} - name: {{ name_prefix }}-{{ labels }} + name: {{ name_prefix }}-{{ agent_name }} slaveOptions: bootSource: - volumeFromImage: + {{ image_type }}: name: {{ image_name }} - volumeSize: {{ volume_size }} +{%- if image_type == "volumeFromImage" %} + volumeSize: {{ volume_size }}{% endif %} {%- if hardware_id %} hardwareId: {{ hardware_id }}{% endif %} {%- if instance_cap %} instanceCap: {{ instance_cap }}{% endif %} {%- if num_executors %} - numExectorts: {{ num_executors }}{% endif %} + numExecutors: {{ num_executors }}{% endif %} {%- if retention_time %} retentionTime: {{ retention_time }} {%- else %} @@ -103,86 +135,71 @@ footertemplate = """\ zone: {{ cloud_zone}} """ -#Command line args section +# Command line args section def dir_path(path): if os.path.isdir(path): return path else: - raise argparse.ArgumentTypeError(f"readable_dir:{path} is not a valid path") + raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path") def parse_arguments(): parser = argparse.ArgumentParser( - description='Create jcasc yaml from path to jenkins config dir.') + description="Create JCasC yaml from path to Jenkins config dir.") + parser.add_argument("--path", type=dir_path, + help="Path to jenkins-config directory") + parser.add_argument("--name", type=str, + help="Cloud name (e.g \"cattle\")") + + parser.add_argument( + "-s", "--sandbox", + help="Configuration is being created for a sandbox", + dest="sandbox", action="store_true" + ) - parser.add_argument('--path', type=dir_path, - help="Path to jenkins-admin directory") - parser.add_argument('--sandbox', type=bool, default=False, - help="Set to True for sandbox yaml generation") return parser.parse_args() parsed_args = parse_arguments() path = (parsed_args.path) path = ("{}**/*.cfg".format(path)) -#sandbox switch section +# Sandbox switch section section_cloud = {} name_prefix = "prd" if parsed_args.sandbox: name_prefix = "snd" section_cloud.update(is_sandbox=True) -#Config parser from merged files section -class Iterfiles: - def __init__(self, filename): - self.filename = filename - - def iterf(self): - shortname = os.path.basename(filename) - shortname1 = os.path.splitext(shortname)[0] - return (self.filename, shortname1) - -class Openfile: - def __init__(self, filename, shortname1): - self.filename = filename - self.shortname1 = shortname1 - def openf(self): - file = open(self.filename, encoding="utf_8") - config.read_file(add_section_header(file, self.shortname1), source=self.filename) - return config - -#cfg files are not real ini files, need to add section headers. +# Config parser from merged files section +def read_config(filename): + shortname = os.path.basename(filename) + header_name = os.path.splitext(shortname)[0] + with open(filename, "r", encoding="utf_8") as config_file: + config.read_file(add_section_header(config_file, header_name), source=filename) + return config + +# Cfg files are not real ini files, need to add section headers. def add_section_header(properties_file, header_name): - yield '[{}]\n'.format(header_name) + yield "[{}]\n".format(header_name) for line in properties_file: yield line config = configparser.ConfigParser() for filename in glob.iglob(path, recursive=True): - #This just returns the filename you sent it and a shortname - # dumb but just messing around with classes. - r1 = Iterfiles(filename) - r3 = r1.iterf() - #now i send the file name and the shortname to Openfile class - #which builds up a big configparser with sections divided by shortname - r5 = Openfile(r3[0], r3[1]) - #config_parser_merged is the configparser object with all the configs from *.cfg - config_parser_merged = r5.openf() - + # config_parser_merged is the configparser object with all the configs from *.cfg + config_parser_merged = read_config(filename) # Global cloud config section -for section in config_parser_merged.sections(): - if section == "cloud": - final = (config.items(section)) +cloud_config = (config.items("cloud")) +name = parsed_args.name +cloud_config_final = (*cloud_config, ("cloud_name", name)) - -for index, _ in enumerate(final): - a = final[index][0] - b = final[index][1] - for x, y in lookuptable.items(): - if b == x: - b = y - section_cloud[a] = b +for index, _ in enumerate(cloud_config_final): + key = cloud_config_final[index][0] + value = cloud_config_final[index][1] + if value in lookuptable.keys(): + value = lookuptable[value] + section_cloud[key] = value j2_template = Template(maintemplate) print(j2_template.render(section_cloud)) @@ -194,29 +211,31 @@ for section in config_parser_merged.sections(): machine = (config.items(section)) section_all_machines = {} for index, _ in enumerate(machine): - a = machine[index][0] - b = machine[index][1] - for x, y in lookuptable.items(): - if b == x: - b = y - #a = format(a) - section_all_machines[a] = b - - - #Default volume size of 10 - if not "volume_size" in section_all_machines: - section_all_machines.update(volume_size="10") - if not "labels" in section_all_machines: - print("LABELS not Set in builder config") - print(section_all_machines) - exit(1) - + key = machine[index][0] + value = machine[index][1] + if value in lookuptable.keys(): + value = lookuptable[value] + section_all_machines[key] = value + + if "volume_size" not in section_all_machines: + section_all_machines.update(image_type="image") + else: + section_all_machines.update(image_type="volumeFromImage") + + # Naming and labels + section_all_machines.update(agent_name=section) + if "labels" not in section_all_machines: + # "section" is the name of the cloud agent, which is the default label + section_all_machines.update(labels=section) + elif section not in section_all_machines["labels"]: + labels = section + " " + section_all_machines["labels"] + section_all_machines.update(labels=labels) j2_template = Template(machinetemplate) section_all_machines.update(name_prefix=name_prefix) print(j2_template.render(section_all_machines)) -#Footer section +# Footer section j2_template = Template(footertemplate) print(j2_template.render(section_cloud))