From: Aric Gardner Date: Thu, 1 Oct 2020 20:07:53 +0000 (-0400) Subject: Preliminary jcasc yaml generator X-Git-Tag: v0.58.0~3 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=a86978478e005c9ab1ad7fbf24712288f7e6db65;p=releng%2Fglobal-jjb.git Preliminary jcasc yaml generator generator that takes values from a projects builder/jenkins-config/*.cfg files and creates a yaml file compatible with jcasc Idea is to replace the groovy scripts in this same directory with something less brittle. A few problems, we can't validate the schema. as none has been written. so loading whatever this generates will be risky. example output: https://gist.github.com/Aricg/4366e4e54e6ab63905ecb7aaac56646f Issue-Id: RELENG-3035 Signed-off-by: Aric Gardner Change-Id: I8fbb6a49f8d13ba36c32b30469410c3a69c0f5f0 --- diff --git a/jenkins-admin/create_jenkins_clouds_openstack_yaml.py b/jenkins-admin/create_jenkins_clouds_openstack_yaml.py new file mode 100755 index 00000000..bd94a9e4 --- /dev/null +++ b/jenkins-admin/create_jenkins_clouds_openstack_yaml.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python + +import argparse +import configparser +import glob +import os +import sys + +from jinja2 import Environment +from jinja2 import FileSystemLoader +from jinja2 import Template + +#Template section +lookuptable = { + "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86", + "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27", + "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547", + "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643", + "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721", + "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27", + "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215", + "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148", + "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006", + "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c", + "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25", + "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48", + "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f", + "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102", + "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78", + "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566", + "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff", + "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d", + "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727", + "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414", + "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1", + "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3", + "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd", + "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81", + "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff", + "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750", + "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d", + "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2", + "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d", + "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d", + "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8", + "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec", + "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949", + "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c", + "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb", + "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919", + "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e" +} +maintemplate = """\ +--- +jenkins: + clouds: + - openstack: + credentialsId: {{ cloud_credential_id }} + endPointUrl: {{ cloud_url }} + ignoreSsl: {{ cloud_ignore_ssl }} + name: "cattle" + slaveOptions: + availabilityZone: {{ availability_zone }} + bootSource: + volumeFromImage: + name: {{ image_name }} + volumeSize: {{ volume_size }} + fsRoot: {{ fs_root }} + hardwareId: {{ hardware_id }} +{%- if is_sandbox is defined %} + instanceCap: {{ sandbox_cap }} +{%- else %} + instanceCap: {{ instance_cap }}{% endif %} + keyPairName: {{ key_pair_name }} + launcherFactory: + ssh: + credentialsId: {{ key_pair_name }} + networkId: {{ network_id }} + retentionTime: {{ retention_time }} + userDataId: {{ user_data_id }} + templates: +""" +machinetemplate = """\ + - labels: {{ labels }} + name: {{ name_prefix }}-{{ labels }} + slaveOptions: + bootSource: + volumeFromImage: + name: {{ image_name }} + volumeSize: {{ volume_size }} +{%- if hardware_id %} + hardwareId: {{ hardware_id }}{% endif %} +{%- if instance_cap %} + instanceCap: {{ instance_cap }}{% endif %} +{%- if num_executors %} + numExectorts: {{ num_executors }}{% endif %} +{%- if retention_time %} + retentionTime: {{ retention_time }} +{%- else %} + retentionTime: 0{% endif %} +""" +footertemplate = """\ + zone: {{ cloud_zone}} +""" + +#Command line args section +def dir_path(path): + if os.path.isdir(path): + return path + else: + raise argparse.ArgumentTypeError(f"readable_dir:{path} is not a valid path") + +def parse_arguments(): + parser = argparse.ArgumentParser(description='Create jcasc yaml from path to jenkins config dir.') + parser.add_argument('--path', type=dir_path) + parser.add_argument('--sandbox', type=bool, default=False, help="Set to True for sandbox yaml generation") + return parser.parse_args() + +parsed_args = parse_arguments() +path = (parsed_args.path) +path = ("{}**/*.cfg".format(path)) + + +#sandbox switch section +section_cloud = {} +name_prefix = "prd" +if parsed_args.sandbox: + name_prefix = "snd" + section_cloud.update(is_sandbox=True) + +#Config parser from merged files section +class Iterfiles: + def __init__(self, filename): + self.filename = filename + + def iterf(self): + shortname = os.path.basename(filename) + shortname1 = os.path.splitext(shortname)[0] + return (self.filename, shortname1) + +class Openfile: + def __init__(self, filename, shortname1): + self.filename = filename + self.shortname1 = shortname1 + def openf(self): + file = open(self.filename, encoding="utf_8") + config.read_file(add_section_header(file, self.shortname1), source=self.filename) + return config + +#cfg files are not real ini files, need to add section headers. +def add_section_header(properties_file, header_name): + yield '[{}]\n'.format(header_name) + for line in properties_file: + yield line + +config = configparser.ConfigParser() +for filename in glob.iglob(path, recursive=True): + #This just returns the filename you sent it and a shortname + # dumb but just messing around with classes. + r1 = Iterfiles(filename) + r3 = r1.iterf() + #now i send the file name and the shortname to Openfile class + #which builds up a big configparser with sections divided by shortname + r5 = Openfile(r3[0], r3[1]) + #config_parser_merged is the configparser object with all the configs from *.cfg + config_parser_merged = r5.openf() + + +# Global cloud config section +for section in config_parser_merged.sections(): + if section == "cloud": + final = (config.items(section)) + + +for index, _ in enumerate(final): + a = final[index][0] + b = final[index][1] + for x, y in lookuptable.items(): + if b == x: + b = y + section_cloud[a] = b + +j2_template = Template(maintemplate) +print(j2_template.render(section_cloud)) + + +# All machines section +for section in config_parser_merged.sections(): + if section != "cloud": + machine = (config.items(section)) + section_all_machines = {} + for index, _ in enumerate(machine): + a = machine[index][0] + b = machine[index][1] + for x, y in lookuptable.items(): + if b == x: + b = y + #a = format(a) + section_all_machines[a] = b + + + #Default volume size of 10 + if not "volume_size" in section_all_machines: + section_all_machines.update(volume_size="10") + if not "labels" in section_all_machines: + print("LABELS not Set in builder config") + print(section_all_machines) + exit(1) + + + j2_template = Template(machinetemplate) + section_all_machines.update(name_prefix=name_prefix) + print(j2_template.render(section_all_machines)) + + + +#Footer section +j2_template = Template(footertemplate) +print(j2_template.render(section_cloud))