jenkins-admin/create_jenkins_clouds_openstack_yaml.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2020 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 """Create JCasC yaml file for the given Openstack cloud config"""
  12
  13 import argparse
  14 import configparser
  15 import glob
  16 import os
  17
  18 from jinja2 import Template
  19
  20 # Template section
  21 lookuptable = {
  22     "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86",
  23     "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27",
  24     "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547",
  25     "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643",
  26     "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721",
  27     "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27",
  28     "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215",
  29     "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148",
  30     "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006",
  31     "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c",
  32     "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25",
  33     "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48",
  34     "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f",
  35     "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102",
  36     "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78",
  37     "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566",
  38     "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff",
  39     "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d",
  40     "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727",
  41     "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414",
  42     "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1",
  43     "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3",
  44     "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd",
  45     "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81",
  46     "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff",
  47     "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750",
  48     "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d",
  49     "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2",
  50     "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d",
  51     "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d",
  52     "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8",
  53     "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec",
  54     "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949",
  55     "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c",
  56     "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb",
  57     "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919",
  58     "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e",
  59     "v3-standard-1": "555dff3a-7ec2-437e-bfc7-5a00113a304d",
  60     "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb",
  61     "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7",
  62     "v3-standard-6": "e145dc6b-7560-4633-ab6e-430028fd877f",
  63     "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc",
  64     "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5",
  65     "v3-standard-20": "6baabd68-258c-4fdc-b0ba-5a77c5b89c21",
  66     "v3-standard-24": "cec3e6ff-667e-431c-9c14-ba7c1d9b4cc2",
  67     "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36",
  68     "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2",
  69     "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7",
  70     "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0",
  71     "v3-starter-1": "4d2a0d31-ebe9-4b99-a6d1-96c075b6c239",
  72     "v3-starter-2": "b542cedb-d3b4-4446-a43f-5416711440ee",
  73     "v3-starter-4": "5f93acce-e8dc-482b-9118-134728a77aa8",
  74     "v3-starter-6": "c5a671a2-2db5-4ffe-b681-ff77ec18bbe5",
  75     "v3-starter-8": "35c0ddb3-4dd8-478c-887c-34620851a66a",
  76     "v3-starter-16": "595dd716-6c7a-4365-9020-2ff10796e29c",
  77     "v3-starter-20": "3e8f788c-50ed-48c5-875e-5dfb3814d1f6",
  78     "v3-starter-24": "eb1af7f9-6b54-4780-a7e6-f76813106227",
  79     "v3-starter-32": "15949005-7952-4e93-be69-ca89dab5b884",
  80     "v3-starter-48": "94eb4cec-3840-4171-ad50-a8bce2757d11",
  81     "v3-starter-64": "4a6e52a2-8f64-4632-adde-72f81616d4f9",
  82     "v3-starter-96": "8e7205fc-3ec7-456c-bff0-e38609e415c1",
  83 }
  84 maintemplate = """\
  85 ---
  86 jenkins:
  87   clouds:
  88     - openstack:
  89         credentialsId: {{ cloud_credential_id }}
  90         endPointUrl: {{ cloud_url }}
  91         ignoreSsl: {{ cloud_ignore_ssl }}
  92         name: {{ cloud_name }}
  93         slaveOptions:
  94           availabilityZone: {{ availability_zone }}
  95           bootSource:
  96             volumeFromImage:
  97               name: {{ image_name }}
  98               volumeSize: {{ volume_size }}
  99           fsRoot: {{ fs_root }}
 100           hardwareId: {{ hardware_id }}
 101 {%- if is_sandbox is defined %}
 102           instanceCap: {{ sandbox_cap }}
 103 {%- else %}
 104           instanceCap: {{ instance_cap }}{% endif %}
 105           keyPairName: {{ key_pair_name }}
 106           launcherFactory:
 107             ssh:
 108               credentialsId: {{ key_pair_name }}
 109           networkId: {{ network_id }}
 110           retentionTime: {{ retention_time }}
 111           userDataId: {{ user_data_id }}
 112         templates:
 113 """
 114 machinetemplate = """\
 115           - labels: {{ labels }}
 116             name: {{ name_prefix }}-{{ agent_name }}
 117             slaveOptions:
 118               bootSource:
 119                 {{ image_type }}:
 120                   name: {{ image_name }}
 121 {%- if image_type == "volumeFromImage"  %}
 122                   volumeSize: {{ volume_size }}{% endif %}
 123 {%- if hardware_id  %}
 124               hardwareId: {{ hardware_id }}{% endif %}
 125 {%- if instance_cap %}
 126               instanceCap: {{ instance_cap }}{% endif %}
 127 {%- if num_executors %}
 128               numExecutors: {{ num_executors }}{% endif %}
 129 {%- if retention_time %}
 130               retentionTime: {{ retention_time }}
 131 {%- else %}
 132               retentionTime: 0{% endif %}
 133 """
 134 footertemplate = """\
 135         zone: {{ cloud_zone}}
 136 """
 137
 138 # Command line args section
 139 def dir_path(path):
 140     if os.path.isdir(path):
 141         return path
 142     else:
 143         raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path")
 144
 145 def parse_arguments():
 146     parser = argparse.ArgumentParser(
 147         description="Create JCasC yaml from path to Jenkins config dir.")
 148
 149     parser.add_argument("--path", type=dir_path,
 150                         help="Path to jenkins-config directory")
 151     parser.add_argument("--name", type=str,
 152                         help="Cloud name (e.g \"cattle\")")
 153
 154     parser.add_argument(
 155         "-s", "--sandbox",
 156         help="Configuration is being created for a sandbox",
 157         dest="sandbox", action="store_true"
 158     )
 159
 160     return parser.parse_args()
 161
 162 parsed_args = parse_arguments()
 163 path = (parsed_args.path)
 164 path = ("{}**/*.cfg".format(path))
 165
 166 # Sandbox switch section
 167 section_cloud = {}
 168 name_prefix = "prd"
 169 if parsed_args.sandbox:
 170     name_prefix = "snd"
 171     section_cloud.update(is_sandbox=True)
 172
 173 # Config parser from merged files section
 174 def read_config(filename):
 175     shortname = os.path.basename(filename)
 176     header_name = os.path.splitext(shortname)[0]
 177     with open(filename, "r", encoding="utf_8") as config_file:
 178         config.read_file(add_section_header(config_file, header_name), source=filename)
 179     return config
 180
 181 # Cfg files are not real ini files, need to add section headers.
 182 def add_section_header(properties_file, header_name):
 183     yield "[{}]\n".format(header_name)
 184     for line in properties_file:
 185         yield line
 186
 187 config = configparser.ConfigParser()
 188 for filename in glob.iglob(path, recursive=True):
 189     # config_parser_merged is the configparser object with all the configs from *.cfg
 190     config_parser_merged = read_config(filename)
 191
 192 # Global cloud config section
 193 cloud_config = (config.items("cloud"))
 194 name = parsed_args.name
 195 cloud_config_final = (*cloud_config, ("cloud_name", name))
 196
 197 for index, _ in enumerate(cloud_config_final):
 198     key = cloud_config_final[index][0]
 199     value = cloud_config_final[index][1]
 200     if value in lookuptable.keys():
 201         value = lookuptable[value]
 202     section_cloud[key] = value
 203
 204 j2_template = Template(maintemplate)
 205 print(j2_template.render(section_cloud))
 206
 207
 208 # All machines section
 209 for section in config_parser_merged.sections():
 210     if section != "cloud":
 211         machine = (config.items(section))
 212         section_all_machines = {}
 213         for index, _ in enumerate(machine):
 214             key = machine[index][0]
 215             value = machine[index][1]
 216             if value in lookuptable.keys():
 217                 value = lookuptable[value]
 218             section_all_machines[key] = value
 219
 220         if "volume_size" not in section_all_machines:
 221             section_all_machines.update(image_type="image")
 222         else:
 223             section_all_machines.update(image_type="volumeFromImage")
 224
 225         # Naming and labels
 226         section_all_machines.update(agent_name=section)
 227         if "labels" not in section_all_machines:
 228             # "section" is the name of the cloud agent, which is the default label
 229             section_all_machines.update(labels=section)
 230         elif section not in section_all_machines["labels"]:
 231             labels = section + " " + section_all_machines["labels"]
 232             section_all_machines.update(labels=labels)
 233
 234         j2_template = Template(machinetemplate)
 235         section_all_machines.update(name_prefix=name_prefix)
 236         print(j2_template.render(section_all_machines))
 237
 238
 239 # Footer section
 240 j2_template = Template(footertemplate)
 241 print(j2_template.render(section_cloud))