jenkins-admin/create_jenkins_clouds_openstack_yaml.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2020 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 """Create JCasC yaml file for the given Openstack cloud config"""
  12
  13 import argparse
  14 import configparser
  15 import glob
  16 import os
  17
  18 from jinja2 import Template
  19
  20 # Template section
  21 lookuptable = {
  22     "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86",
  23     "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27",
  24     "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547",
  25     "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643",
  26     "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721",
  27     "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27",
  28     "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215",
  29     "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148",
  30     "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006",
  31     "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c",
  32     "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25",
  33     "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48",
  34     "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f",
  35     "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102",
  36     "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78",
  37     "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566",
  38     "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff",
  39     "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d",
  40     "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727",
  41     "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414",
  42     "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1",
  43     "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3",
  44     "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd",
  45     "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81",
  46     "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff",
  47     "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750",
  48     "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d",
  49     "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2",
  50     "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d",
  51     "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d",
  52     "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8",
  53     "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec",
  54     "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949",
  55     "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c",
  56     "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb",
  57     "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919",
  58     "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e",
  59     "v3-standard-1": "555dff3a-7ec2-437e-bfc7-5a00113a304d",
  60     "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb",
  61     "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7",
  62     "v3-standard-6": "e145dc6b-7560-4633-ab6e-430028fd877f",
  63     "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc",
  64     "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5",
  65     "v3-standard-20": "6baabd68-258c-4fdc-b0ba-5a77c5b89c21",
  66     "v3-standard-24": "cec3e6ff-667e-431c-9c14-ba7c1d9b4cc2",
  67     "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36",
  68     "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2",
  69     "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7",
  70     "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0",
  71     "v3-starter-1": "4d2a0d31-ebe9-4b99-a6d1-96c075b6c239",
  72     "v3-starter-2": "b542cedb-d3b4-4446-a43f-5416711440ee",
  73     "v3-starter-4": "5f93acce-e8dc-482b-9118-134728a77aa8",
  74     "v3-starter-6": "c5a671a2-2db5-4ffe-b681-ff77ec18bbe5",
  75     "v3-starter-8": "35c0ddb3-4dd8-478c-887c-34620851a66a",
  76     "v3-starter-16": "595dd716-6c7a-4365-9020-2ff10796e29c",
  77     "v3-starter-20": "3e8f788c-50ed-48c5-875e-5dfb3814d1f6",
  78     "v3-starter-24": "eb1af7f9-6b54-4780-a7e6-f76813106227",
  79     "v3-starter-32": "15949005-7952-4e93-be69-ca89dab5b884",
  80     "v3-starter-48": "94eb4cec-3840-4171-ad50-a8bce2757d11",
  81     "v3-starter-64": "4a6e52a2-8f64-4632-adde-72f81616d4f9",
  82     "v3-starter-96": "8e7205fc-3ec7-456c-bff0-e38609e415c1",
  83 }
  84 maintemplate = """\
  85 ---
  86 jenkins:
  87   clouds:
  88     - openstack:
  89         credentialsId: {{ cloud_credential_id }}
  90         endPointUrl: {{ cloud_url }}
  91         ignoreSsl: {{ cloud_ignore_ssl }}
  92         name: {{ cloud_name }}
  93         slaveOptions:
  94           availabilityZone: {{ availability_zone }}
  95           bootSource:
  96             {{ image_type }}:
  97               name: {{ image_name }}
  98 {%- if image_type == "volumeFromImage"  %}
  99               volumeSize: {{ volume_size }}{% endif %}
 100           fsRoot: {{ fs_root }}
 101           hardwareId: {{ hardware_id }}
 102 {%- if is_sandbox is defined %}
 103           instanceCap: {{ sandbox_cap }}
 104 {%- else %}
 105           instanceCap: {{ instance_cap }}{% endif %}
 106           keyPairName: {{ key_pair_name }}
 107           launcherFactory:
 108             ssh:
 109               credentialsId: {{ key_pair_name }}
 110           networkId: {{ network_id }}
 111           retentionTime: {{ retention_time }}
 112           userDataId: {{ user_data_id }}
 113         templates:
 114 """
 115 machinetemplate = """\
 116           - labels: {{ labels }}
 117             name: {{ name_prefix }}-{{ agent_name }}
 118             slaveOptions:
 119               bootSource:
 120                 {{ image_type }}:
 121                   name: {{ image_name }}
 122 {%- if image_type == "volumeFromImage"  %}
 123                   volumeSize: {{ volume_size }}{% endif %}
 124 {%- if hardware_id  %}
 125               hardwareId: {{ hardware_id }}{% endif %}
 126 {%- if instance_cap %}
 127               instanceCap: {{ instance_cap }}{% endif %}
 128 {%- if num_executors %}
 129               numExecutors: {{ num_executors }}{% endif %}
 130 {%- if retention_time %}
 131               retentionTime: {{ retention_time }}
 132 {%- else %}
 133               retentionTime: 0{% endif %}
 134 """
 135 footertemplate = """\
 136         zone: {{ cloud_zone}}
 137 """
 138
 139 # Command line args section
 140 def dir_path(path):
 141     if os.path.isdir(path):
 142         return path
 143     else:
 144         raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path")
 145
 146 def parse_arguments():
 147     parser = argparse.ArgumentParser(
 148         description="Create JCasC yaml from path to Jenkins config dir.")
 149
 150     parser.add_argument("--path", type=dir_path,
 151                         help="Path to jenkins-config directory")
 152     parser.add_argument("--name", type=str,
 153                         help="Cloud name (e.g \"cattle\")")
 154
 155     parser.add_argument(
 156         "-s", "--sandbox",
 157         help="Configuration is being created for a sandbox",
 158         dest="sandbox", action="store_true"
 159     )
 160
 161     return parser.parse_args()
 162
 163 parsed_args = parse_arguments()
 164 path = (parsed_args.path)
 165 path = ("{}**/*.cfg".format(path))
 166
 167 # Sandbox switch section
 168 section_cloud = {}
 169 name_prefix = "prd"
 170 if parsed_args.sandbox:
 171     name_prefix = "snd"
 172     section_cloud.update(is_sandbox=True)
 173
 174 # Config parser from merged files section
 175 def read_config(filename):
 176     shortname = os.path.basename(filename)
 177     header_name = os.path.splitext(shortname)[0]
 178     with open(filename, "r", encoding="utf_8") as config_file:
 179         config.read_file(add_section_header(config_file, header_name), source=filename)
 180     return config
 181
 182 # Cfg files are not real ini files, need to add section headers.
 183 def add_section_header(properties_file, header_name):
 184     yield "[{}]\n".format(header_name)
 185     for line in properties_file:
 186         yield line
 187
 188 config = configparser.ConfigParser()
 189 for filename in glob.iglob(path, recursive=True):
 190     # config_parser_merged is the configparser object with all the configs from *.cfg
 191     config_parser_merged = read_config(filename)
 192
 193 # Global cloud config section
 194 cloud_config = (config.items("cloud"))
 195 name = parsed_args.name
 196 cloud_config_final = (*cloud_config, ("cloud_name", name))
 197
 198 for index, _ in enumerate(cloud_config_final):
 199     key = cloud_config_final[index][0]
 200     value = cloud_config_final[index][1]
 201     if value in lookuptable.keys():
 202         value = lookuptable[value]
 203     section_cloud[key] = value
 204
 205     if "volume_size" not in section_cloud:
 206         section_cloud.update(image_type="image")
 207     else:
 208         section_cloud.update(image_type="volumeFromImage")
 209
 210 j2_template = Template(maintemplate)
 211 print(j2_template.render(section_cloud))
 212
 213
 214 # All machines section
 215 for section in config_parser_merged.sections():
 216     if section != "cloud":
 217         machine = (config.items(section))
 218         section_all_machines = {}
 219         for index, _ in enumerate(machine):
 220             key = machine[index][0]
 221             value = machine[index][1]
 222             if value in lookuptable.keys():
 223                 value = lookuptable[value]
 224             section_all_machines[key] = value
 225
 226         if "volume_size" not in section_all_machines:
 227             section_all_machines.update(image_type="image")
 228         else:
 229             section_all_machines.update(image_type="volumeFromImage")
 230
 231         # Naming and labels
 232         section_all_machines.update(agent_name=section)
 233         if "labels" not in section_all_machines:
 234             # "section" is the name of the cloud agent, which is the default label
 235             section_all_machines.update(labels=section)
 236         elif section not in section_all_machines["labels"]:
 237             labels = section + " " + section_all_machines["labels"]
 238             section_all_machines.update(labels=labels)
 239
 240         j2_template = Template(machinetemplate)
 241         section_all_machines.update(name_prefix=name_prefix)
 242         print(j2_template.render(section_all_machines))
 243
 244
 245 # Footer section
 246 j2_template = Template(footertemplate)
 247 print(j2_template.render(section_cloud))