jenkins-admin/create_jenkins_clouds_openstack_yaml.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2020 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 """Create JCasC yaml file for the given Openstack cloud config"""
  12
  13 import argparse
  14 import configparser
  15 import glob
  16 import os
  17
  18 from jinja2 import Template
  19
  20 # Template section
  21 lookuptable = {
  22     "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86",
  23     "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27",
  24     "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547",
  25     "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643",
  26     "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721",
  27     "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27",
  28     "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215",
  29     "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148",
  30     "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006",
  31     "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c",
  32     "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25",
  33     "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48",
  34     "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f",
  35     "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102",
  36     "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78",
  37     "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566",
  38     "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff",
  39     "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d",
  40     "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727",
  41     "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414",
  42     "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1",
  43     "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3",
  44     "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd",
  45     "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81",
  46     "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff",
  47     "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750",
  48     "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d",
  49     "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2",
  50     "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d",
  51     "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d",
  52     "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8",
  53     "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec",
  54     "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949",
  55     "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c",
  56     "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb",
  57     "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919",
  58     "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e",
  59     "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb",
  60     "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7",
  61     "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc",
  62     "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5",
  63     "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36",
  64     "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2",
  65     "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7",
  66     "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0",
  67     "v3-starter-1": "4d2a0d31-ebe9-4b99-a6d1-96c075b6c239",
  68     "v3-starter-2": "b542cedb-d3b4-4446-a43f-5416711440ee",
  69     "v3-starter-4": "5f93acce-e8dc-482b-9118-134728a77aa8",
  70     "v3-starter-6": "c5a671a2-2db5-4ffe-b681-ff77ec18bbe5",
  71     "v3-starter-8": "35c0ddb3-4dd8-478c-887c-34620851a66a",
  72     "v3-starter-16": "595dd716-6c7a-4365-9020-2ff10796e29c",
  73     "v3-starter-20": "3e8f788c-50ed-48c5-875e-5dfb3814d1f6",
  74     "v3-starter-24": "eb1af7f9-6b54-4780-a7e6-f76813106227",
  75     "v3-starter-32": "15949005-7952-4e93-be69-ca89dab5b884",
  76     "v3-starter-48": "94eb4cec-3840-4171-ad50-a8bce2757d11",
  77     "v3-starter-64": "4a6e52a2-8f64-4632-adde-72f81616d4f9",
  78     "v3-starter-96": "8e7205fc-3ec7-456c-bff0-e38609e415c1",
  79 }
  80 maintemplate = """\
  81 ---
  82 jenkins:
  83   clouds:
  84     - openstack:
  85         credentialsId: {{ cloud_credential_id }}
  86         endPointUrl: {{ cloud_url }}
  87         ignoreSsl: {{ cloud_ignore_ssl }}
  88         name: {{ cloud_name }}
  89         slaveOptions:
  90           availabilityZone: {{ availability_zone }}
  91           bootSource:
  92             volumeFromImage:
  93               name: {{ image_name }}
  94               volumeSize: {{ volume_size }}
  95           fsRoot: {{ fs_root }}
  96           hardwareId: {{ hardware_id }}
  97 {%- if is_sandbox is defined %}
  98           instanceCap: {{ sandbox_cap }}
  99 {%- else %}
 100           instanceCap: {{ instance_cap }}{% endif %}
 101           keyPairName: {{ key_pair_name }}
 102           launcherFactory:
 103             ssh:
 104               credentialsId: {{ key_pair_name }}
 105           networkId: {{ network_id }}
 106           retentionTime: {{ retention_time }}
 107           userDataId: {{ user_data_id }}
 108         templates:
 109 """
 110 machinetemplate = """\
 111           - labels: {{ labels }}
 112             name: {{ name_prefix }}-{{ agent_name }}
 113             slaveOptions:
 114               bootSource:
 115                 {{ image_type }}:
 116                   name: {{ image_name }}
 117 {%- if image_type == "volumeFromImage"  %}
 118                   volumeSize: {{ volume_size }}{% endif %}
 119 {%- if hardware_id  %}
 120               hardwareId: {{ hardware_id }}{% endif %}
 121 {%- if instance_cap %}
 122               instanceCap: {{ instance_cap }}{% endif %}
 123 {%- if num_executors %}
 124               numExectorts: {{ num_executors }}{% endif %}
 125 {%- if retention_time %}
 126               retentionTime: {{ retention_time }}
 127 {%- else %}
 128               retentionTime: 0{% endif %}
 129 """
 130 footertemplate = """\
 131         zone: {{ cloud_zone}}
 132 """
 133
 134 # Command line args section
 135 def dir_path(path):
 136     if os.path.isdir(path):
 137         return path
 138     else:
 139         raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path")
 140
 141 def parse_arguments():
 142     parser = argparse.ArgumentParser(
 143         description="Create JCasC yaml from path to Jenkins config dir.")
 144
 145     parser.add_argument("--path", type=dir_path,
 146                         help="Path to jenkins-config directory")
 147     parser.add_argument("--name", type=str,
 148                         help="Cloud name (e.g \"cattle\")")
 149
 150     parser.add_argument(
 151         "-s", "--sandbox",
 152         help="Configuration is being created for a sandbox",
 153         dest="sandbox", action="store_true"
 154     )
 155
 156     return parser.parse_args()
 157
 158 parsed_args = parse_arguments()
 159 path = (parsed_args.path)
 160 path = ("{}**/*.cfg".format(path))
 161
 162 # Sandbox switch section
 163 section_cloud = {}
 164 name_prefix = "prd"
 165 if parsed_args.sandbox:
 166     name_prefix = "snd"
 167     section_cloud.update(is_sandbox=True)
 168
 169 # Config parser from merged files section
 170 def read_config(filename):
 171     shortname = os.path.basename(filename)
 172     header_name = os.path.splitext(shortname)[0]
 173     with open(filename, "r", encoding="utf_8") as config_file:
 174         config.read_file(add_section_header(config_file, header_name), source=filename)
 175     return config
 176
 177 # Cfg files are not real ini files, need to add section headers.
 178 def add_section_header(properties_file, header_name):
 179     yield "[{}]\n".format(header_name)
 180     for line in properties_file:
 181         yield line
 182
 183 config = configparser.ConfigParser()
 184 for filename in glob.iglob(path, recursive=True):
 185     # config_parser_merged is the configparser object with all the configs from *.cfg
 186     config_parser_merged = read_config(filename)
 187
 188 # Global cloud config section
 189 cloud_config = (config.items("cloud"))
 190 name = parsed_args.name
 191 cloud_config_final = (*cloud_config, ("cloud_name", name))
 192
 193 for index, _ in enumerate(cloud_config_final):
 194     key = cloud_config_final[index][0]
 195     value = cloud_config_final[index][1]
 196     if value in lookuptable.keys():
 197         value = lookuptable[value]
 198     section_cloud[key] = value
 199
 200 j2_template = Template(maintemplate)
 201 print(j2_template.render(section_cloud))
 202
 203
 204 # All machines section
 205 for section in config_parser_merged.sections():
 206     if section != "cloud":
 207         machine = (config.items(section))
 208         section_all_machines = {}
 209         for index, _ in enumerate(machine):
 210             key = machine[index][0]
 211             value = machine[index][1]
 212             if value in lookuptable.keys():
 213                 value = lookuptable[value]
 214             section_all_machines[key] = value
 215
 216         if "volume_size" not in section_all_machines:
 217             section_all_machines.update(image_type="image")
 218         else:
 219             section_all_machines.update(image_type="volumeFromImage")
 220
 221         # Naming and labels
 222         section_all_machines.update(agent_name=section)
 223         if "labels" not in section_all_machines:
 224             # "section" is the name of the cloud agent, which is the default label
 225             section_all_machines.update(labels=section)
 226         elif section not in section_all_machines["labels"]:
 227             labels = section + " " + section_all_machines["labels"]
 228             section_all_machines.update(labels=labels)
 229
 230         j2_template = Template(machinetemplate)
 231         section_all_machines.update(name_prefix=name_prefix)
 232         print(j2_template.render(section_all_machines))
 233
 234
 235 # Footer section
 236 j2_template = Template(footertemplate)
 237 print(j2_template.render(section_cloud))