jenkins-admin/create_jenkins_clouds_openstack_yaml.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2020 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 """Create JCasC yaml file for the given Openstack cloud config"""
  12
  13 import argparse
  14 import configparser
  15 import glob
  16 import os
  17
  18 from jinja2 import Template
  19
  20 # Template section
  21 lookuptable = {
  22     "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86",
  23     "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27",
  24     "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547",
  25     "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643",
  26     "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721",
  27     "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27",
  28     "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215",
  29     "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148",
  30     "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006",
  31     "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c",
  32     "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25",
  33     "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48",
  34     "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f",
  35     "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102",
  36     "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78",
  37     "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566",
  38     "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff",
  39     "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d",
  40     "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727",
  41     "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414",
  42     "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1",
  43     "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3",
  44     "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd",
  45     "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81",
  46     "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff",
  47     "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750",
  48     "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d",
  49     "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2",
  50     "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d",
  51     "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d",
  52     "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8",
  53     "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec",
  54     "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949",
  55     "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c",
  56     "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb",
  57     "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919",
  58     "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e",
  59     "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb",
  60     "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7",
  61     "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc",
  62     "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5",
  63     "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36",
  64     "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2",
  65     "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7",
  66     "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0"
  67 }
  68 maintemplate = """\
  69 ---
  70 jenkins:
  71   clouds:
  72     - openstack:
  73         credentialsId: {{ cloud_credential_id }}
  74         endPointUrl: {{ cloud_url }}
  75         ignoreSsl: {{ cloud_ignore_ssl }}
  76         name: {{ cloud_name }}
  77         slaveOptions:
  78           availabilityZone: {{ availability_zone }}
  79           bootSource:
  80             volumeFromImage:
  81               name: {{ image_name }}
  82               volumeSize: {{ volume_size }}
  83           fsRoot: {{ fs_root }}
  84           hardwareId: {{ hardware_id }}
  85 {%- if is_sandbox is defined %}
  86           instanceCap: {{ sandbox_cap }}
  87 {%- else %}
  88           instanceCap: {{ instance_cap }}{% endif %}
  89           keyPairName: {{ key_pair_name }}
  90           launcherFactory:
  91             ssh:
  92               credentialsId: {{ key_pair_name }}
  93           networkId: {{ network_id }}
  94           retentionTime: {{ retention_time }}
  95           userDataId: {{ user_data_id }}
  96         templates:
  97 """
  98 machinetemplate = """\
  99           - labels: {{ labels }}
 100             name: {{ name_prefix }}-{{ labels }}
 101             slaveOptions:
 102               bootSource:
 103                 volumeFromImage:
 104                   name: {{ image_name }}
 105                   volumeSize: {{ volume_size }}
 106 {%- if hardware_id  %}
 107               hardwareId: {{ hardware_id }}{% endif %}
 108 {%- if instance_cap %}
 109               instanceCap: {{ instance_cap }}{% endif %}
 110 {%- if num_executors %}
 111               numExectorts: {{ num_executors }}{% endif %}
 112 {%- if retention_time %}
 113               retentionTime: {{ retention_time }}
 114 {%- else %}
 115               retentionTime: 0{% endif %}
 116 """
 117 footertemplate = """\
 118         zone: {{ cloud_zone}}
 119 """
 120
 121 # Command line args section
 122 def dir_path(path):
 123     if os.path.isdir(path):
 124         return path
 125     else:
 126         raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path")
 127
 128 def parse_arguments():
 129     parser = argparse.ArgumentParser(
 130         description="Create JCasC yaml from path to Jenkins config dir.")
 131
 132     parser.add_argument("--path", type=dir_path,
 133                         help="Path to jenkins-config directory")
 134     parser.add_argument("--name", type=str,
 135                         help="Cloud name (e.g \"cattle\")")
 136
 137     parser.add_argument(
 138         "-s", "--sandbox",
 139         help="Configuration is being created for a sandbox",
 140         dest="sandbox", action="store_true"
 141     )
 142
 143     return parser.parse_args()
 144
 145 parsed_args = parse_arguments()
 146 path = (parsed_args.path)
 147 path = ("{}**/*.cfg".format(path))
 148
 149 # Sandbox switch section
 150 section_cloud = {}
 151 name_prefix = "prd"
 152 if parsed_args.sandbox:
 153     name_prefix = "snd"
 154     section_cloud.update(is_sandbox=True)
 155
 156 # Config parser from merged files section
 157 def read_config(filename):
 158     shortname = os.path.basename(filename)
 159     header_name = os.path.splitext(shortname)[0]
 160     with open(filename, "r", encoding="utf_8") as config_file:
 161         config.read_file(add_section_header(config_file, header_name), source=filename)
 162     return config
 163
 164 # Cfg files are not real ini files, need to add section headers.
 165 def add_section_header(properties_file, header_name):
 166     yield "[{}]\n".format(header_name)
 167     for line in properties_file:
 168         yield line
 169
 170 config = configparser.ConfigParser()
 171 for filename in glob.iglob(path, recursive=True):
 172     # config_parser_merged is the configparser object with all the configs from *.cfg
 173     config_parser_merged = read_config(filename)
 174
 175 # Global cloud config section
 176 cloud_config = (config.items("cloud"))
 177 name = parsed_args.name
 178 cloud_config_final = (*cloud_config, ("cloud_name", name))
 179
 180 for index, _ in enumerate(cloud_config_final):
 181     key = cloud_config_final[index][0]
 182     value = cloud_config_final[index][1]
 183     if value in lookuptable.keys():
 184         value = lookuptable[value]
 185     section_cloud[key] = value
 186
 187 j2_template = Template(maintemplate)
 188 print(j2_template.render(section_cloud))
 189
 190
 191 # All machines section
 192 for section in config_parser_merged.sections():
 193     if section != "cloud":
 194         machine = (config.items(section))
 195         section_all_machines = {}
 196         for index, _ in enumerate(machine):
 197             key = machine[index][0]
 198             value = machine[index][1]
 199             if value in lookuptable.keys():
 200                 value = lookuptable[value]
 201             section_all_machines[key] = value
 202
 203         # Default volume size of 10
 204         if "volume_size" not in section_all_machines:
 205             section_all_machines.update(volume_size="10")
 206         if "labels" not in section_all_machines:
 207             print("LABELS not Set in builder config")
 208             print(section_all_machines)
 209             exit(1)
 210
 211         j2_template = Template(machinetemplate)
 212         section_all_machines.update(name_prefix=name_prefix)
 213         print(j2_template.render(section_all_machines))
 214
 215
 216 # Footer section
 217 j2_template = Template(footertemplate)
 218 print(j2_template.render(section_cloud))