jenkins-admin/create_jenkins_clouds_openstack_yaml.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2020 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 """Create JCasC yaml file for the given Openstack cloud config"""
  12
  13 import argparse
  14 import configparser
  15 import glob
  16 import os
  17
  18 from jinja2 import Template
  19
  20 # Template section
  21 lookuptable = {
  22     "acumos-highcpu-4-avx": "c720c1f8-62e9-4695-823d-f7f54db46c86",
  23     "lf-highcpu-2": "1051d06a-61ea-45e3-b9b4-93de92880b27",
  24     "lf-highcpu-4": "35eb8e11-490f-4d1a-9f19-76091fc04547",
  25     "lf-highcpu-8": "68af673f-54ee-4255-871c-158c18e4f643",
  26     "lf-standard-1": "7d76cbb0-f547-4c2c-beaf-554f33832721",
  27     "lf-standard-2": "ef454088-7839-42a0-bf23-5e0ab6386a27",
  28     "lf-standard-4": "bd74e1e6-c2ed-475b-ab3f-2ce13936a215",
  29     "lf-standard-8": "32d74024-8418-41b6-9675-b77816748148",
  30     "odl-highcpu-2": "def1b86f-b7f8-4943-b430-4a0599170006",
  31     "odl-highcpu-4": "0c8ec795-2ff8-4623-98cf-b4c1d92bb37c",
  32     "odl-highcpu-8": "458d6499-e2c8-4580-aa88-a4a04a33ee25",
  33     "odl-standard-1": "35800a3f-0c69-428d-b5cb-136d17d46c48",
  34     "odl-standard-2": "8ead227a-acfe-4290-be70-fbab92e6dd2f",
  35     "odl-standard-4": "f76fb18d-d5fb-4175-95c1-b29d8039d102",
  36     "odl-standard-8": "ba38b1af-4f87-4e4e-860e-94e8329d0d78",
  37     "v1-standard-1": "bbcb7eb5-5c8d-498f-9d7e-307c575d3566",
  38     "v1-standard-2": "ca2a6e9c-2236-4107-8905-7ae9427132ff",
  39     "v1-standard-4": "5cf64088-893b-46b5-9bb1-ee020277635d",
  40     "v1-standard-8": "6eec77b4-2286-4e3b-b3f0-cac67aa2c727",
  41     "v1-standard-16": "2f8730dd-7688-4b72-a512-99fb9a482414",
  42     "v1-standard-32": "0da688af-bb0c-4116-a158-cbf37240a8b1",
  43     "v1-standard-48": "69471d69-61fb-40dd-bdf3-e6b7f4e6daa3",
  44     "v1-standard-64": "0c1d9008-f546-4608-9e8f-f8bdaec8dddd",
  45     "v1-standard-96": "5741c775-92a4-4488-bd77-dd7b08e2be81",
  46     "v1-standard-128": "e82d0a5b-8031-4526-9a5d-a15f7b4d48ff",
  47     "v2-highcpu-1": "c04abb7a-2b61-4ed3-8ce8-6c40ad9df750",
  48     "v2-highcpu-2": "03bdf34e-8905-46bc-a4b9-8dbf94b6e06d",
  49     "v2-highcpu-4": "3b72e578-7875-4e0e-91b7-71ed292f3ca2",
  50     "v2-highcpu-8": "221de281-95ec-414f-8e42-c86c9e0b318d",
  51     "v2-highcpu-16": "ddd6863a-ef4f-475c-9aee-61d46898651d",
  52     "v2-highcpu-32": "21dfb8a3-c472-4a2c-a8e1-4da8de415ff8",
  53     "v2-standard-1": "52a01f6b-e660-48b5-8c06-5fb2a0fab0ec",
  54     "v2-standard-2": "ac2c4d17-8d6f-4e3c-a9eb-57c155f0a949",
  55     "v2-standard-4": "d9115351-defe-4fac-986b-1a1187e2c31c",
  56     "v2-standard-8": "e6fe2e37-0e38-438c-8fa5-fc2d79d0a7bb",
  57     "v2-standard-16": "9e4b01cd-6744-4120-aafe-1b5e17584919",
  58     "v2-standard-360": "f0d27f44-a410-4f0f-9781-d722f5b5489e",
  59     "v3-standard-2": "d6906d2a-e83f-42be-b33e-fbaeb5c511cb",
  60     "v3-standard-4": "5f1eb09f-e764-4642-a16f-a7230ec025e7",
  61     "v3-standard-8": "47d3707a-c6c6-46ea-a15b-095e336b1edc",
  62     "v3-standard-16": "8587d458-69de-4fc5-be51-c5e671bc35d5",
  63     "v3-standard-32": "3e01b39f-45a9-4b7b-b6dc-14378433dc36",
  64     "v3-standard-48": "06a0e8b7-949a-439d-a185-208ae9e645b2",
  65     "v3-standard-64": "402a2759-cc01-481d-a8b7-2c7056f153f7",
  66     "v3-standard-96": "883b0564-dec6-4e51-88c7-83d86994fcf0"
  67 }
  68 maintemplate = """\
  69 ---
  70 jenkins:
  71   clouds:
  72     - openstack:
  73         credentialsId: {{ cloud_credential_id }}
  74         endPointUrl: {{ cloud_url }}
  75         ignoreSsl: {{ cloud_ignore_ssl }}
  76         name: {{ cloud_name }}
  77         slaveOptions:
  78           availabilityZone: {{ availability_zone }}
  79           bootSource:
  80             volumeFromImage:
  81               name: {{ image_name }}
  82               volumeSize: {{ volume_size }}
  83           fsRoot: {{ fs_root }}
  84           hardwareId: {{ hardware_id }}
  85 {%- if is_sandbox is defined %}
  86           instanceCap: {{ sandbox_cap }}
  87 {%- else %}
  88           instanceCap: {{ instance_cap }}{% endif %}
  89           keyPairName: {{ key_pair_name }}
  90           launcherFactory:
  91             ssh:
  92               credentialsId: {{ key_pair_name }}
  93           networkId: {{ network_id }}
  94           retentionTime: {{ retention_time }}
  95           userDataId: {{ user_data_id }}
  96         templates:
  97 """
  98 machinetemplate = """\
  99           - labels: {{ labels }}
 100             name: {{ name_prefix }}-{{ agent_name }}
 101             slaveOptions:
 102               bootSource:
 103                 {{ image_type }}:
 104                   name: {{ image_name }}
 105 {%- if image_type == "volumeFromImage"  %}
 106                   volumeSize: {{ volume_size }}{% endif %}
 107 {%- if hardware_id  %}
 108               hardwareId: {{ hardware_id }}{% endif %}
 109 {%- if instance_cap %}
 110               instanceCap: {{ instance_cap }}{% endif %}
 111 {%- if num_executors %}
 112               numExectorts: {{ num_executors }}{% endif %}
 113 {%- if retention_time %}
 114               retentionTime: {{ retention_time }}
 115 {%- else %}
 116               retentionTime: 0{% endif %}
 117 """
 118 footertemplate = """\
 119         zone: {{ cloud_zone}}
 120 """
 121
 122 # Command line args section
 123 def dir_path(path):
 124     if os.path.isdir(path):
 125         return path
 126     else:
 127         raise argparse.ArgumentTypeError(f"readable_dir: {path} is not a valid path")
 128
 129 def parse_arguments():
 130     parser = argparse.ArgumentParser(
 131         description="Create JCasC yaml from path to Jenkins config dir.")
 132
 133     parser.add_argument("--path", type=dir_path,
 134                         help="Path to jenkins-config directory")
 135     parser.add_argument("--name", type=str,
 136                         help="Cloud name (e.g \"cattle\")")
 137
 138     parser.add_argument(
 139         "-s", "--sandbox",
 140         help="Configuration is being created for a sandbox",
 141         dest="sandbox", action="store_true"
 142     )
 143
 144     return parser.parse_args()
 145
 146 parsed_args = parse_arguments()
 147 path = (parsed_args.path)
 148 path = ("{}**/*.cfg".format(path))
 149
 150 # Sandbox switch section
 151 section_cloud = {}
 152 name_prefix = "prd"
 153 if parsed_args.sandbox:
 154     name_prefix = "snd"
 155     section_cloud.update(is_sandbox=True)
 156
 157 # Config parser from merged files section
 158 def read_config(filename):
 159     shortname = os.path.basename(filename)
 160     header_name = os.path.splitext(shortname)[0]
 161     with open(filename, "r", encoding="utf_8") as config_file:
 162         config.read_file(add_section_header(config_file, header_name), source=filename)
 163     return config
 164
 165 # Cfg files are not real ini files, need to add section headers.
 166 def add_section_header(properties_file, header_name):
 167     yield "[{}]\n".format(header_name)
 168     for line in properties_file:
 169         yield line
 170
 171 config = configparser.ConfigParser()
 172 for filename in glob.iglob(path, recursive=True):
 173     # config_parser_merged is the configparser object with all the configs from *.cfg
 174     config_parser_merged = read_config(filename)
 175
 176 # Global cloud config section
 177 cloud_config = (config.items("cloud"))
 178 name = parsed_args.name
 179 cloud_config_final = (*cloud_config, ("cloud_name", name))
 180
 181 for index, _ in enumerate(cloud_config_final):
 182     key = cloud_config_final[index][0]
 183     value = cloud_config_final[index][1]
 184     if value in lookuptable.keys():
 185         value = lookuptable[value]
 186     section_cloud[key] = value
 187
 188 j2_template = Template(maintemplate)
 189 print(j2_template.render(section_cloud))
 190
 191
 192 # All machines section
 193 for section in config_parser_merged.sections():
 194     if section != "cloud":
 195         machine = (config.items(section))
 196         section_all_machines = {}
 197         for index, _ in enumerate(machine):
 198             key = machine[index][0]
 199             value = machine[index][1]
 200             if value in lookuptable.keys():
 201                 value = lookuptable[value]
 202             section_all_machines[key] = value
 203
 204         if "volume_size" not in section_all_machines:
 205             section_all_machines.update(image_type="image")
 206         else:
 207             section_all_machines.update(image_type="volumeFromImage")
 208
 209         # Naming and labels
 210         section_all_machines.update(agent_name=section)
 211         if "labels" not in section_all_machines:
 212             # "section" is the name of the cloud agent, which is the default label
 213             section_all_machines.update(labels=section)
 214         elif section not in section_all_machines["labels"]:
 215             labels = section + " " + section_all_machines["labels"]
 216             section_all_machines.update(labels=labels)
 217
 218         j2_template = Template(machinetemplate)
 219         section_all_machines.update(name_prefix=name_prefix)
 220         print(j2_template.render(section_all_machines))
 221
 222
 223 # Footer section
 224 j2_template = Template(footertemplate)
 225 print(j2_template.render(section_cloud))