From 6e2fdc9fb5be1d9843d14b6aa6f51356f9655459 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Fri, 16 Jun 2023 14:00:04 +1000 Subject: [PATCH] Feat!: Convert packer templates to HCL2 format MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit As of packer version 1.7.0 HCL2 is the preferred way to write Packer templates. HCL2 preserves existing workflows while leveraging HCL2’s advanced features like variable interpolation and configuration composability. Migrate packer templates from JSON to HCL2 format. JSON format templates are deprecated and no longer works with packer version > 1.9.x. Packer version 1.9.1 will be minimum required version for packer jobs. This version requires installing the cloud specific plugins through packer config and needs to be initalized and downloaded with `packer init` before running `packer build`. Add support for host key algorithms to work with local ssh proxy without which packer builds results in "failed to handshake" error. Workaround is to pass additional params with "extra_arguments". Support for '.json' templates will be removed from common-packer in subsequent releases. All projects specific templates not available in this repository are required to convert existing '.json' to '.pkr.hcl' format. This change requires updating CI jobs with the additional steps. Ref: https://developer.hashicorp.com/packer/docs/templates/hcl_templates https://github.com/hashicorp/packer-plugin-openstack/blob/main/README.md https://github.com/hashicorp/packer-plugin-ansible/issues/140 Issue: RELENG-4764 Change-Id: Ie63d9551bd1bab224dc9335d45a21d5ee3e09550 Signed-off-by: Anil Belur --- docs/requirements.rst | 2 +- docs/usage.rst | 2 +- ...ert-templates-json-to-hcl-aaf848118544e70f.yaml | 46 +++++ templates/builder-aws.pkr.hcl | 191 +++++++++++++++++++++ templates/builder.pkr.hcl | 152 ++++++++++++++++ templates/devstack-pre-pip-yoga.pkr.hcl | 155 +++++++++++++++++ templates/devstack.pkr.hcl | 157 +++++++++++++++++ templates/docker-aws.pkr.hcl | 181 +++++++++++++++++++ templates/docker.pkr.hcl | 167 ++++++++++++++++++ templates/variables.auto.pkr.hcl | 139 +++++++++++++++ templates/windows-builder.pkr.hcl | 167 ++++++++++++++++++ vars/centos-7-arm64.pkrvars.hcl | 10 ++ vars/centos-7.pkrvars.hcl | 8 + vars/centos-8.pkrvars.hcl | 8 + vars/centos-cs-8.pkrvars.hcl | 8 + vars/centos-cs-9.pkrvars.hcl | 8 + vars/cloud-env-aws.pkrvars.hcl | 6 + vars/cloud-env.pkrvars.hcl | 6 + vars/ubuntu-16.04-arm64.pkrvars.hcl | 10 ++ vars/ubuntu-16.04.pkrvars.hcl | 8 + vars/ubuntu-18.04-arm64.pkrvars.hcl | 10 ++ vars/ubuntu-18.04.pkrvars.hcl | 8 + vars/ubuntu-20.04-arm64.pkrvars.hcl | 10 ++ vars/ubuntu-20.04.pkrvars.hcl | 8 + vars/windows-server-2016.pkrvars.hcl | 8 + 25 files changed, 1473 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/convert-templates-json-to-hcl-aaf848118544e70f.yaml create mode 100644 templates/builder-aws.pkr.hcl create mode 100644 templates/builder.pkr.hcl create mode 100644 templates/devstack-pre-pip-yoga.pkr.hcl create mode 100644 templates/devstack.pkr.hcl create mode 100644 templates/docker-aws.pkr.hcl create mode 100644 templates/docker.pkr.hcl create mode 100644 templates/variables.auto.pkr.hcl create mode 100644 templates/windows-builder.pkr.hcl create mode 100644 vars/centos-7-arm64.pkrvars.hcl create mode 100644 vars/centos-7.pkrvars.hcl create mode 100644 vars/centos-8.pkrvars.hcl create mode 100644 vars/centos-cs-8.pkrvars.hcl create mode 100644 vars/centos-cs-9.pkrvars.hcl create mode 100644 vars/cloud-env-aws.pkrvars.hcl create mode 100644 vars/cloud-env.pkrvars.hcl create mode 100644 vars/ubuntu-16.04-arm64.pkrvars.hcl create mode 100644 vars/ubuntu-16.04.pkrvars.hcl create mode 100644 vars/ubuntu-18.04-arm64.pkrvars.hcl create mode 100644 vars/ubuntu-18.04.pkrvars.hcl create mode 100644 vars/ubuntu-20.04-arm64.pkrvars.hcl create mode 100644 vars/ubuntu-20.04.pkrvars.hcl create mode 100644 vars/windows-server-2016.pkrvars.hcl diff --git a/docs/requirements.rst b/docs/requirements.rst index e0f42de..84fa5c5 100644 --- a/docs/requirements.rst +++ b/docs/requirements.rst @@ -3,7 +3,7 @@ Requirements ############ * Ansible 2.9.27 or later -* Packer 1.8.2 or later +* Packer 1.9.1 or later Install Ansible via pip in a virtualenv to build images. diff --git a/docs/usage.rst b/docs/usage.rst index d64c583..7ee71ba 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -30,7 +30,7 @@ templates provided by common-packer as necessary. # Instructions assume the working directory is the ci-management repo root cd packer mkdir provision templates - ln -rs common-packer/templates/builder.json templates/builder.json + ln -rs common-packer/templates/builder.pkr.hcl templates/builder.pkr.hcl cp common-packer/provision/local-builder.yaml provision/local-builder.yaml .. _custom-template: diff --git a/releasenotes/notes/convert-templates-json-to-hcl-aaf848118544e70f.yaml b/releasenotes/notes/convert-templates-json-to-hcl-aaf848118544e70f.yaml new file mode 100644 index 0000000..8b09acc --- /dev/null +++ b/releasenotes/notes/convert-templates-json-to-hcl-aaf848118544e70f.yaml @@ -0,0 +1,46 @@ +--- +prelude: > + As of packer version 1.7.0 HCL2 is the preferred way to write Packer + templates. HCL2 preserves existing workflows while leveraging HCL2’s + advanced features like variable interpolation and configuration + composability. +upgrade: + - | + Migrate packer templates from JSON to HCL2 format. JSON format templates + are deprecated and no longer works with packer version > 1.9.x. + + Existing JSON templates can be converted to '.pkr.hcl' using: + + .. code-block:: bash + + packer hcl2_upgrade -with-anotations + + Packer version 1.9.1 will be minimum required version for packer jobs. + This version requires installing the cloud specific plugin through + packer config and needs to be initalize and download before running + `packer build`. + + `.pkr.hcl` includes the sources and builds are defined. + `.auto.pkrvars.hcl` includes variables that are loaded automatically. + These variables load automatically from the same directory and are common + across templates. `variables.pkr.hcl` includes variable declarations that + are common across templates. + + Reference: + https://developer.hashicorp.com/packer/guides/hcl/variables + https://developer.hashicorp.com/packer/docs/templates/hcl_templates + https://github.com/hashicorp/packer-plugin-openstack/blob/main/README.md +issues: + - | + Add support for host key algorithms to work with local ssh proxy + without which packer builds results in "failed to handshake" error. + Workaround is to pass additional params with "extra_arguments". + + Reference: + https://github.com/hashicorp/packer-plugin-ansible/issues/140 +deprecations: + - | + Support for '.json' templates will be removed from common-packer in + subsequent release to give enough time for projects consuming to upgrade. + All projects specific templates not available in this repository are + required to convert existing '.json' to '.pkr.hcl' format. diff --git a/templates/builder-aws.pkr.hcl b/templates/builder-aws.pkr.hcl new file mode 100644 index 0000000..0878e63 --- /dev/null +++ b/templates/builder-aws.pkr.hcl @@ -0,0 +1,191 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.2.6" + source = "github.com/hashicorp/amazon" + } + } +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "aws_access_key" { + type = string + default = null +} + +variable "aws_instance_type" { + type = string + default = "t2.micro" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "aws_secret_key" { + type = string + default = null +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "flavor" { + type = string + default = null +} + +variable "security_group_id" { + type = string + default = null +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "ssh_user" { + type = string + default = null +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "subnet_id" { + type = string + default = null +} + +variable "vpc_id" { + type = string + default = null +} + +data "amazon-ami" "builder-aws" { + access_key = "${var.aws_access_key}" + filters = { + name = "${var.source_ami_filter_name}" + product-code = "${var.source_ami_filter_product_code}" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["${var.source_ami_filter_owner}"] + region = "${var.aws_region}" + secret_key = "${var.aws_secret_key}" +} + +source "amazon-ebs" "aws" { + access_key = "${var.aws_access_key}" + ami_name = "ZZCI - ${var.distro} - builder-aws - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_type = "${var.aws_instance_type}" + region = "${var.aws_region}" + secret_key = "${var.aws_secret_key}" + security_group_id = "${var.security_group_id}" + source_ami = "${data.amazon-ami.builder-aws.id}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + subnet_id = "${var.subnet_id}" + user_data_file = "${var.cloud_user_data}" + vpc_id = "${var.vpc_id}" +} + +build { + description = "Build an AMI for use as a CI builder" + + sources = ["source.amazon-ebs.aws"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=False", + "ANSIBLE_HOST_KEY_CHECKING=False", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = [ + "--scp-extra-args", "'-O'", + "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa" + ] + playbook_file = "provision/local-builder.yaml" + skip_version_check = true + user = "${var.ssh_user}" + } +} diff --git a/templates/builder.pkr.hcl b/templates/builder.pkr.hcl new file mode 100644 index 0000000..25e7c7b --- /dev/null +++ b/templates/builder.pkr.hcl @@ -0,0 +1,152 @@ +packer { + required_plugins { + openstack = { + version = ">= 1.0.0" + source = "github.com/hashicorp/openstack" + } + } +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "base_image" { + type = string +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "cloud_network" { + type = string +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_user_data" { + type = string +} + +variable "distro" { + type = string +} + +variable "docker_source_image" { + type = string +} + +variable "flavor" { + type = string + default = "v3-standard-2" +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "ssh_user" { + type = string +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +source "docker" "builder" { + changes = ["ENTRYPOINT [\"\"]", "CMD [\"\"]"] + commit = true + image = "${var.docker_source_image}" +} + +source "openstack" "builder" { + flavor = "${var.flavor}" + image_disk_format = "${var.vm_image_disk_format}" + image_name = "ZZCI - ${var.distro} - builder - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_name = "${var.distro}-builder-${uuidv4()}" + metadata = { + ci_managed = "yes" + } + networks = ["${var.cloud_network}"] + region = "${var.cloud_region}" + source_image_name = "${var.base_image}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + use_blockstorage_volume = "${var.vm_use_block_storage}" + user_data_file = "${var.cloud_user_data}" + volume_size = "${var.vm_volume_size}" +} + +build { + sources = ["source.docker.builder", "source.openstack.builder"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = ["ANSIBLE_NOCOWS=1", "ANSIBLE_PIPELINING=True", "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", "ANSIBLE_STDOUT_CALLBACK=debug"] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = ["--scp-extra-args", "'-O'", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"] + playbook_file = "provision/local-builder.yaml" + skip_version_check = true + } +} diff --git a/templates/devstack-pre-pip-yoga.pkr.hcl b/templates/devstack-pre-pip-yoga.pkr.hcl new file mode 100644 index 0000000..3453914 --- /dev/null +++ b/templates/devstack-pre-pip-yoga.pkr.hcl @@ -0,0 +1,155 @@ +packer { + required_plugins { + openstack = { + version = ">= 1.0.0" + source = "github.com/hashicorp/openstack" + } + } +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_user_data" { + type = string +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "distro" { + type = string +} + +variable "docker_source_image" { + type = string +} + +variable "flavor" { + type = string + default = "v3-standard-2" +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "ssh_user" { + type = string + default = null +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +source "docker" "devstack-pre-pip-yoga" { + changes = ["ENTRYPOINT [\"\"]", "CMD [\"\"]"] + commit = true + image = "${var.docker_source_image}" +} + +source "openstack" "devstack-pre-pip-yoga" { + flavor = "${var.flavor}" + image_disk_format = "${var.vm_image_disk_format}" + image_name = "ZZCI - ${var.distro} - devstack-yoga - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_name = "${var.distro}-devstack-yoga-${uuidv4()}" + metadata = { + ci_managed = "yes" + } + networks = ["${var.cloud_network}"] + region = "${var.cloud_region}" + source_image_name = "${var.base_image}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + use_blockstorage_volume = "${var.vm_use_block_storage}" + user_data_file = "${var.cloud_user_data}" + volume_size = "${var.vm_volume_size}" +} + +build { + sources = ["source.docker.devstack-pre-pip-yoga", "source.openstack.devstack-pre-pip-yoga"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = ["ANSIBLE_DEBUG=False", "ANSIBLE_NOCOWS=1", "ANSIBLE_PIPELINING=True", "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", "ANSIBLE_STDOUT_CALLBACK=debug"] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = ["--extra-vars", "os_branch=stable/yoga rdo_branch=yoga", "--scp-extra-args", "'-O'", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"] + playbook_file = "provision/devstack-pre-pip-centos.yaml" + skip_version_check = true + } +} diff --git a/templates/devstack.pkr.hcl b/templates/devstack.pkr.hcl new file mode 100644 index 0000000..9a815e5 --- /dev/null +++ b/templates/devstack.pkr.hcl @@ -0,0 +1,157 @@ +packer { + required_plugins { + openstack = { + version = ">= 1.0.0" + source = "github.com/hashicorp/openstack" + } + } +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "flavor" { + type = string + default = "v3-standard-2" +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "ssh_user" { + type = string +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +source "docker" "devstack" { + changes = ["ENTRYPOINT [\"\"]", "CMD [\"\"]"] + commit = true + image = "${var.docker_source_image}" +} + +source "openstack" "devstack" { + flavor = "${var.flavor}" + image_disk_format = "${var.vm_image_disk_format}" + image_name = "ZZCI - ${var.distro} - devstack - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_name = "${var.distro}-devstack-${uuidv4()}" + metadata = { + ci_managed = "yes" + } + networks = ["${var.cloud_network}"] + region = "${var.cloud_region}" + source_image_name = "${var.base_image}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + use_blockstorage_volume = "${var.vm_use_block_storage}" + user_data_file = "${var.cloud_user_data}" + volume_size = "${var.vm_volume_size}" +} + +build { + sources = ["source.docker.devstack", "source.openstack.devstack"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = ["ANSIBLE_NOCOWS=1", "ANSIBLE_PIPELINING=True", "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", "ANSIBLE_STDOUT_CALLBACK=debug"] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = ["--scp-extra-args", "'-O'", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"] + playbook_file = "provision/devstack-centos.yaml" + skip_version_check = true + } +} diff --git a/templates/docker-aws.pkr.hcl b/templates/docker-aws.pkr.hcl new file mode 100644 index 0000000..63f0f50 --- /dev/null +++ b/templates/docker-aws.pkr.hcl @@ -0,0 +1,181 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.2.6" + source = "github.com/hashicorp/amazon" + } + } +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "aws_access_key" { + type = string + default = null +} + +variable "aws_instance_type" { + type = string + default = "t2.micro" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "aws_secret_key" { + type = string + default = null +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "flavor" { + type = string + default = null +} + +variable "security_group_id" { + type = string + default = null +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "ssh_user" { + type = string + default = null +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "subnet_id" { + type = string + default = null +} + +variable "vpc_id" { + type = string + default = null +} + +data "amazon-ami" "docker-aws" { + access_key = "${var.aws_access_key}" + filters = { + name = "${var.source_ami_filter_name}" + product-code = "${var.source_ami_filter_product_code}" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["${var.source_ami_filter_owner}"] + region = "${var.aws_region}" + secret_key = "${var.aws_secret_key}" +} + +source "amazon-ebs" "aws" { + access_key = "${var.aws_access_key}" + ami_name = "ZZCI - ${var.distro} - docker-aws - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_type = "${var.aws_instance_type}" + region = "${var.aws_region}" + secret_key = "${var.aws_secret_key}" + security_group_id = "${var.security_group_id}" + source_ami = "${data.amazon-ami.docker-aws.id}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + subnet_id = "${var.subnet_id}" + user_data_file = "${var.cloud_user_data}" + vpc_id = "${var.vpc_id}" +} + +build { + description = "Build an AMI for use as a CI builder" + + sources = ["source.amazon-ebs.aws"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = ["ANSIBLE_NOCOWS=1", "ANSIBLE_PIPELINING=False", "ANSIBLE_HOST_KEY_CHECKING=False", "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", "ANSIBLE_STDOUT_CALLBACK=debug"] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = ["--scp-extra-args", "'-O'", "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"] + playbook_file = "provision/local-docker.yaml" + skip_version_check = true + user = "${var.ssh_user}" + } +} diff --git a/templates/docker.pkr.hcl b/templates/docker.pkr.hcl new file mode 100644 index 0000000..c4ebe54 --- /dev/null +++ b/templates/docker.pkr.hcl @@ -0,0 +1,167 @@ +packer { + required_plugins { + openstack = { + version = ">= 1.0.0" + source = "github.com/hashicorp/openstack" + } + } +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "flavor" { + type = string + default = "v2-highcpu-1" +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "ssh_user" { + type = string + default = null +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +source "docker" "docker" { + changes = ["ENTRYPOINT [\"\"]", "CMD [\"\"]"] + commit = true + image = "${var.docker_source_image}" +} + +source "openstack" "docker" { + flavor = "${var.flavor}" + image_disk_format = "${var.vm_image_disk_format}" + image_name = "ZZCI - ${var.distro} - docker - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_name = "${var.distro}-docker-${uuidv4()}" + metadata = { + ci_managed = "yes" + } + networks = ["${var.cloud_network}"] + region = "${var.cloud_region}" + source_image_name = "${var.base_image}" + ssh_proxy_host = "${var.ssh_proxy_host}" + ssh_username = "${var.ssh_user}" + use_blockstorage_volume = "${var.vm_use_block_storage}" + user_data_file = "${var.cloud_user_data}" + volume_size = "${var.vm_volume_size}" +} + +build { + sources = ["source.docker.docker", "source.openstack.docker"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = [ + "--scp-extra-args", "'-O'", + "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa" + ] + playbook_file = "provision/local-docker.yaml" + skip_version_check = true + } +} diff --git a/templates/variables.auto.pkr.hcl b/templates/variables.auto.pkr.hcl new file mode 100644 index 0000000..5fd45ef --- /dev/null +++ b/templates/variables.auto.pkr.hcl @@ -0,0 +1,139 @@ +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "aws_access_key" { + type = string + default = null +} + +variable "aws_instance_type" { + type = string + default = "t2.micro" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "aws_secret_key" { + type = string + default = null +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "flavor" { + type = string + default = "v2-highcpu-4" +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "security_group_id" { + type = string + default = null +} + +variable "ssh_proxy_host" { + type = string + default = "" +} + +variable "ssh_user" { + type = string + default = null +} + +variable "subnet_id" { + type = string + default = null +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +variable "vpc_id" { + type = string + default = null +} diff --git a/templates/windows-builder.pkr.hcl b/templates/windows-builder.pkr.hcl new file mode 100644 index 0000000..94065fe --- /dev/null +++ b/templates/windows-builder.pkr.hcl @@ -0,0 +1,167 @@ +packer { + required_plugins { + openstack = { + version = ">= 1.0.0" + source = "github.com/hashicorp/openstack" + } + } +} + +variable "cloud_auth_url" { + type = string + default = null +} + +variable "cloud_tenant" { + type = string + default = null +} + +variable "cloud_user" { + type = string + default = null +} + +variable "cloud_pass" { + type = string + default = null +} + +variable "ansible_roles_path" { + type = string + default = ".galaxy" +} + +variable "arch" { + type = string + default = "x86_64" +} + +variable "base_image" { + type = string + default = null +} + +variable "cloud_network" { + type = string + default = null +} + +variable "cloud_region" { + type = string + default = "ca-ymq-1" +} + +variable "cloud_user_data" { + type = string + default = null +} + +variable "docker_source_image" { + type = string + default = null +} + +variable "distro" { + type = string + default = null +} + +variable "flavor" { + type = string + default = "v2-highcpu-4" +} + +variable "ssh_proxy_host" { + type = string + default = null +} + +variable "ssh_user" { + type = string + default = null +} + +variable "source_ami_filter_name" { + type = string + default = null +} + +variable "source_ami_filter_product_code" { + type = string + default = null +} + +variable "source_ami_filter_owner" { + type = string + default = null +} + +variable "vm_image_disk_format" { + type = string + default = "" +} + +variable "vm_use_block_storage" { + type = string + default = "true" +} + +variable "vm_volume_size" { + type = string + default = "20" +} + +source "openstack" "windows-builder" { + communicator = "winrm" + flavor = "${var.flavor}" + image_disk_format = "${var.vm_image_disk_format}" + image_name = "ZZCI - ${var.distro} - win-builder - ${var.arch} - ${legacy_isotime("20060102-150405.000")}" + instance_name = "${var.distro}-win-builder-${uuidv4()}" + metadata = { + ci_managed = "yes" + } + networks = ["${var.cloud_network}"] + region = "${var.cloud_region}" + source_image_name = "${var.base_image}" + use_blockstorage_volume = "${var.vm_use_block_storage}" + user_data_file = "${var.cloud_user_data}" + volume_size = "${var.vm_volume_size}" + winrm_insecure = true + winrm_password = "W!nRMB00tStrap." + winrm_timeout = "3600s" + winrm_use_ssl = true + winrm_username = "Administrator" +} + +build { + sources = ["source.openstack.windows-builder"] + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + scripts = ["common-packer/provision/install-python.sh"] + } + + provisioner "shell-local" { + command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}" + } + + provisioner "ansible" { + ansible_env_vars = [ + "ANSIBLE_NOCOWS=1", + "ANSIBLE_PIPELINING=True", + "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}", + "ANSIBLE_CALLBACK_WHITELIST=profile_tasks", + "ANSIBLE_STDOUT_CALLBACK=debug" + ] + command = "./common-packer/ansible-playbook.sh" + extra_arguments = [ + "--extra-vars", "ansible_shell_type=powershell", + "--extra-vars", "ansible_shell_executable=None", + "--scp-extra-args", "'-O'", + "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa" + ] + playbook_file = "provision/local-windows-builder.yaml" + skip_version_check = true + } +} diff --git a/vars/centos-7-arm64.pkrvars.hcl b/vars/centos-7-arm64.pkrvars.hcl new file mode 100644 index 0000000..30bff2c --- /dev/null +++ b/vars/centos-7-arm64.pkrvars.hcl @@ -0,0 +1,10 @@ +arch = "arm64" +base_image = "CentOS 7.8 (aarch64) [2020-04-22]" +cloud_user_data = "common-packer/provision/rh-user_data.sh" +distro = "CentOS 7" +docker_source_image = "arm64v8/centos:7" +flavor = "v3-standard-2" +source_ami_filter_name = "*CentOS 7*aarch64*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "6zber5dti4jyo9khb62tg49o8" +ssh_user = "centos" diff --git a/vars/centos-7.pkrvars.hcl b/vars/centos-7.pkrvars.hcl new file mode 100644 index 0000000..4474ce7 --- /dev/null +++ b/vars/centos-7.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "*CentOS-7*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "cvugziknvmxgqna9noibqnnsy" +base_image = "CentOS 7-x86_64 [2020-04-22]" +distro = "CentOS 7" +docker_source_image = "centos:7" +ssh_user = "centos" +cloud_user_data = "common-packer/provision/rh-user_data.sh" diff --git a/vars/centos-8.pkrvars.hcl b/vars/centos-8.pkrvars.hcl new file mode 100644 index 0000000..cf29f0e --- /dev/null +++ b/vars/centos-8.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "*CentOS-8*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "47k9ia2igxpcce2bzo8u3kj03" +base_image = "CentOS 8.2 (x86_64) [2020-06-11]" +distro = "CentOS 8" +docker_source_image = "centos:8" +ssh_user = "centos" +cloud_user_data = "common-packer/provision/rh-user_data.sh" diff --git a/vars/centos-cs-8.pkrvars.hcl b/vars/centos-cs-8.pkrvars.hcl new file mode 100644 index 0000000..a4c537a --- /dev/null +++ b/vars/centos-cs-8.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "*CentOS-cs-8*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "0418c980c296f36ce" +base_image = "CentOS Stream 8 (x86_64) [2022-01-25]" +distro = "CentOS Stream 8" +docker_source_image = "centos:8" +ssh_user = "centos" +cloud_user_data = "common-packer/provision/rh-user_data.sh" diff --git a/vars/centos-cs-9.pkrvars.hcl b/vars/centos-cs-9.pkrvars.hcl new file mode 100644 index 0000000..82a0897 --- /dev/null +++ b/vars/centos-cs-9.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "*CentOS-cs-9*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "0454011e44daf8e6d" +base_image = "CentOS Stream 9 (x86_64) [2022-02-24]" +distro = "CentOS Stream 9" +docker_source_image = "centos:9" +ssh_user = "cloud-user" +cloud_user_data = "common-packer/provision/rh-user_data.sh" diff --git a/vars/cloud-env-aws.pkrvars.hcl b/vars/cloud-env-aws.pkrvars.hcl new file mode 100644 index 0000000..d69c300 --- /dev/null +++ b/vars/cloud-env-aws.pkrvars.hcl @@ -0,0 +1,6 @@ +aws_secret_key = "SECRETKEY" +aws_access_key = "ACCESSKEY" +subnet_id = "subnet-null" +security_group_id = "sg-null" +vpc_id = "vpc-null" +ssh_proxy_host = "" diff --git a/vars/cloud-env.pkrvars.hcl b/vars/cloud-env.pkrvars.hcl new file mode 100644 index 0000000..efd7563 --- /dev/null +++ b/vars/cloud-env.pkrvars.hcl @@ -0,0 +1,6 @@ +cloud_auth_url = "" +cloud_tenant = "" +cloud_user = "" +cloud_pass = "" +cloud_network = "" +ssh_proxy_host = "" diff --git a/vars/ubuntu-16.04-arm64.pkrvars.hcl b/vars/ubuntu-16.04-arm64.pkrvars.hcl new file mode 100644 index 0000000..ebc7fd9 --- /dev/null +++ b/vars/ubuntu-16.04-arm64.pkrvars.hcl @@ -0,0 +1,10 @@ +arch = "arm64" +base_image = "Ubuntu 16.04 LTS (arm64) [2020-03-11]" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" +distro = "Ubuntu 16.04" +docker_source_image = "arm64v8/ubuntu:16.04" +flavor = "v3-standard-2" +source_ami_filter_name = "*ubuntu*16.04*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "7x1es1d2cyv0hf36v5nqpcp32" +ssh_user = "ubuntu" diff --git a/vars/ubuntu-16.04.pkrvars.hcl b/vars/ubuntu-16.04.pkrvars.hcl new file mode 100644 index 0000000..52005b7 --- /dev/null +++ b/vars/ubuntu-16.04.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "ubuntu/images/*ubuntu-xenial-16.04-amd64-server*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "csv6h7oyg29b7epjzg7qdr7no" +base_image = "LF - Ubuntu 16.04 LTS (2019-12-11)" +distro = "Ubuntu 16.04" +docker_source_image = "ubuntu:16.04" +ssh_user = "ubuntu" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" diff --git a/vars/ubuntu-18.04-arm64.pkrvars.hcl b/vars/ubuntu-18.04-arm64.pkrvars.hcl new file mode 100644 index 0000000..785528f --- /dev/null +++ b/vars/ubuntu-18.04-arm64.pkrvars.hcl @@ -0,0 +1,10 @@ +arch = "arm64" +base_image = "Ubuntu 18.04.6 LTS (aarch64) [2021-09-28]" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" +distro = "Ubuntu 18.04" +docker_source_image = "arm64v8/ubuntu:18.04" +flavor = "v3-standard-2" +source_ami_filter_name = "*ubuntu*18.04*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "zud1u4kjmxu2j2jf0n36beqt" +ssh_user = "ubuntu" diff --git a/vars/ubuntu-18.04.pkrvars.hcl b/vars/ubuntu-18.04.pkrvars.hcl new file mode 100644 index 0000000..21a2686 --- /dev/null +++ b/vars/ubuntu-18.04.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "ubuntu/images/*ubuntu-bionic-18.04-amd64-server-*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "3iplms73etrdhxdepv72l6ywj" +base_image = "Ubuntu 18.04 LTS [2022-04-11]" +distro = "Ubuntu 18.04" +docker_source_image = "ubuntu:18.04" +ssh_user = "ubuntu" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" diff --git a/vars/ubuntu-20.04-arm64.pkrvars.hcl b/vars/ubuntu-20.04-arm64.pkrvars.hcl new file mode 100644 index 0000000..90926c0 --- /dev/null +++ b/vars/ubuntu-20.04-arm64.pkrvars.hcl @@ -0,0 +1,10 @@ +arch = "arm64" +base_image = "Ubuntu 20.04.3 LTS (aarch64) [2021-10-04]" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" +distro = "Ubuntu 20.04" +docker_source_image = "arm64v8/ubuntu:20.04" +flavor = "v3-standard-2" +source_ami_filter_name = "*ubuntu*20.04*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "3j3t5wgc6orwshxai7wk5sux9" +ssh_user = "ubuntu" diff --git a/vars/ubuntu-20.04.pkrvars.hcl b/vars/ubuntu-20.04.pkrvars.hcl new file mode 100644 index 0000000..18c1cea --- /dev/null +++ b/vars/ubuntu-20.04.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "*ubuntu*20.04*" +source_ami_filter_owner = "aws-marketplace" +source_ami_filter_product_code = "a8jyynf4hjutohctm41o2z18m" +base_image = "Ubuntu 20.04 LTS [2022-04-11]" +distro = "Ubuntu 20.04" +docker_source_image = "ubuntu:20.04" +ssh_user = "ubuntu" +cloud_user_data = "common-packer/provision/ubuntu-user_data.sh" diff --git a/vars/windows-server-2016.pkrvars.hcl b/vars/windows-server-2016.pkrvars.hcl new file mode 100644 index 0000000..48a846b --- /dev/null +++ b/vars/windows-server-2016.pkrvars.hcl @@ -0,0 +1,8 @@ +source_ami_filter_name = "" +source_ami_filter_owner = "" +source_ami_filter_product_code = "" +base_image = "Windows Server Standard 2016 [2017-02-20]" +distro = "Windows 2016" +docker_source_image = "windows:2016" +ssh_user = "admin" +cloud_user_data = "common-packer/provision/windows-user_data.ps1" -- 2.16.6