Feat!: Convert packer templates to HCL2 format

As of packer version 1.7.0 HCL2 is the preferred way to write Packer
templates. HCL2 preserves existing workflows while leveraging HCL2’s
advanced features like variable interpolation and configuration
composability.

Migrate packer templates from JSON to HCL2 format. JSON format
templates are deprecated and no longer works with packer
version > 1.9.x.

Packer version 1.9.1 will be minimum required version for packer jobs.
This version requires installing the cloud specific plugins through
packer config and needs to be initalized and downloaded with
`packer init` before running `packer build`.

Add support for host key algorithms to work with local ssh proxy
without which packer builds results in "failed to handshake" error.
Workaround is to pass additional params with "extra_arguments".

Support for '.json' templates will be removed from common-packer in
subsequent releases. All projects specific templates not available in this
repository are required to convert existing '.json' to '.pkr.hcl' format.

This change requires updating CI jobs with the additional steps.

Ref:
https://developer.hashicorp.com/packer/docs/templates/hcl_templates
https://github.com/hashicorp/packer-plugin-openstack/blob/main/README.md
hashicorp/packer-plugin-ansible#140

Issue: RELENG-4764
Change-Id: Ie63d9551bd1bab224dc9335d45a21d5ee3e09550
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>

docs/requirements.rst

@@ -3,7 +3,7 @@ Requirements
 ############
 
 * Ansible 2.9.27 or later
-* Packer 1.8.2 or later
+* Packer 1.9.1 or later
 
 Install Ansible via pip in a virtualenv to build images.
 

docs/usage.rst

@@ -30,7 +30,7 @@ templates provided by common-packer as necessary.
    # Instructions assume the working directory is the ci-management repo root
    cd packer
    mkdir provision templates
-   ln -rs common-packer/templates/builder.json templates/builder.json
+   ln -rs common-packer/templates/builder.pkr.hcl templates/builder.pkr.hcl
    cp common-packer/provision/local-builder.yaml provision/local-builder.yaml
 
 .. _custom-template:

releasenotes/notes/convert-templates-json-to-hcl-aaf848118544e70f.yaml

@@ -0,0 +1,46 @@
+---
+prelude: >
+  As of packer version 1.7.0 HCL2 is the preferred way to write Packer
+  templates. HCL2 preserves existing workflows while leveraging HCL2’s
+  advanced features like variable interpolation and configuration
+  composability.
+upgrade:
+  - |
+    Migrate packer templates from JSON to HCL2 format. JSON format templates
+    are deprecated and no longer works with packer version > 1.9.x.
+
+    Existing JSON templates can be converted to '.pkr.hcl' using:
+
+    .. code-block:: bash
+
+       packer hcl2_upgrade -with-anotations <folder|filename>
+
+    Packer version 1.9.1 will be minimum required version for packer jobs.
+    This version requires installing the cloud specific plugin through
+    packer config and needs to be initalize and download before running
+    `packer build`.
+
+    `<temmplate>.pkr.hcl` includes the sources and builds are defined.
+    `.auto.pkrvars.hcl` includes variables that are loaded automatically.
+    These variables load automatically from the same directory and are common
+    across templates. `variables.pkr.hcl` includes variable declarations that
+    are common across templates.
+
+    Reference:
+    https://developer.hashicorp.com/packer/guides/hcl/variables
+    https://developer.hashicorp.com/packer/docs/templates/hcl_templates
+    https://github.com/hashicorp/packer-plugin-openstack/blob/main/README.md
+issues:
+  - |
+    Add support for host key algorithms to work with local ssh proxy
+    without which packer builds results in "failed to handshake" error.
+    Workaround is to pass additional params with "extra_arguments".
+
+    Reference:
+    https://github.com/hashicorp/packer-plugin-ansible/issues/140
+deprecations:
+  - |
+    Support for '.json' templates will be removed from common-packer in
+    subsequent release to give enough time for projects consuming to upgrade.
+    All projects specific templates not available in this repository are
+    required to convert existing '.json' to '.pkr.hcl' format.

templates/builder-aws.pkr.hcl

@@ -0,0 +1,191 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 1.2.6"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+variable "ansible_roles_path" {
+  type    = string
+  default = ".galaxy"
+}
+
+variable "arch" {
+  type    = string
+  default = "x86_64"
+}
+
+variable "aws_access_key" {
+  type = string
+  default = null
+}
+
+variable "aws_instance_type" {
+  type    = string
+  default = "t2.micro"
+}
+
+variable "aws_region" {
+  type    = string
+  default = "us-west-2"
+}
+
+variable "aws_secret_key" {
+  type = string
+  default = null
+}
+
+variable "base_image" {
+  type    = string
+  default = null
+}
+
+variable "cloud_auth_url" {
+  type    = string
+  default = null
+}
+
+variable "cloud_user_data" {
+  type = string
+  default = null
+}
+
+variable "cloud_network" {
+  type    = string
+  default = null
+}
+
+variable "cloud_tenant" {
+  type    = string
+  default = null
+}
+
+variable "cloud_pass" {
+  type    = string
+  default = null
+}
+
+variable "cloud_user" {
+  type    = string
+  default = null
+}
+
+variable "distro" {
+  type = string
+  default = null
+}
+
+variable "docker_source_image" {
+  type    = string
+  default = null
+}
+
+variable "flavor" {
+  type    = string
+  default = null
+}
+
+variable "security_group_id" {
+  type = string
+  default = null
+}
+
+variable "ssh_proxy_host" {
+  type    = string
+  default = ""
+}
+
+variable "ssh_user" {
+  type = string
+  default = null
+}
+
+variable "source_ami_filter_name" {
+  type    = string
+  default = null
+}
+
+variable "source_ami_filter_product_code" {
+  type    = string
+  default = null
+}
+
+variable "source_ami_filter_owner" {
+  type    = string
+  default = null
+}
+
+variable "subnet_id" {
+  type = string
+  default = null
+}
+
+variable "vpc_id" {
+  type = string
+  default = null
+}
+
+data "amazon-ami" "builder-aws" {
+  access_key = "${var.aws_access_key}"
+  filters = {
+    name                = "${var.source_ami_filter_name}"
+    product-code        = "${var.source_ami_filter_product_code}"
+    root-device-type    = "ebs"
+    virtualization-type = "hvm"
+  }
+  most_recent = true
+  owners      = ["${var.source_ami_filter_owner}"]
+  region      = "${var.aws_region}"
+  secret_key  = "${var.aws_secret_key}"
+}
+
+source "amazon-ebs" "aws" {
+  access_key        = "${var.aws_access_key}"
+  ami_name          = "ZZCI - ${var.distro} - builder-aws - ${var.arch} - ${legacy_isotime("20060102-150405.000")}"
+  instance_type     = "${var.aws_instance_type}"
+  region            = "${var.aws_region}"
+  secret_key        = "${var.aws_secret_key}"
+  security_group_id = "${var.security_group_id}"
+  source_ami        = "${data.amazon-ami.builder-aws.id}"
+  ssh_proxy_host    = "${var.ssh_proxy_host}"
+  ssh_username      = "${var.ssh_user}"
+  subnet_id         = "${var.subnet_id}"
+  user_data_file    = "${var.cloud_user_data}"
+  vpc_id            = "${var.vpc_id}"
+}
+
+build {
+  description = "Build an AMI for use as a CI builder"
+
+  sources = ["source.amazon-ebs.aws"]
+
+  provisioner "shell" {
+    execute_command = "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
+    scripts         = ["common-packer/provision/install-python.sh"]
+  }
+
+  provisioner "shell-local" {
+    command = "./common-packer/ansible-galaxy.sh ${var.ansible_roles_path}"
+  }
+
+  provisioner "ansible" {
+    ansible_env_vars   = [
+        "ANSIBLE_NOCOWS=1",
+        "ANSIBLE_PIPELINING=False",
+        "ANSIBLE_HOST_KEY_CHECKING=False",
+        "ANSIBLE_ROLES_PATH=${var.ansible_roles_path}",
+        "ANSIBLE_CALLBACK_WHITELIST=profile_tasks",
+        "ANSIBLE_STDOUT_CALLBACK=debug"
+    ]
+    command            = "./common-packer/ansible-playbook.sh"
+    extra_arguments    = [
+        "--scp-extra-args", "'-O'",
+        "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"
+    ]
+    playbook_file      = "provision/local-builder.yaml"
+    skip_version_check = true
+    user               = "${var.ssh_user}"
+  }
+}