gitlab agent and docker-machine is not present , EC2 instance is up and running

[dimashenkov]

Discussed in #1157

^{Originally posted by dimashenkov July 26, 2024}
Hi guys, I have this module and Im able to create all resources but on ec2 main agent runner , gitlab agent is missing docker-machine too. What could be wrong? I would appreciate some help!

module "runner" {
  source  = "cattle-ops/gitlab-runner/aws"
  version = "7.9.0"

  environment = var.environment

  vpc_id    = data.terraform_remote_state.vpc.outputs.vpc_id
  subnet_id = element(data.terraform_remote_state.vpc.outputs.vpc_private_subnet_ids, 0)

  runner_gitlab = {
    url                = var.gitlab_url
    runner_version     = var.gitlab_runner_version
    preregistered_runner_token_ssm_parameter_name = "${module.context.prefix}-gitlab-registration-token"
  }

  runner_instance = {
    max_lifetime_seconds        = var.runner_max_instance_lifetime_seconds
    use_eip                     = var.runner_enable_eip
    collect_autoscaling_metrics = var.runner_collect_autoscaling_metrics
    monitoring                  = var.runner_instance_enable_monitoring
    name                        = "${module.context.prefix}-runner"
    ssm_access                  = var.runner_enable_ssm_access
    private_address_only        = var.runners_use_private_address
    root_device_config          = var.runner_root_block_device
    ebs_optimized               = var.runner_ebs_optimized
    spot_price                  = var.runner_instance_spot_price
    name_prefix                 = "${module.context.prefix}-runner-main"
    type                        = var.runner_instance_type
    additional_tags             = module.context.tags
  }

  runner_worker_docker_machine_instance = {
    private_address_only       = var.runners_use_private_address
    monitoring                 = var.runner_worker_docker_machine_enable_monitoring
    start_script               = ""
    volume_type                = var.runner_worker_docker_machine_ec2_volume_type
    root_size                  = var.runners_root_size
    ebs_optimized              = var.runner_worker_docker_machine_ec2_ebs_optimized
    destroy_after_max_builds   = var.runner_worker_docker_machine_max_builds
    docker_registry_mirror_url = var.runner_worker_docker_machine_docker_registry_mirror_url
    types                      = [var.docker_machine_instance_type]
    idle_time                  = var.runner_worker_idle_time
    idle_count                 = var.runner_worker_idle_count
    name_prefix                = "${var.department}-${var.application}-worker"
  }

  runner_worker_docker_machine_instance_spot = {
    enable    = var.runners_request_spot_instance
    max_price = var.docker_machine_spot_price_bid
  }

  runner_worker_docker_machine_autoscaling_options = var.runner_worker_docker_machine_autoscaling_options

  runner_gitlab_registration_config = {
    registration_token = data.aws_ssm_parameter.gitlab_registration_token.value
    tag_list           = var.gitlab_runner_config["tag_list"]
    description        = var.gitlab_runner_config["description"]
    locked_to_project  = var.gitlab_runner_config["locked_to_project"]
    run_untagged       = var.gitlab_runner_config["run_untagged"]
    maximum_timeout    = var.gitlab_runner_config["maximum_timeout"]
  }

  runner_worker_docker_options = {
    image          = var.runners_image
    privileged     = var.runners_privileged
    disable_cache  = var.runners_disable_cache
    extra_hosts    = var.runners_extra_hosts
    shm_size       = var.runners_shm_size
    runtime        = var.runners_docker_runtime
    helper_image   = var.runners_helper_image
    pull_policies  = var.runners_pull_policies
    volumes        = concat(["/cache"], var.runners_additional_volumes)
  }

  runner_install = {
    post_install_script = templatefile("${path.module}/user_data.sh", {})
    start_script        = var.runner_user_data_extra
  }

  runner_worker_gitlab_pipeline = {
    pre_clone_script  = var.runner_worker_pre_clone_script
    pre_build_script  = var.runner_worker_pre_build_script
    post_build_script = var.runner_worker_post_build_script
  }

  runner_worker = {
    max_jobs            = var.runner_worker_max_jobs
    request_concurrency = var.runner_worker_request_concurrency
    output_limit        = var.runner_worker_output_limit
    type                = var.runner_worker_type
    environment_vars = [
    "DOCKER_AUTH_CONFIG=${local.docker_auth_config}",
    "CI_GIT_USERNAME=${var.git_username}",
    "CI_GIT_PASSWORD=${data.aws_ssm_parameter.gitlab_git_token.value}"
  ]    
  }

  runner_cloudwatch = {
    enable         = var.enable_cloudwatch_logging
    retention_days = var.runner_cloudwatch_retention_days
    log_group_name = var.runner_cloudwatch_log_group_name
  }

  runner_networking = {
    allow_incoming_ping                    = var.runner_ping_enable
    security_group_ids                     = var.runner_extra_security_group_ids
    allow_incoming_ping_security_group_ids = var.runner_ping_allow_from_security_groups
  }

  runner_worker_docker_machine_role = {
    profile_name            = var.runner_worker_docker_machine_iam_instance_profile_name
    assume_role_policy_json = var.runner_worker_docker_machine_assume_role_json
  }

  runner_ami_filter = var.gitlab_runner_docker_machine_ami_name
  runner_ami_owners = var.runner_ami_owners

  debug = var.debug_options
}

variable "aws_region" {
  description = "The AWS region to deploy to (e.g. us-east-1)."
  type        = string
}

variable "company_prefix" {
  type = string
}

# variable "department" {
#   type = string
# }

# variable "environment" {
#   type = string
# }

# variable "application" {
#   type = string
# }

variable "vpc_name" {
  description = "The VPC name, needed when accessing remote state"
  type        = string
}

variable "terraform_state_aws_region" {
  description = "The AWS region of the S3 bucket used to store Terraform remote state"
  type        = string
}

variable "terraform_state_s3_bucket" {
  description = "The name of the S3 bucket used to store Terraform remote state"
  type        = string
}

variable "terraform_state_role_arn" {
  description = "The name of the S3 bucket used to store Terraform remote state"
  type        = string
}

# ---------------------------------------------------------------------------------------------------------------------
# Gitlab Runner
# ---------------------------------------------------------------------------------------------------------------------

variable "gitlab_runner_config" {
  type = map(string)
  default = {
    gitlab_runner_tag = "production"
    tag_list          = "production"
    description       = "prod core team runner"
    locked_to_project = "false"
    run_untagged      = "false"
    maximum_timeout   = "10800"
  }
}

variable "gitlab_runner_version" {
  description = "Version of the [GitLab runner](https://gitlab.com/gitlab-org/gitlab-runner/-/releases)."
  type        = string
  default     = "14.10.1"
}

variable "gitlab_url" {
  type    = string
  default = ""
}

variable "runners_use_private_address" {
  type    = bool
  default = false
}

variable "runners_privileged" {
  type    = bool
  default = true
}

variable "runners_additional_volumes" {
  type    = list(string)
  default = []
}

variable "runners_root_size" {
  type    = number
  default = 50
}

variable "allow_cert_validation" {
  type    = bool
  default = false
}

variable "docker_machine_instance_type" {
  description = "Override the default runner type (m4.large). More details about available spot instances can be obtained from aws spot advisor link: https://aws.amazon.com/ec2/spot/instance-advisor"
  type        = string
  default     = "m4.large"
}

variable "docker_machine_spot_price_bid" {
  description = "Spot price bid"
  type        = string
  default     = "on-demand-price"
}

variable "runner_instance_spot_price" {
  type        = string
  description = "By setting a spot price bid price the runner agent will be created via a spot request. Be aware that spot instances can be stopped by AWS. Choose 'on-demand-price' to pay up to the current on demand price for the instance type chosen."
  default     = "on-demand-price"
}

variable "runners_request_spot_instance" {
  type        = bool
  description = "Whether or not to request spot instances via docker-machine."
  default     = true
}

variable "runners_install_amazon_ecr_credential_helper" {
  type    = bool
  default = true
}

variable "enable_cloudwatch_logging" {
  type    = bool
  default = false
}

variable "enable_runner_ssm_access" {
  type    = bool
  default = true
}

variable "enable_docker_machine_ssm_access" {
  type    = bool
  default = true
}

# variable "gitlab_runner_docker_machine_ami_name" {
#   description = "List of maps used to create the AMI filter for the Runner AMI. Must resolve to an Amazon Linux 1, 2 or 2023 image."
#   type        = map(list(string))
#   default     = {
#     name = ["gitlab-runner-docker-machine-ami"]
#   }
# }

# variable "runner_ami_owners" {
#   description = "List of owner IDs used to filter the AMIs."
#   type        = list(string)
#   default     = ["*******"]
# }

variable "gitlab_runner_docker_machine_ami_name" {
  description = "List of maps used to create the AMI filter for the Runner AMI. Must resolve to an Amazon Linux 1, 2 or 2023 image."
  type        = map(list(string))
  default     = {
    name = ["amzn2-ami-hvm-*-x86_64-gp2"]  # Default to Amazon Linux 2 AMI
  }
}

variable "runner_ami_owners" {
  description = "List of owner IDs used to filter the AMIs."
  type        = list(string)
  default     = ["137112412989"]  # Amazon's official AWS account ID for Amazon Linux AMIs
}

variable "gl_runner_access_from_other_accounts" {
  type    = list(string)
  default = []
}

variable "docker_runner_attach_admin_policy" {
  description = "If true, attach an Admin policy to runner"
  type        = bool
  default     = false
}

variable "runners_image" {
  type    = string
  default = ""
}

variable "runner_worker_docker_machine_autoscaling_options" {
  description = "Set autoscaling parameters based on periods, see https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnersmachine-section"
  type = list(object({
    periods    = list(string)
    idle_count = number
    idle_time  = number
    timezone   = string
  }))
  default = []
}

variable "git_username" {
  default     = "gitlab-runner"
  description = "Git username, for runner environment variable. Token is in SSM"
  type        = string
}

variable "additional_docker_machine_policy_arns" {
  type        = list(string)
  description = "List of policy arns to allow pipelines to access AWS resources"
  default     = []
}

# Newly added variables
variable "runner_gitlab_ca_certificate" {
  type    = string
  default = ""
}

variable "runner_gitlab_certificate" {
  type    = string
  default = ""
}

variable "gitlab_clone_url" {
  type    = string
  default = ""
}

variable "runner_max_instance_lifetime_seconds" {
  type    = number
  default = 0
}

variable "runner_enable_eip" {
  type    = bool
  default = false
}

variable "runner_collect_autoscaling_metrics" {
  type    = list(string)
  default = []
}

variable "runner_instance_enable_monitoring" {
  type    = bool
  default = false
}

variable "runner_enable_ssm_access" {
  type    = bool
  default = true
}

variable "runner_root_block_device" {
  type    = any
  default = {}
}

variable "runner_ebs_optimized" {
  type    = bool
  default = false
}

variable "runner_instance_type" {
  type    = string
  default = "m4.large"
}

variable "runner_worker_docker_machine_enable_monitoring" {
  type    = bool
  default = false
}

variable "runner_worker_docker_machine_userdata" {
  type    = string
  default = ""
}

variable "runner_worker_docker_machine_ec2_volume_type" {
  type    = string
  default = "gp2"
}

variable "runner_worker_docker_machine_ec2_ebs_optimized" {
  type    = bool
  default = false
}

variable "runner_worker_docker_machine_max_builds" {
  type    = number
  default = 0
}

variable "runner_worker_docker_machine_docker_registry_mirror_url" {
  type    = string
  default = ""
}

variable "runner_worker_idle_time" {
  type    = number
  default = 300
}

variable "runner_worker_idle_count" {
  type    = number
  default = 1
}

variable "runner_worker_cache_shared" {
  type    = bool
  default = false
}

variable "runner_worker_cache_s3_authentication_type" {
  type    = string
  default = "access_key"
}

variable "runner_worker_cache_s3_expiration_days" {
  type    = number
  default = 0
}

variable "runner_worker_cache_s3_enable_versioning" {
  type    = bool
  default = false
}

variable "runner_worker_cache_s3_bucket_enable_random_suffix" {
  type    = bool
  default = false
}

variable "runner_worker_cache_s3_logging_bucket_id" {
  type    = string
  default = ""
}

variable "runner_worker_cache_s3_logging_bucket_prefix" {
  type    = string
  default = ""
}

variable "runner_cloudwatch_retention_days" {
  description = "Number of days to retain CloudWatch logs."
  type        = number
  default     = 14
}

variable "runner_cloudwatch_log_group_name" {
  type    = string
  default = ""
}

variable "runner_ping_enable" {
  type    = bool
  default = false
}

variable "runner_extra_security_group_ids" {
  type    = list(string)
  default = []
}

variable "runner_ping_allow_from_security_groups" {
  type    = list(string)
  default = []
}

variable "runner_worker_docker_machine_iam_instance_profile_name" {
  type    = string
  default = ""
}

variable "runner_worker_docker_machine_assume_role_json" {
  type    = string
  default = ""
}

variable "runner_worker_pre_clone_script" {
  type    = string
  default = ""
}

variable "runner_worker_pre_build_script" {
  type    = string
  default = ""
}

variable "runner_worker_post_build_script" {
  type    = string
  default = ""
}

variable "runner_worker_max_jobs" {
  type    = number
  default = 0
}

variable "runner_worker_request_concurrency" {
  type    = number
  default = 0
}

variable "runner_worker_output_limit" {
  type    = number
  default = 0
}

variable "runner_worker_type" {
  type    = string
  default = "docker+machine"
}

variable "runner_docker_machine_download_url" {
  type    = string
  default = ""
}

variable "runner_docker_machine_version" {
  type    = string
  default = ""
}

variable "runner_userdata_pre_install" {
  type    = string
  default = ""
}

variable "runner_user_data_extra" {
  type    = string
  default = ""
}

variable "runner_yum_update" {
  type    = bool
  default = false
}

variable "runner_user_data_enable_trace_log" {
  type    = bool
  default = false
}

variable "show_user_data_in_plan" {
  type    = bool
  default = false
}

variable "runners_disable_cache" {
  type    = bool
  default = false
}

variable "runners_extra_hosts" {
  type    = list(string)
  default = []
}

variable "runners_shm_size" {
  type    = number
  default = 64
}

variable "runners_docker_runtime" {
  type    = string
  default = "runc"
}

variable "runners_helper_image" {
  type    = string
  default = ""
}

variable "runners_pull_policies" {
  type    = list(string)
  default = ["if-not-present"]
}

variable "debug_options" {
  description = "Debug options for the GitLab Runner module"
  type = object({
    trace_runner_user_data         = optional(bool, false)
    write_runner_config_to_file    = optional(bool, false)
    write_runner_user_data_to_file = optional(bool, false)
  })
  default = {
    trace_runner_user_data         = false
    write_runner_config_to_file    = false
    write_runner_user_data_to_file = false
  }
}

[dimashenkov]

the problem was not supported AMI