From d643900f8badb77d2268d354525289f49b837400 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Sat, 2 Sep 2023 22:35:42 +0800 Subject: [PATCH] Add support for ElasticSearch/OpenSearch as a storage (#30) - Refactor database variables to support multiple storages more easily. --- .../templates/skywalking-oap.env.j2 | 5 +- .../group_vars/skywalking_oap.yaml.tftpl | 25 +++++ ansible/template/inventory.yaml.tftpl | 8 -- aws/alb-main.tf | 11 +++ aws/configurations.md | 15 +-- aws/ec2-main.tf | 14 +-- aws/elasticsearch-main.tf | 99 +++++++++++++++++++ aws/elasticsearch-output.tf | 21 ++++ aws/h2-main.tf | 31 ++++++ aws/modules/skywalking/README.md | 2 +- aws/modules/skywalking/variables.tf | 6 +- aws/rds-postgresql-main.tf | 34 +++++-- aws/skywalking-main.tf | 10 +- aws/variables.tf | 70 ++++++------- 14 files changed, 272 insertions(+), 79 deletions(-) create mode 100644 ansible/template/group_vars/skywalking_oap.yaml.tftpl create mode 100644 aws/elasticsearch-main.tf create mode 100644 aws/elasticsearch-output.tf create mode 100644 aws/h2-main.tf diff --git a/ansible/roles/skywalking/templates/skywalking-oap.env.j2 b/ansible/roles/skywalking/templates/skywalking-oap.env.j2 index 981d0d4..77c9ead 100644 --- a/ansible/roles/skywalking/templates/skywalking-oap.env.j2 +++ b/ansible/roles/skywalking/templates/skywalking-oap.env.j2 @@ -21,13 +21,16 @@ {% set storage = database['type'] %} {% if storage and (storage | length) %} -SW_STORAGE={{ storage | regex_replace('^rds-', '')}} +SW_STORAGE={{ storage | regex_replace('^rds_', '')}} {% endif %} {% if "postgresql" in storage %} SW_JDBC_URL=jdbc:postgresql://{{ database["host"] }}:{{ database["port"] }}/{{ database["name"] }} SW_DATA_SOURCE_USER={{ database['user'] }} SW_DATA_SOURCE_PASSWORD={{ database['password'] }} +{% elif "elasticsearch" in storage %} +SW_STORAGE_ES_CLUSTER_NODES={{ database["host"] }} +SW_STORAGE_ES_HTTP_PROTOCOL=https {% endif %} {% for key, value in skywalking_oap_environment.items() %} diff --git a/ansible/template/group_vars/skywalking_oap.yaml.tftpl b/ansible/template/group_vars/skywalking_oap.yaml.tftpl new file mode 100644 index 0000000..63c15bf --- /dev/null +++ b/ansible/template/group_vars/skywalking_oap.yaml.tftpl @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +database: + type: ${database_type} + host: ${database_host} + port: ${database_port} + name: ${database_name} + user: ${database_user} + password: ${database_password} + diff --git a/ansible/template/inventory.yaml.tftpl b/ansible/template/inventory.yaml.tftpl index a979a64..b8bbb40 100644 --- a/ansible/template/inventory.yaml.tftpl +++ b/ansible/template/inventory.yaml.tftpl @@ -34,14 +34,6 @@ skywalking_oap: %{ for oap in oap_instances ~} ${oap.private_ip}: %{ endfor ~} - vars: - database: - type: ${database_type} - host: ${database_host} - port: ${database_port} - name: ${database_name} - user: ${database_user} - password: ${database_password} skywalking_ui: hosts: diff --git a/aws/alb-main.tf b/aws/alb-main.tf index 83d1dcf..1e85c16 100644 --- a/aws/alb-main.tf +++ b/aws/alb-main.tf @@ -57,6 +57,17 @@ module "alb" { port = 8080 } ] + health_check = { + enabled = true + interval = 30 + path = "/internal/l7check" + port = "traffic-port" + healthy_threshold = 3 + unhealthy_threshold = 3 + timeout = 6 + protocol = "HTTP" + matcher = "200" + } } ] diff --git a/aws/configurations.md b/aws/configurations.md index 431266f..8907397 100644 --- a/aws/configurations.md +++ b/aws/configurations.md @@ -24,12 +24,18 @@ No requirements. | Name | Type | |------|------| +| [aws_elasticsearch_domain.elasticsearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/elasticsearch_domain) | resource | | [aws_security_group.alb-skywalking-ui](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.allow_apps](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.elasticsearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.public-egress-access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [local_file.elasticsearch_vars](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.h2_vars](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [local_file.inventories](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.rds_postgresql_vars](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [random_password.rds_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | ## Inputs @@ -43,12 +49,6 @@ No requirements. | [cluster\_name](#input\_cluster\_name) | Name of the cluster | `string` | `"skywalking-cluster"` | no | | [create\_lb](#input\_create\_lb) | Create load balancer for SkyWalking UI | `bool` | `true` | no | | [database\_subnets](#input\_database\_subnets) | CIDR used for database subnets | `set(string)` | 
[
  "11.0.104.0/24",
  "11.0.105.0/24",
  "11.0.106.0/24"
]
| no | -| [db\_instance\_class](#input\_db\_instance\_class) | Instance class for the database | `string` | `"db.t3.medium"` | no | -| [db\_max\_storage\_size](#input\_db\_max\_storage\_size) | Maximum storage size for the database, in GB | `number` | `100` | no | -| [db\_name](#input\_db\_name) | Name of the database | `string` | `"skywalking"` | no | -| [db\_password](#input\_db\_password) | Password for the database, if not set, a random password will be generated. | `string` | `null` | no | -| [db\_storage\_size](#input\_db\_storage\_size) | Storage size for the database, in GB | `number` | `5` | no | -| [db\_username](#input\_db\_username) | Username for the database | `string` | `"skywalking"` | no | | [extra\_tags](#input\_extra\_tags) | Additional tags to be added to all resources | `map(string)` | `{}` | no | | [oap\_instance\_count](#input\_oap\_instance\_count) | Number of OAP instances, if you want to use H2 storage, you must set it to 1. | `number` | `1` | no | | [oap\_instance\_type](#input\_oap\_instance\_type) | CPU, memory, storage and networking capacity for OAP instances | `string` | `"c5.xlarge"` | no | @@ -57,7 +57,7 @@ No requirements. | [public\_subnets](#input\_public\_subnets) | CIDR used for public subnets | `set(string)` | 
[
  "11.0.101.0/24",
  "11.0.102.0/24",
  "11.0.103.0/24"
]
| no | | [region](#input\_region) | Physical location for clustered data centers. | `string` | `"us-east-1"` | no | | [secret\_key](#input\_secret\_key) | Secret key of the AWS account, if you have configured AWS CLI, you can leave it empty. | `string` | `""` | no | -| [storage](#input\_storage) | Storage type for SkyWalking OAP, can be 'h2', or 'rds-postgresql' | `string` | `"rds-postgresql"` | no | +| [storage](#input\_storage) | Storage configuration for SkyWalking OAP | 
object({
    h2 = optional(object({}))
    rds_postgresql = optional(object({
      db_storage_size_gb     = optional(number)
      db_max_storage_size_gb = optional(number)
      db_instance_class      = optional(string)
      db_name                = optional(string)
      db_username            = optional(string)
      db_password            = optional(string)
    }))
    elasticsearch = optional(object({
      domain_name                = optional(string)
      version                    = optional(string)
      instance_type              = optional(string)
      instance_count             = optional(number)
      additional_security_groups = optional(list(string))
      zone_awareness_enabled     = optional(bool)
      availability_zone_count    = optional(number)
      ebs_enabled                = optional(bool)
    }))
  })
| 
{
  "h2": {}
}
| no | | [ui\_instance\_count](#input\_ui\_instance\_count) | Number of UI instances | `number` | `1` | no | | [ui\_instance\_type](#input\_ui\_instance\_type) | CPU, memory, storage and networking capacity for UI instances | `string` | `"t2.medium"` | no | @@ -72,6 +72,7 @@ No requirements. | [database\_password](#output\_database\_password) | The database password | | [database\_port](#output\_database\_port) | The database port | | [database\_username](#output\_database\_username) | The database username | +| [elasticsearch\_endpoint](#output\_elasticsearch\_endpoint) | The elasticsearch endpoint | | [oap\_ips](#output\_oap\_ips) | The private IPs of the OAP instances | | [ssh\_user\_key\_file](#output\_ssh\_user\_key\_file) | The SSH private key file to use to connect to the bastion host | | [ui\_ips](#output\_ui\_ips) | The IPs of the SkyWalking UI instances | diff --git a/aws/ec2-main.tf b/aws/ec2-main.tf index 50d27c6..288f86b 100644 --- a/aws/ec2-main.tf +++ b/aws/ec2-main.tf @@ -34,15 +34,9 @@ resource "local_file" "inventories" { filename = "${path.module}/../ansible/inventory/skywalking.yaml" file_permission = "0600" content = templatefile("${path.module}/../ansible/template/inventory.yaml.tftpl", { - bastion = module.skywalking.bastion_instances[0] - oap_instances = module.skywalking.oap_instances - ui_instances = module.skywalking.ui_instances - private_key_file = module.skywalking.ssh_user_key_file - database_type = var.storage - database_host = var.storage == "rds-postgresql" ? module.rds[0].db_instance_address : "" - database_port = var.storage == "rds-postgresql" ? module.rds[0].db_instance_port : "" - database_user = var.storage == "rds-postgresql" ? module.rds[0].db_instance_username : "" - database_name = var.storage == "rds-postgresql" ? module.rds[0].db_instance_name : "" - database_password = var.storage == "rds-postgresql" ? local.database_password : "" + bastion = module.skywalking.bastion_instances[0] + oap_instances = module.skywalking.oap_instances + ui_instances = module.skywalking.ui_instances + private_key_file = module.skywalking.ssh_user_key_file }) } diff --git a/aws/elasticsearch-main.tf b/aws/elasticsearch-main.tf new file mode 100644 index 0000000..e781916 --- /dev/null +++ b/aws/elasticsearch-main.tf @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +locals { + elasticsearch_domain_name = coalesce(lookup(local.storage_config, "domain_name"), var.cluster_name) + elasticsearch_version = coalesce(lookup(local.storage_config, "version"), "7.10") + elasticsearch_instance_type = coalesce(lookup(local.storage_config, "instance_type"), "m3.medium.elasticsearch") + elasticsearch_instance_count = coalesce(lookup(local.storage_config, "instance_count"), 2) + elasticsearch_additional_security_groups = coalesce(lookup(local.storage_config, "additional_security_groups"), []) + elasticsearch_zone_awareness_enabled = coalesce(lookup(local.storage_config, "zone_awareness_enabled"), false) + elasticsearch_availability_zone_count = coalesce(lookup(local.storage_config, "availability_zone_count"), 2) + elasticsearch_ebs_enabled = coalesce(lookup(local.storage_config, "ebs_enabled"), false) +} + +data "aws_caller_identity" "current" {} + +resource "aws_elasticsearch_domain" "elasticsearch" { + count = local.storage_name == "elasticsearch" ? 1 : 0 + + domain_name = local.elasticsearch_domain_name + elasticsearch_version = local.elasticsearch_version + + cluster_config { + instance_type = local.elasticsearch_instance_type + instance_count = local.elasticsearch_instance_count + zone_awareness_enabled = local.elasticsearch_zone_awareness_enabled + zone_awareness_config { + availability_zone_count = local.elasticsearch_availability_zone_count + } + } + + vpc_options { + subnet_ids = slice(module.vpc.private_subnets, 0, local.elasticsearch_zone_awareness_enabled ? 2 : 1) + + security_group_ids = [aws_security_group.elasticsearch.id] + } + + ebs_options { + ebs_enabled = local.elasticsearch_ebs_enabled + } + + access_policies = < [oap\_instance\_subnet\_id](#input\_oap\_instance\_subnet\_id) | Subnet ID for OAP instances | `string` | n/a | yes | | [oap\_instance\_type](#input\_oap\_instance\_type) | CPU, memory, storage and networking capacity for OAP instances | `string` | `"c5.xlarge"` | no | | [public\_key\_path](#input\_public\_key\_path) | Path to store the key file for SSH access to the instances. | `string` | `"~/.ssh"` | no | -| [storage](#input\_storage) | Storage type for SkyWalking OAP, can be `h2`, or `rds-postgresql` | `string` | `"rds-postgresql"` | no | +| [storage](#input\_storage) | Storage type for SkyWalking OAP, can be `h2`, `elasticsearch` or `rds-postgresql` | `string` | `"rds-postgresql"` | no | | [ui\_instance\_ami\_id](#input\_ui\_instance\_ami\_id) | AMI ID for UI instances, if not set, a suitable AMI ID will be selected automatically. | `string` | `""` | no | | [ui\_instance\_count](#input\_ui\_instance\_count) | Number of UI instances | `number` | `1` | no | | [ui\_instance\_security\_group\_ids](#input\_ui\_instance\_security\_group\_ids) | Additional security groups for UI instances | `list(string)` | `[]` | no | diff --git a/aws/modules/skywalking/variables.tf b/aws/modules/skywalking/variables.tf index 6e8818e..ef2769f 100644 --- a/aws/modules/skywalking/variables.tf +++ b/aws/modules/skywalking/variables.tf @@ -120,12 +120,12 @@ variable "extra_tags" { ## Storage variable "storage" { type = string - description = "Storage type for SkyWalking OAP, can be `h2`, or `rds-postgresql`" + description = "Storage type for SkyWalking OAP, can be `h2`, `elasticsearch` or `rds-postgresql`" default = "rds-postgresql" validation { - condition = contains(["h2", "rds-postgresql"], var.storage) - error_message = "Allowed values for storage are \"h2\", \"rds-postgresql\"." + condition = contains(["h2", "elasticsearch", "rds_postgresql"], var.storage) + error_message = "Allowed values for storage are \"h2\", \"rds_postgresql\"." } } diff --git a/aws/rds-postgresql-main.tf b/aws/rds-postgresql-main.tf index fd685c9..a4bfe46 100644 --- a/aws/rds-postgresql-main.tf +++ b/aws/rds-postgresql-main.tf @@ -21,22 +21,27 @@ resource "random_password" "rds_password" { } locals { - database_password = var.db_password != null ? var.db_password : random_password.rds_password.result + database_instance_class = coalesce(lookup(local.storage_config, "db_instance_class", "db.t3.small")) + database_storage_size = coalesce(lookup(local.storage_config, "db_storage_size_gb", 20)) + database_max_storage_size = coalesce(lookup(local.storage_config, "db_max_storage_size_gb", 100)) + database_name = coalesce(lookup(local.storage_config, "db_name", "skywalking")) + database_username = coalesce(lookup(local.storage_config, "db_username", "skywalking")) + database_password = coalesce(lookup(local.storage_config, "db_password", random_password.rds_password.result)) } module "rds" { source = "terraform-aws-modules/rds/aws" version = "~> 5.0" - count = var.storage == "rds-postgresql" ? 1 : 0 + count = local.storage_name == "rds_postgresql" ? 1 : 0 identifier = var.cluster_name - allocated_storage = var.db_storage_size - max_allocated_storage = var.db_max_storage_size + allocated_storage = local.database_storage_size + max_allocated_storage = local.database_max_storage_size - db_name = var.db_name - username = var.db_username + db_name = local.database_name + username = local.database_username password = local.database_password create_random_password = false port = "5432" @@ -62,7 +67,7 @@ module "rds" { engine_version = "15" family = "postgres15" major_engine_version = "15.3" - instance_class = var.db_instance_class + instance_class = local.database_instance_class create_db_option_group = "false" parameters = [ @@ -92,3 +97,18 @@ resource "aws_security_group" "allow_apps" { cidr_blocks = ["0.0.0.0/0"] } } + +resource "local_file" "rds_postgresql_vars" { + count = local.storage_name == "rds_postgresql" ? 1 : 0 + + filename = "${path.module}/../ansible/inventory/group_vars/skywalking_oap.yaml" + file_permission = "0600" + content = templatefile("${path.module}/../ansible/template/group_vars/skywalking_oap.yaml.tftpl", { + database_type = local.storage_name + database_host = local.storage_name == "rds_postgresql" ? module.rds[0].db_instance_address : "" + database_port = local.storage_name == "rds_postgresql" ? module.rds[0].db_instance_port : "" + database_user = local.storage_name == "rds_postgresql" ? module.rds[0].db_instance_username : "" + database_name = local.storage_name == "rds_postgresql" ? module.rds[0].db_instance_name : "" + database_password = local.storage_name == "rds_postgresql" ? local.database_password : "" + }) +} diff --git a/aws/skywalking-main.tf b/aws/skywalking-main.tf index 6ca40d1..64179d3 100644 --- a/aws/skywalking-main.tf +++ b/aws/skywalking-main.tf @@ -15,11 +15,19 @@ # specific language governing permissions and limitations # under the License. +locals { + storage = { + for storage, config in var.storage : storage => config if config != null + } + storage_name = keys(local.storage)[0] + storage_config = values(local.storage)[0] +} + module "skywalking" { source = "./modules/skywalking" cluster_name = var.cluster_name - storage = var.storage + storage = local.storage_name oap_instance_count = var.oap_instance_count oap_instance_type = var.oap_instance_type diff --git a/aws/variables.tf b/aws/variables.tf index f78b241..a0ae681 100644 --- a/aws/variables.tf +++ b/aws/variables.tf @@ -118,52 +118,40 @@ variable "database_subnets" { ## Storage variable "storage" { - type = string - description = "Storage type for SkyWalking OAP, can be 'h2', or 'rds-postgresql'" - default = "rds-postgresql" + description = "Storage configuration for SkyWalking OAP" + + type = object({ + h2 = optional(object({})) + rds_postgresql = optional(object({ + db_storage_size_gb = optional(number) + db_max_storage_size_gb = optional(number) + db_instance_class = optional(string) + db_name = optional(string) + db_username = optional(string) + db_password = optional(string) + })) + elasticsearch = optional(object({ + domain_name = optional(string) + version = optional(string) + instance_type = optional(string) + instance_count = optional(number) + additional_security_groups = optional(list(string)) + zone_awareness_enabled = optional(bool) + availability_zone_count = optional(number) + ebs_enabled = optional(bool) + })) + }) + + default = { + h2 = {} + } validation { - condition = contains(["h2", "rds-postgresql"], var.storage) - error_message = "Allowed values for storage are \"h2\", \"rds-postgresql\"." + condition = length([for storage, value in var.storage : storage if value != null]) == 1 + error_message = "Please only set one storage type." } } -variable "db_name" { - type = string - description = "Name of the database" - default = "skywalking" -} - -variable "db_username" { - type = string - description = "Username for the database" - default = "skywalking" -} - -variable "db_password" { - type = string - description = "Password for the database, if not set, a random password will be generated." - default = null -} - -variable "db_storage_size" { - type = number - description = "Storage size for the database, in GB" - default = 5 -} - -variable "db_max_storage_size" { - type = number - description = "Maximum storage size for the database, in GB" - default = 100 -} - -variable "db_instance_class" { - type = string - description = "Instance class for the database" - default = "db.t3.medium" -} - variable "create_lb" { type = bool description = "Create load balancer for SkyWalking UI"