From 5ca9e49a8ad7970dc0614ca22b02f1c4fe03ec62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Apr 2026 11:12:29 +0200 Subject: [PATCH 1/5] Add fastpath module to terraform --- tf/environments/dev/main.tf | 71 +-------- tf/environments/prod/main.tf | 174 ++++------------------ tf/modules/ooni_docker_build/variables.tf | 4 - tf/modules/ooni_fastpath/main.tf | 93 ++++++++++++ tf/modules/ooni_fastpath/outputs.tf | 27 ++++ tf/modules/ooni_fastpath/variables.tf | 77 ++++++++++ 6 files changed, 234 insertions(+), 212 deletions(-) create mode 100644 tf/modules/ooni_fastpath/main.tf create mode 100644 tf/modules/ooni_fastpath/outputs.tf create mode 100644 tf/modules/ooni_fastpath/variables.tf diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 68656ae7..fa775c42 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -554,7 +554,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "1183-fix-smoketest" + branch_name = "master" environment = local.environment trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" @@ -892,78 +892,27 @@ resource "aws_route53_record" "monitoring_proxy_alias" { ### Fastpath module "ooni_fastpath" { - source = "../../modules/ec2" + source = "../../modules/ooni_fastpath" - stage = local.environment + name = "fastpath" + env = local.environment vpc_id = module.network.vpc_id subnet_id = module.network.vpc_subnet_public[0].id private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + public_subnet_cidr = module.network.vpc_subnet_public[*].cidr_block dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "t3a.small" - name = "oonifastpath" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 8472, - to_port = 8472, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 8475, # for serving jsonl files - to_port = 8475, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 9100, - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }, { - from_port = 9102, # For fastpath metrics - to_port = 9102, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - sg_prefix = "oonifastpath" tg_prefix = "fstp" - disk_size = 150 + monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip + monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip - tags = merge( - local.tags, - { Name = "ooni-tier0-fastpath" } - ) -} - -resource "aws_route53_record" "fastpath_alias" { - zone_id = local.dns_zone_ooni_io - name = "fastpath.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_fastpath.aws_instance_public_dns - ] + tags = local.tags } module "fastpath_builder" { @@ -979,8 +928,6 @@ module "fastpath_builder" { codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket - - ecs_cluster_name = module.ooniapi_cluster.cluster_name } #### OONI Run service @@ -1313,8 +1260,6 @@ module "testlists_builder" { codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket - - ecs_cluster_name = module.ooniapi_cluster.cluster_name } #### OONI Tier0 API Frontend diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 1a41da5a..73f4adcf 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -923,180 +923,66 @@ module "ooniapi_ooniprobe" { ### Fastpath module "ooni_fastpath" { - source = "../../modules/ec2" + source = "../../modules/ooni_fastpath" - stage = local.environment + name = "fastpath" + env = local.environment vpc_id = module.network.vpc_id subnet_id = module.network.vpc_subnet_public[0].id private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + public_subnet_cidr = module.network.vpc_subnet_public[*].cidr_block dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "c6i.large" - name = "oonifastpath" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 8472, - to_port = 8472, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 8479, - to_port = 8479, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 8475, # for serving jsonl files - to_port = 8475, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 9100, - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }, { - from_port = 9102, # For fastpath metrics - to_port = 9102, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32", "${module.ooni_monitoring_proxy.aws_instance_public_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - sg_prefix = "oonifastpath" tg_prefix = "fstp" - disk_size = 150 + monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip + monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip - tags = merge( - local.tags, - { Name = "ooni-tier0-fastpath" } - ) + tags = local.tags +} + +module "fastpath_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "fastpath" + repo = "ooni/backend" + branch_name = "master" + environment = local.environment + buildspec_path = "fastpath/buildspec.yml" + trigger_path = "fastpath/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } module "ooni_fastpath2" { - source = "../../modules/ec2" + source = "../../modules/ooni_fastpath" - stage = local.environment + name = "fastpath2" + env = local.environment vpc_id = module.network.vpc_id subnet_id = module.network.vpc_subnet_public[0].id private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + public_subnet_cidr = module.network.vpc_subnet_public[*].cidr_block dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "c6i.large" - name = "oonifastpath2" - ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 8472, - to_port = 8472, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 8479, - to_port = 8479, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 8475, # for serving jsonl files - to_port = 8475, - protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), - }, { - from_port = 9100, - to_port = 9100, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] - }, { - from_port = 9102, # For fastpath metrics - to_port = 9102, - protocol = "tcp" - cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32", "${module.ooni_monitoring_proxy.aws_instance_public_ip}/32"] - }] - - egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", - cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", - ipv6_cidr_blocks = ["::/0"], - }] - sg_prefix = "oonifstp2" tg_prefix = "fp2" - disk_size = 150 + monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip + monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip - tags = merge( - local.tags, - { Name = "ooni-tier0-fastpath2" } - ) -} - -resource "aws_route53_record" "fastpath_alias" { - zone_id = local.dns_zone_ooni_io - name = "fastpath.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_fastpath.aws_instance_public_dns - ] -} - -resource "aws_route53_record" "fastpath2_alias" { - zone_id = local.dns_zone_ooni_io - name = "fastpath2.${local.environment}.ooni.io" - type = "CNAME" - ttl = 300 - - records = [ - module.ooni_fastpath2.aws_instance_public_dns - ] -} - -module "fastpath_builder" { - source = "../../modules/ooni_docker_build" - trigger_tag = "" - - service_name = "fastpath" - repo = "ooni/backend" - branch_name = "master" - environment = local.environment - buildspec_path = "fastpath/buildspec.yml" - trigger_path = "fastpath/**" - codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn - - codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket - - ecs_cluster_name = module.ooniapi_cluster.cluster_name + tags = local.tags } @@ -1441,8 +1327,6 @@ module "testlists_builder" { codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket - - ecs_cluster_name = module.ooniapi_cluster.cluster_name } #### OONI Tier0 API Frontend diff --git a/tf/modules/ooni_docker_build/variables.tf b/tf/modules/ooni_docker_build/variables.tf index 4383eb52..0185bea5 100644 --- a/tf/modules/ooni_docker_build/variables.tf +++ b/tf/modules/ooni_docker_build/variables.tf @@ -30,10 +30,6 @@ variable "repo" { default = "ooni/backend" } -variable "ecs_cluster_name" { - description = "id of the cluster to deploy into" -} - variable "trigger_path" { description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**" } diff --git a/tf/modules/ooni_fastpath/main.tf b/tf/modules/ooni_fastpath/main.tf new file mode 100644 index 00000000..379224bd --- /dev/null +++ b/tf/modules/ooni_fastpath/main.tf @@ -0,0 +1,93 @@ +locals { + vpc_cidrs = concat(var.private_subnet_cidr, var.public_subnet_cidr) + + ingress_rules = [ + { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + }, + { + from_port = 8472 + to_port = 8472 + protocol = "tcp" + cidr_blocks = local.vpc_cidrs + }, + { + from_port = 8475 # for serving jsonl files + to_port = 8475 + protocol = "tcp" + cidr_blocks = local.vpc_cidrs + }, + { + from_port = 9100 + to_port = 9100 + protocol = "tcp" + cidr_blocks = ["${var.monitoring_proxy_private_ip}/32"] + }, + { + from_port = 9102 # For fastpath metrics + to_port = 9102 + protocol = "tcp" + cidr_blocks = [ + "${var.monitoring_proxy_private_ip}/32", + "${var.monitoring_proxy_public_ip}/32", + ] + }, + ] + + egress_rules = [ + { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + }, + { + from_port = 0 + to_port = 0 + protocol = "-1" + ipv6_cidr_blocks = ["::/0"] + }, + ] +} + +module "ec2" { + source = "../ec2" + + stage = var.env + + vpc_id = var.vpc_id + subnet_id = var.subnet_id + private_subnet_cidr = var.private_subnet_cidr + dns_zone_ooni_io = var.dns_zone_ooni_io + + key_name = var.key_name + instance_type = var.instance_type + + name = "ooni${var.name}" + ingress_rules = local.ingress_rules + egress_rules = local.egress_rules + + sg_prefix = var.sg_prefix + tg_prefix = var.tg_prefix + + disk_size = var.disk_size + + tags = merge( + var.tags, + { Name = "ooni-tier0-${var.name}" }, + ) +} + +resource "aws_route53_record" "fastpath_alias" { + zone_id = var.dns_zone_ooni_io + name = "${var.name}.${var.env}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ec2.aws_instance_public_dns + ] +} diff --git a/tf/modules/ooni_fastpath/outputs.tf b/tf/modules/ooni_fastpath/outputs.tf new file mode 100644 index 00000000..90d25fea --- /dev/null +++ b/tf/modules/ooni_fastpath/outputs.tf @@ -0,0 +1,27 @@ +output "aws_instance_id" { + value = module.ec2.aws_instance_id +} + +output "aws_instance_private_ip" { + value = module.ec2.aws_instance_private_ip +} + +output "aws_instance_public_ip" { + value = module.ec2.aws_instance_public_ip +} + +output "aws_instance_public_dns" { + value = module.ec2.aws_instance_public_dns +} + +output "ec2_sg_id" { + value = module.ec2.ec2_sg_id +} + +output "alb_target_group_id" { + value = module.ec2.alb_target_group_id +} + +output "dns_name" { + value = aws_route53_record.fastpath_alias.name +} diff --git a/tf/modules/ooni_fastpath/variables.tf b/tf/modules/ooni_fastpath/variables.tf new file mode 100644 index 00000000..04d24974 --- /dev/null +++ b/tf/modules/ooni_fastpath/variables.tf @@ -0,0 +1,77 @@ +variable "name" { + description = "Short name used for the instance and DNS record (e.g. 'fastpath' or 'fastpath2')" + type = string +} + +variable "env" { + description = "Deployment environment, used for the ec2 module stage and the DNS record (e.g. dev, prod)" + type = string +} + +variable "vpc_id" { + description = "ID of the VPC" + type = string +} + +variable "subnet_id" { + description = "ID of the subnet to deploy the instance into" + type = string +} + +variable "private_subnet_cidr" { + description = "CIDR blocks of the private subnets" + type = list(string) +} + +variable "public_subnet_cidr" { + description = "CIDR blocks of the public subnets" + type = list(string) +} + +variable "dns_zone_ooni_io" { + description = "Route53 zone id for ooni.io" + type = string +} + +variable "key_name" { + description = "Name of the AWS key pair" + type = string +} + +variable "instance_type" { + description = "EC2 instance type" + type = string + default = "t3a.small" +} + +variable "disk_size" { + description = "Root disk size in GB" + type = number + default = 150 +} + +variable "sg_prefix" { + description = "Security group name prefix" + type = string +} + +variable "tg_prefix" { + description = "ALB target group name prefix (prefixed with 'oo')" + type = string +} + +variable "monitoring_proxy_private_ip" { + description = "Private IP of the monitoring proxy, allowed to scrape metrics on ports 9100 and 9102" + type = string +} + +variable "monitoring_proxy_public_ip" { + description = "Public IP of the monitoring proxy, allowed to scrape fastpath metrics on port 9102" + type = string +} + +variable "tags" { + description = "Extra tags to apply to the resources" + type = map(string) + default = {} +} From 06974c3c98fb7d6f15cdb1e23cfafd3c532815c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Apr 2026 11:45:39 +0200 Subject: [PATCH 2/5] Restore old fastpath 2 to avoid taking down that host from tf --- tf/environments/prod/main.tf | 71 ++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 73f4adcf..97fa4f82 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -961,28 +961,85 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } +# TODO: Delete this module after traffic is migrated back to the original fastpath module "ooni_fastpath2" { - source = "../../modules/ooni_fastpath" + source = "../../modules/ec2" - name = "fastpath2" - env = local.environment + stage = local.environment vpc_id = module.network.vpc_id subnet_id = module.network.vpc_subnet_public[0].id private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - public_subnet_cidr = module.network.vpc_subnet_public[*].cidr_block dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "c6i.large" + name = "oonifastpath2" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8472, + to_port = 8472, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 8479, + to_port = 8479, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 8475, # for serving jsonl files + to_port = 8475, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }, { + from_port = 9102, # For fastpath metrics + to_port = 9102, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32", "${module.ooni_monitoring_proxy.aws_instance_public_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] + sg_prefix = "oonifstp2" tg_prefix = "fp2" - monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip - monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip + disk_size = 150 - tags = local.tags + tags = merge( + local.tags, + { Name = "ooni-tier0-fastpath2" } + ) +} + +resource "aws_route53_record" "fastpath2_alias" { + zone_id = local.dns_zone_ooni_io + name = "fastpath2.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_fastpath2.aws_instance_public_dns + ] } From 31cf8a0efe45f0e847254e953d3a4e3cafde37df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Apr 2026 12:12:59 +0200 Subject: [PATCH 3/5] Add default value to name variables --- tf/modules/ooni_fastpath/variables.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tf/modules/ooni_fastpath/variables.tf b/tf/modules/ooni_fastpath/variables.tf index 04d24974..1889e701 100644 --- a/tf/modules/ooni_fastpath/variables.tf +++ b/tf/modules/ooni_fastpath/variables.tf @@ -1,6 +1,7 @@ variable "name" { - description = "Short name used for the instance and DNS record (e.g. 'fastpath' or 'fastpath2')" + description = "Short name used for the instance and DNS record" type = string + default = "fastpath" } variable "env" { From 6a49731495450c20ed6625fc6acc2849b468584f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 23 Apr 2026 12:50:15 +0200 Subject: [PATCH 4/5] Change the collector id per fastpath instance --- ansible/host_vars/fastpath.dev.ooni.io/vars.yml | 1 + ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 3 ++- ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index 984d3944..fbd3d273 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -1,5 +1,6 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}" clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" bucket_name: "ooni-data-eu-fra-test" +# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" env: "dev" \ No newline at end of file diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index dce929d0..e7210f1f 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,5 +1,6 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" -collector_id: "4" +# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE +collector_id: "1" env: "prod" \ No newline at end of file diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 984efcdf..43b97e17 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,5 +1,6 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni" bucket_name: "ooni-data-eu-fra" +# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" From 6ca812223d12d806c43ba4516d2e5b23c0dfd4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 24 Apr 2026 13:11:47 +0200 Subject: [PATCH 5/5] Remove unused ecs_cluster_name variable --- tf/environments/prod/main.tf | 101 +++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 23 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 97fa4f82..17b34f41 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -923,45 +923,74 @@ module "ooniapi_ooniprobe" { ### Fastpath module "ooni_fastpath" { - source = "../../modules/ooni_fastpath" + source = "../../modules/ec2" - name = "fastpath" - env = local.environment + stage = local.environment vpc_id = module.network.vpc_id subnet_id = module.network.vpc_subnet_public[0].id private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - public_subnet_cidr = module.network.vpc_subnet_public[*].cidr_block dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "c6i.large" - sg_prefix = "oonifastpath" - tg_prefix = "fstp" - - monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip - monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip + name = "oonifastpath" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 8472, + to_port = 8472, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 8479, + to_port = 8479, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 8475, # for serving jsonl files + to_port = 8475, + protocol = "tcp", + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block), + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }, { + from_port = 9102, # For fastpath metrics + to_port = 9102, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32", "${module.ooni_monitoring_proxy.aws_instance_public_ip}/32"] + }] - tags = local.tags -} + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"], + }] -module "fastpath_builder" { - source = "../../modules/ooni_docker_build" - trigger_tag = "" + sg_prefix = "oonifastpath" + tg_prefix = "fstp" - service_name = "fastpath" - repo = "ooni/backend" - branch_name = "master" - environment = local.environment - buildspec_path = "fastpath/buildspec.yml" - trigger_path = "fastpath/**" - codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + disk_size = 150 - codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + tags = merge( + local.tags, + { Name = "ooni-tier0-fastpath" } + ) } -# TODO: Delete this module after traffic is migrated back to the original fastpath module "ooni_fastpath2" { source = "../../modules/ec2" @@ -1031,6 +1060,17 @@ module "ooni_fastpath2" { ) } +resource "aws_route53_record" "fastpath_alias" { + zone_id = local.dns_zone_ooni_io + name = "fastpath.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_fastpath.aws_instance_public_dns + ] +} + resource "aws_route53_record" "fastpath2_alias" { zone_id = local.dns_zone_ooni_io name = "fastpath2.${local.environment}.ooni.io" @@ -1042,6 +1082,21 @@ resource "aws_route53_record" "fastpath2_alias" { ] } +module "fastpath_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "fastpath" + repo = "ooni/backend" + branch_name = "master" + environment = local.environment + buildspec_path = "fastpath/buildspec.yml" + trigger_path = "fastpath/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket +} + #### OONI Run service