new Fastpath TF module

ansible/host_vars/fastpath.dev.ooni.io/vars.yml

@@ -1,5 +1,6 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_dev') }}"
 clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"
 bucket_name: "ooni-data-eu-fra-test"
+# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
 collector_id: "3"
 env: "dev"

ansible/host_vars/fastpath.prod.ooni.io/vars.yml

@@ -1,5 +1,6 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}"
 clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"
 bucket_name: "ooni-data-eu-fra"
-collector_id: "4"
+# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
+collector_id: "1"
 env: "prod"

ansible/host_vars/fastpath2.prod.ooni.io/vars.yml

@@ -1,5 +1,6 @@
 s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}"
 clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.prod.ooni.io/ooni"
 bucket_name: "ooni-data-eu-fra"
+# COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE
 collector_id: "4"
 env: "prod"

tf/environments/dev/main.tf

@@ -554,7 +554,7 @@ module "ooniapi_ooniprobe_deployer" {
 
   service_name            = "ooniprobe"
   repo                    = "ooni/backend"
-  branch_name             = "1183-fix-smoketest"
+  branch_name             = "master"
   environment             = local.environment
   trigger_path            = "ooniapi/services/ooniprobe/**"
   buildspec_path          = "ooniapi/services/ooniprobe/buildspec.yml"
@@ -892,78 +892,27 @@ resource "aws_route53_record" "monitoring_proxy_alias" {
 
 ### Fastpath
 module "ooni_fastpath" {
-  source = "../../modules/ec2"
+  source = "../../modules/ooni_fastpath"
 
-  stage = local.environment
+  name = "fastpath"
+  env  = local.environment
 
   vpc_id              = module.network.vpc_id
   subnet_id           = module.network.vpc_subnet_public[0].id
   private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block
+  public_subnet_cidr  = module.network.vpc_subnet_public[*].cidr_block
   dns_zone_ooni_io    = local.dns_zone_ooni_io
 
   key_name      = module.adm_iam_roles.oonidevops_key_name
   instance_type = "t3a.small"
 
-  name = "oonifastpath"
-  ingress_rules = [{
-    from_port   = 22,
-    to_port     = 22,
-    protocol    = "tcp",
-    cidr_blocks = ["0.0.0.0/0"],
-    }, {
-    from_port   = 8472,
-    to_port     = 8472,
-    protocol    = "tcp",
-    cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block),
-    }, {
-    from_port   = 8475, # for serving jsonl files
-    to_port     = 8475,
-    protocol    = "tcp",
-    cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, module.network.vpc_subnet_public[*].cidr_block),
-    }, {
-    from_port   = 9100,
-    to_port     = 9100,
-    protocol    = "tcp"
-    cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"]
-    }, {
-    from_port   = 9102, # For fastpath metrics
-    to_port     = 9102,
-    protocol    = "tcp"
-    cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"]
-  }]
-
-  egress_rules = [{
-    from_port   = 0,
-    to_port     = 0,
-    protocol    = "-1",
-    cidr_blocks = ["0.0.0.0/0"],
-    }, {
-    from_port        = 0,
-    to_port          = 0,
-    protocol         = "-1",
-    ipv6_cidr_blocks = ["::/0"],
-  }]
-
   sg_prefix = "oonifastpath"
   tg_prefix = "fstp"
 
-  disk_size = 150
+  monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip
+  monitoring_proxy_public_ip  = module.ooni_monitoring_proxy.aws_instance_public_ip
 
-  tags = merge(
-    local.tags,
-    { Name = "ooni-tier0-fastpath" }
-  )
-}
-
-resource "aws_route53_record" "fastpath_alias" {
-  zone_id = local.dns_zone_ooni_io
-  name    = "fastpath.${local.environment}.ooni.io"
-  type    = "CNAME"
-  ttl     = 300
-
-  records = [
-    module.ooni_fastpath.aws_instance_public_dns
-  ]
+  tags = local.tags
 }
 
 module "fastpath_builder" {
@@ -979,8 +928,6 @@ module "fastpath_builder" {
   codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
 
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
-
-  ecs_cluster_name = module.ooniapi_cluster.cluster_name
 }
 
 #### OONI Run service
@@ -1313,8 +1260,6 @@ module "testlists_builder" {
   codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
 
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
-
-  ecs_cluster_name = module.ooniapi_cluster.cluster_name
 }
 
 #### OONI Tier0 API Frontend

tf/environments/prod/main.tf

@@ -1095,8 +1095,6 @@ module "fastpath_builder" {
   codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
 
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
-
-  ecs_cluster_name = module.ooniapi_cluster.cluster_name
 }
 
 
@@ -1441,8 +1439,6 @@ module "testlists_builder" {
   codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
 
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
-
-  ecs_cluster_name = module.ooniapi_cluster.cluster_name
 }
 
 #### OONI Tier0 API Frontend

tf/modules/ooni_docker_build/variables.tf

@@ -30,10 +30,6 @@ variable "repo" {
   default = "ooni/backend"
 }
 
-variable "ecs_cluster_name" {
-  description = "id of the cluster to deploy into"
-}
-
 variable "trigger_path" {
   description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**"
 }

tf/modules/ooni_fastpath/main.tf

@@ -0,0 +1,93 @@
+locals {
+  vpc_cidrs = concat(var.private_subnet_cidr, var.public_subnet_cidr)
+
+  ingress_rules = [
+    {
+      from_port   = 22
+      to_port     = 22
+      protocol    = "tcp"
+      cidr_blocks = ["0.0.0.0/0"]
+    },
+    {
+      from_port   = 8472
+      to_port     = 8472
+      protocol    = "tcp"
+      cidr_blocks = local.vpc_cidrs
+    },
+    {
+      from_port   = 8475 # for serving jsonl files
+      to_port     = 8475
+      protocol    = "tcp"
+      cidr_blocks = local.vpc_cidrs
+    },
+    {
+      from_port   = 9100
+      to_port     = 9100
+      protocol    = "tcp"
+      cidr_blocks = ["${var.monitoring_proxy_private_ip}/32"]
+    },
+    {
+      from_port = 9102 # For fastpath metrics
+      to_port   = 9102
+      protocol  = "tcp"
+      cidr_blocks = [
+        "${var.monitoring_proxy_private_ip}/32",
+        "${var.monitoring_proxy_public_ip}/32",
+      ]
+    },
+  ]
+
+  egress_rules = [
+    {
+      from_port   = 0
+      to_port     = 0
+      protocol    = "-1"
+      cidr_blocks = ["0.0.0.0/0"]
+    },
+    {
+      from_port        = 0
+      to_port          = 0
+      protocol         = "-1"
+      ipv6_cidr_blocks = ["::/0"]
+    },
+  ]
+}
+
+module "ec2" {
+  source = "../ec2"
+
+  stage = var.env
+
+  vpc_id              = var.vpc_id
+  subnet_id           = var.subnet_id
+  private_subnet_cidr = var.private_subnet_cidr
+  dns_zone_ooni_io    = var.dns_zone_ooni_io
+
+  key_name      = var.key_name
+  instance_type = var.instance_type
+
+  name          = "ooni${var.name}"
+  ingress_rules = local.ingress_rules
+  egress_rules  = local.egress_rules
+
+  sg_prefix = var.sg_prefix
+  tg_prefix = var.tg_prefix
+
+  disk_size = var.disk_size
+
+  tags = merge(
+    var.tags,
+    { Name = "ooni-tier0-${var.name}" },
+  )
+}
+
+resource "aws_route53_record" "fastpath_alias" {
+  zone_id = var.dns_zone_ooni_io
+  name    = "${var.name}.${var.env}.ooni.io"
+  type    = "CNAME"
+  ttl     = 300
+
+  records = [
+    module.ec2.aws_instance_public_dns
+  ]
+}

tf/modules/ooni_fastpath/outputs.tf

@@ -0,0 +1,27 @@
+output "aws_instance_id" {
+  value = module.ec2.aws_instance_id
+}
+
+output "aws_instance_private_ip" {
+  value = module.ec2.aws_instance_private_ip
+}
+
+output "aws_instance_public_ip" {
+  value = module.ec2.aws_instance_public_ip
+}
+
+output "aws_instance_public_dns" {
+  value = module.ec2.aws_instance_public_dns
+}
+
+output "ec2_sg_id" {
+  value = module.ec2.ec2_sg_id
+}
+
+output "alb_target_group_id" {
+  value = module.ec2.alb_target_group_id
+}
+
+output "dns_name" {
+  value = aws_route53_record.fastpath_alias.name
+}

tf/modules/ooni_fastpath/variables.tf

@@ -0,0 +1,78 @@
+variable "name" {
+  description = "Short name used for the instance and DNS record"
+  type        = string
+  default = "fastpath"
+}
+
+variable "env" {
+  description = "Deployment environment, used for the ec2 module stage and the DNS record (e.g. dev, prod)"
+  type        = string
+}
+
+variable "vpc_id" {
+  description = "ID of the VPC"
+  type        = string
+}
+
+variable "subnet_id" {
+  description = "ID of the subnet to deploy the instance into"
+  type        = string
+}
+
+variable "private_subnet_cidr" {
+  description = "CIDR blocks of the private subnets"
+  type        = list(string)
+}
+
+variable "public_subnet_cidr" {
+  description = "CIDR blocks of the public subnets"
+  type        = list(string)
+}
+
+variable "dns_zone_ooni_io" {
+  description = "Route53 zone id for ooni.io"
+  type        = string
+}
+
+variable "key_name" {
+  description = "Name of the AWS key pair"
+  type        = string
+}
+
+variable "instance_type" {
+  description = "EC2 instance type"
+  type        = string
+  default     = "t3a.small"
+}
+
+variable "disk_size" {
+  description = "Root disk size in GB"
+  type        = number
+  default     = 150
+}
+
+variable "sg_prefix" {
+  description = "Security group name prefix"
+  type        = string
+}
+
+variable "tg_prefix" {
+  description = "ALB target group name prefix (prefixed with 'oo')"
+  type        = string
+}
+
+variable "monitoring_proxy_private_ip" {
+  description = "Private IP of the monitoring proxy, allowed to scrape metrics on ports 9100 and 9102"
+  type        = string
+}
+
+variable "monitoring_proxy_public_ip" {
+  description = "Public IP of the monitoring proxy, allowed to scrape fastpath metrics on port 9102"
+  type        = string
+}
+
+variable "tags" {
+  description = "Extra tags to apply to the resources"
+  type        = map(string)
+  default     = {}
+}