Add a Terraform configuration to deploy lnt.llvm.org

.github/workflows/deploy-lnt.llvm.org.yaml

@@ -0,0 +1,39 @@
+name: Deploy lnt.llvm.org
+
+on:
+  workflow_dispatch:
+  # Eventually:
+  # pull_request_target:
+  #   paths:
+  #     - '.github/workflows/deploy-lnt.llvm.org.yaml'
+  #     - 'deployment/*'
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    runs-on: ubuntu-24.04
+
+    steps:
+    - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
+      with:
+        aws-region: us-west-2
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: Initialize Terraform
+      run: terraform -chdir=deployment init
+
+    - name: Plan Terraform changes
+      run: terraform -chdir=deployment plan
+
+    - name: Apply Terraform changes
+      if: ${{ github.ref == 'refs/heads/main' }} # only apply changes on pushes to main
+      run: terraform -chdir=deployment apply -auto-approve

.gitignore

@@ -3,6 +3,7 @@
 .tox/
 /llvm_lnt.egg-info
 build
+deployment/.terraform
 dist
 docs/_build
 lnt/server/ui/static/docs

deployment/README.md

@@ -0,0 +1,23 @@
+This directory contains configuration files to deploy lnt.llvm.org.
+
+The https://lnt.llvm.org instance gets re-deployed automatically whenever changes
+are made to the configuration files under `deployment/` on the `main` branch via
+a Github Action. Manually deploying the instance is also possible by directly using
+Terraform:
+
+```bash
+aws configure # provide appropriate access keys
+terraform -chdir=deployment init
+terraform -chdir=deployment plan # to see what will be done
+terraform -chdir=deployment apply
+```
+
+At a high level, lnt.llvm.org is running in a Docker container on an EC2 instance.
+The database is stored in an independent EBS storage that gets attached and detached
+to/from the EC2 instance when it is created/destroyed, but the EBS storage has its own
+independent life cycle (because we want the data to outlive any specific EC2 instance).
+
+The state used by Terraform to track the current status of the instance, EBS storage, etc
+is located in a S3 bucket defined in the Terraform file. It is updated automatically when
+changes are performed via the `terraform` command-line. Terraform is able to access that
+data via the AWS credentials that are set up by `aws configure`.

deployment/compose.env.tpl

@@ -0,0 +1,4 @@
+LNT_DB_PASSWORD=${__db_password__}
+LNT_AUTH_TOKEN=${__auth_token__}
+LNT_IMAGE=${__lnt_image__}
+LNT_HOST_PORT=${__lnt_host_port__}

deployment/ec2-startup.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#
+# This is the startup script that gets executed when the EC2 instance running lnt.llvm.org
+# is brought up. This script references some files under /etc/lnt that are put into place
+# by cloud-init, which is specified in the Terraform configuration file.
+#
+
+set -e
+
+echo "Installing docker"
+sudo yum update -y
+sudo yum install -y docker
+docker --version
+
+echo "Installing docker compose"
+sudo mkdir -p /usr/local/lib/docker/cli-plugins
+sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-linux-$(uname -m) \
+          -o /usr/local/lib/docker/cli-plugins/docker-compose
+sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
+docker compose version
+
+echo "Starting the Docker service"
+sudo service docker start
+sudo systemctl enable docker # also ensure the Docker service starts on reboot
+
+if ! lsblk --output FSTYPE -f /dev/sdh | grep --quiet ext4; then
+    echo "Formatting /dev/sdh -- this is a new EBS volume"
+    sudo mkfs -t ext4 /dev/sdh
+else
+    echo "/dev/sdh already contains a filesystem -- reusing previous EBS volume"
+fi
+
+echo "Mounting EBS volume with persistent information at /persistent-state"
+sudo mkdir /persistent-state
+sudo mount /dev/sdh /persistent-state
+
+echo "Creating folders to map volumes in the Docker container to locations on the EC2 instance"
+sudo mkdir -p /persistent-state/var/lib/lnt
+(cd /var/lib && ln -s /persistent-state/var/lib/lnt lnt)
+sudo mkdir -p /persistent-state/var/lib/postgresql
+(cd /var/lib && ln -s /persistent-state/var/lib/postgresql postgresql)
+sudo mkdir -p /var/log/lnt # logs are not persisted
+
+echo "Starting LNT service with Docker compose"
+sudo docker compose --file /etc/lnt/compose.yaml               \
+                    --file /etc/lnt/ec2-volume-mapping.yaml    \
+                    --env-file /etc/lnt/compose.env            \
+                    up --detach

deployment/ec2-volume-mapping.yaml

@@ -0,0 +1,27 @@
+#
+# This file maps volumes in the Docker container to actual locations on the EC2 instance.
+# We basically bind the volumes inside the Docker image to the same filesystem location
+# on the EC2 instance (e.g. /var/lib/lnt -> /var/lib/lnt) for ease of access.
+#
+
+volumes:
+  instance:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/lib/lnt
+
+  logs:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/log/lnt
+
+  database:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: /var/lib/postgresql

deployment/main.tf

@@ -0,0 +1,163 @@
+#
+# Terraform file for deploying lnt.llvm.org.
+#
+
+terraform {
+  backend "s3" {
+    bucket  = "lnt.llvm.org-test-bucket" # TODO: Adjust this for the real LLVM Foundation account
+    key     = "terraform.tfstate"
+    region  = "us-west-2"
+    encrypt = true
+  }
+}
+
+locals {
+  availability_zone = "us-west-2a"
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+#
+# Setup secrets and other variables
+#
+# Note that the LNT database password and the LNT authentication token for destructive actions
+# must be stored in the AWS Secrets Manager under a secrets named `lnt.llvm.org-secrets`, and
+# with the `lnt-db-password` and `lnt-auth-token` keys respectively. This secrets must exist
+# in whatever AWS account is currently authenticated when running Terraform.
+#
+data "aws_secretsmanager_secret" "lnt_secrets" {
+  name = "lnt.llvm.org-secrets"
+}
+
+data "aws_secretsmanager_secret_version" "lnt_secrets_latest" {
+  secret_id = data.aws_secretsmanager_secret.lnt_secrets.id
+}
+
+locals {
+  # The Docker image to use for the webserver part of the LNT service
+  lnt_image     = "d9ffa5317a9a42a1d2fa337cba97ec51d931f391"
+
+  # The port on the EC2 instance used by the Docker webserver for communication
+  lnt_host_port = "80"
+
+  # The database password for the lnt.llvm.org database.
+  lnt_db_password = jsondecode(data.aws_secretsmanager_secret_version.lnt_secrets_latest.secret_string)["lnt-db-password"]
+
+  # The authentication token to perform destructive operations on lnt.llvm.org.
+  lnt_auth_token = jsondecode(data.aws_secretsmanager_secret_version.lnt_secrets_latest.secret_string)["lnt-auth-token"]
+}
+
+#
+# Setup the EC2 instance
+#
+data "aws_ami" "amazon_linux_2023" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["al2023-ami-ecs-hvm-*-kernel-*-x86_64"]
+  }
+}
+
+data "cloudinit_config" "startup_scripts" {
+  base64_encode = true
+
+  part {
+    filename     = "ec2-startup.sh"
+    content_type = "text/x-shellscript"
+    content      = file("${path.module}/ec2-startup.sh")
+  }
+
+  part {
+    content_type = "text/cloud-config"
+    content = yamlencode({
+      write_files = [
+        {
+          path        = "/etc/lnt/compose.yaml"
+          permissions = "0400" # read-only for owner
+          content     = file("${path.module}/../docker/compose.yaml")
+        },
+        {
+          path        = "/etc/lnt/ec2-volume-mapping.yaml"
+          permissions = "0400" # read-only for owner
+          content     = file("${path.module}/ec2-volume-mapping.yaml")
+        },
+        {
+          path        = "/etc/lnt/compose.env"
+          permissions = "0400" # read-only for owner
+          content     = templatefile("${path.module}/compose.env.tpl", {
+            __db_password__   = local.lnt_db_password,
+            __auth_token__    = local.lnt_auth_token,
+            __lnt_image__     = local.lnt_image,
+            __lnt_host_port__ = local.lnt_host_port,
+          })
+        }
+      ]
+    })
+  }
+}
+
+resource "aws_security_group" "server" {
+  name        = "lnt.llvm.org/server-security-group"
+  description = "Allow SSH and HTTP traffic"
+
+  ingress {
+    description = "Allow incoming SSH traffic from anywhere"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "Allow incoming HTTP traffic from anywhere"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "Allow outgoing traffic to anywhere"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_instance" "server" {
+  ami                         = data.aws_ami.amazon_linux_2023.id
+  availability_zone           = local.availability_zone
+  instance_type               = "t2.micro" # TODO: Adjust the size of the real instance
+  associate_public_ip_address = true
+  security_groups             = [aws_security_group.server.name]
+  tags = {
+    Name = "lnt.llvm.org/server"
+  }
+
+  user_data_base64 = data.cloudinit_config.startup_scripts.rendered
+}
+
+#
+# Setup the EBS volume attached to the instance that stores the DB
+# and other instance-related configuration (e.g. the schema files,
+# profiles and anything else that should persist).
+#
+resource "aws_ebs_volume" "persistent_state" {
+  availability_zone = local.availability_zone
+  size              = 128 # GiB
+  type              = "gp2"
+  tags = {
+    Name = "lnt.llvm.org/persistent-state"
+  }
+}
+
+resource "aws_volume_attachment" "persistent_state_attachment" {
+  instance_id = aws_instance.server.id
+  volume_id   = aws_ebs_volume.persistent_state.id
+  device_name = "/dev/sdh"
+}