From 0a3b1e9675c14d664b462bde3bb217efffbf5230 Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 02:04:47 +0200 Subject: [PATCH 01/27] terraform --- .github/workflows/ci.yaml | 145 ++++++++++++++----- .gitignore | 16 ++- terraform/config.template.yaml | 29 ++++ terraform/deployment.tf | 252 +++++++++++++++++++++++++++++++++ terraform/main.tf | 15 ++ terraform/providers.tf | 40 ++++++ terraform/variables.tf | 51 +++++++ 7 files changed, 511 insertions(+), 37 deletions(-) create mode 100644 terraform/config.template.yaml create mode 100644 terraform/deployment.tf create mode 100644 terraform/main.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/variables.tf diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bb03bc18..7c22c4ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: API Lint, Build, Test, Deploy +name: EventAPI CI on: push: @@ -14,26 +14,27 @@ on: workflow_dispatch: inputs: deploy: - description: "Deploy location" + description: "Which environment to deploy to" required: true default: "none" type: choice options: - production - - staging + - test - none concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + DEPLOY: ${{ (inputs.deploy != 'none' && inputs.deploy) || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'production') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') && 'production') || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) && 'test') || 'none' }} + jobs: ci: - name: EventAPI Lint, Build, Test, Deploy + name: EventAPI Lint & Build runs-on: aws-runner env: - DEPLOY_PROD: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'production') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') }} - DEPLOY_STAGE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'staging') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) }} GOLANGCI_LINT_CACHE: /home/runner/.cache/golangci-lint concurrency: group: ${{ github.workflow }}-ci-${{ github.ref }} @@ -100,7 +101,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: ${{ env.DEPLOY != 'none' }} with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -108,54 +109,126 @@ jobs: - name: Login to Amazon ECR id: login-ecr - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: ${{ env.DEPLOY != 'none' }} uses: aws-actions/amazon-ecr-login@v1 - name: Make build context - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: ${{ env.DEPLOY != 'none' }} run: | docker context create builders - name: Setup buildx uses: docker/setup-buildx-action@v2 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: ${{ env.DEPLOY != 'none' }} with: install: true endpoint: builders - name: Build docker image uses: docker/build-push-action@v3 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: inputs.deploy != 'none' with: context: . file: docker/partial.Dockerfile - # cache-from: | - # type=registry,ref=gha - # cache-to: | - # type=registry,ref=gha,mode=max tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/eventapi:latest - ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/eventapi:${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', inputs.deploy) }}/eventapi:latest + ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', inputs.deploy) }}/eventapi:${{ github.sha }} push: true - - name: Update deployment template - uses: danielr1996/envsubst-action@1.1.0 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} - env: - IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/eventapi:${{ github.sha }} - with: - input: k8s/${{ (env.DEPLOY_PROD == 'true' && 'production') || 'staging' }}.template.yaml - output: k8s/deploy.yaml - - - name: Setup Kubectl - uses: azure/setup-kubectl@v3.0 + deploy: + name: EventAPI Deploy + needs: ci + runs-on: aws-runner + if: ${{ inputs.deploy != 'none' }} + permissions: + pull-requests: write + defaults: + run: + working-directory: terraform - - name: Deploy to k8s - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + steps: + - name: "Setup Terraform" + uses: hashicorp/setup-terraform@v1 + if: ${{ env.DEPLOY != 'none' }} + with: + cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} + + - name: "Terraform Workspace" + run: terraform workspace select -or-create=true ${{ env.DEPLOY }} + + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: "Terraform Init" + id: init + run: terraform init -upgrade + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - name: "Terraform Plan" + id: plan + run: terraform plan + continue-on-error: true + + - name: "Update" + uses: actions/github-script@v6 + if: github.event_name == 'pull_request' env: - KUBE_CONFIG_DATA: ${{ (env.DEPLOY_PROD == 'true' && secrets.KUBECONFIG) || secrets.KUBECONFIG_STAGE }} - run: | - mkdir -p ~/.kube - (echo $KUBE_CONFIG_DATA | base64 -d) >> ~/.kube/config - - kubectl apply -f k8s/deploy.yaml + PLAN: "terraform" + with: + github-token: ${{ secrets.PAT }} + script: | + // 1. Retrieve existing bot comments for the PR + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }) + const botComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style') + }) + + // 2. Prepare format of the comment + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` +
Validation Output + + \`\`\`\n + ${{ steps.validate.outputs.stdout }} + \`\`\` + +
+ + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + +
Show Plan + + \`\`\`\n + ${process.env.PLAN} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`; + + // 3. If we have a comment, update it, otherwise create a new one + if (botComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: output + }) + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + } diff --git a/.gitignore b/.gitignore index e6b41779..dd9748ef 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,18 @@ config.yaml go.work go.work.sum -shutdown \ No newline at end of file +shutdown + +# Terraform local state files +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfplan +crash.log +*.tfvars +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraformrc +terraform.rc diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml new file mode 100644 index 00000000..aa592355 --- /dev/null +++ b/terraform/config.template.yaml @@ -0,0 +1,29 @@ +level: info +redis: + addresses: + - ${redis_address} + sentinel: true + master_name: "mymaster" + +api: + enabled: true + bind: ${bind} + bridge_url: ${bridge_url} + heartbeat_interval: ${heartbeat_interval} + subscription_limit: ${subscription_limit} + connection_limit: ${connection_limit} + ttl: ${ttl} + v1: false + v3: true + +health: + enabled: true + bind: 0.0.0.0:9200 + +monitoring: + enabled: true + bind: 0.0.0.0:9100 + +pprof: + enabled: false + bind: 0.0.0.0:9300 diff --git a/terraform/deployment.tf b/terraform/deployment.tf new file mode 100644 index 00000000..83b9e5a5 --- /dev/null +++ b/terraform/deployment.tf @@ -0,0 +1,252 @@ +resource "kubernetes_namespace" "app" { + metadata { + name = var.namespace + } +} + +resource "kubernetes_secret" "app" { + metadata { + name = "eventapi" + namespace = var.namespace + } + + data = { + "config.yaml" = templatefile("${path.module}/config.yaml", { + redis_address = var.infra.redis_host, + redis_password = var.infra.redis_password, + bind = "0.0.0.0:3000", + heartbeat_interval = var.heartbeat_interval, + subscription_limit = var.subscription_limit, + connection_limit = var.connection_limit, + ttl = var.ttl, + }) + } +} + +resource "kubernetes_deployment" "app" { + metadata { + name = "eventapi" + namespace = kubernetes_namespace.app.metadata[0].name + labels = { + app = "eventapi" + } + } + + timeouts { + create = "2m" + update = "2m" + delete = "2m" + } + + spec { + selector { + match_labels = { + app = "eventapi" + } + } + + template { + metadata { + labels = { + app = "eventapi" + } + } + + spec { + container { + name = "eventapi" + image = var.app_docker_image + + port "http" { + name = "http" + container_port = 3000 + protocol = "TCP" + } + + port "metrics" { + name = "metrics" + container_port = 9100 + protocol = "TCP" + } + + port "health" { + name = "health" + container_port = 9200 + protocol = "TCP" + } + + port "pprof" { + name = "pprof" + container_port = 9300 + protocol = "TCP" + } + + env { + name = "EVENTS_K8S_POD_NAME" + value_from { + field_ref { + field_path = "metadata.name" + } + } + } + + lifecycle { + // Pre-stop hook is used to send a fallback signal to the container + // to gracefully remove all connections ahead of shutdown + pre_stop { + exec { + command = ["sh", "-c", "sleep 5 && echo \"1\" >> shutdown"] + } + } + } + + resources { + requests = { + cpu = "350m" + memory = "3000Mi" + } + limits = { + cpu = "1000m" + memory = "3225Mi" + } + } + + volume_mount { + name = "config" + mount_path = "/app/config.yaml" + sub_path = "config.yaml" + } + + liveness_probe { + tcp_socket { + port = "health" + } + initial_delay_seconds = 3 + timeout_seconds = 5 + period_seconds = 5 + success_threshold = 1 + failure_threshold = 6 + } + + readiness_probe { + tcp_socket { + port = "health" + } + initial_delay_seconds = 3 + timeout_seconds = 5 + period_seconds = 5 + success_threshold = 1 + failure_threshold = 6 + } + + // TODO: This should be "IfNotPresent", but first requires the image url to use hashes + image_pull_policy = "Always" + } + + volume { + name = "config" + secret { + secret_name = kubernetes_secret.app.metadata[0].name + } + } + } + } + } +} + +resource "kubernetes_service" "app" { + metadata { + name = "eventapi" + namespace = kubernetes_namespace.app.metadata[0].name + } + + spec { + selector = { + app = "eventapi" + } + + port { + name = "http" + port = 3000 + target_port = "http" + } + + port { + name = "metrics" + port = 9100 + target_port = "metrics" + } + + port { + name = "health" + port = 9200 + target_port = "health" + } + + port { + name = "pprof" + port = 9300 + target_port = "pprof" + } + } +} + +resource "kubernetes_ingress_v1" "app" { + metadata { + name = "eventapi" + namespace = kubernetes_namespace.app.metadata[0].name + annotations = { + "kubernetes.io/ingress.class" = "nginx" + "external-dns.alpha.kubernetes.io/hostname" = local.infra.cloudflare_tunnel_hostname + "external-dns.alpha.kubernetes.io/cloudflare-proxied" = "true" + } + } + + spec { + rule { + host = local.infra.secondary_zone + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = kubernetes_service.app.metadata[0].name + port { + name = "http" + } + } + } + } + } + } + } +} + +resource "kubernetes_horizontal_pod_autoscaler_v2" "eventapil" { + metadata { + name = "eventapi" + namespace = kubernetes_namespace.app.metadata[0].name + } + + spec { + scale_target_ref { + api_version = "apps/v1" + kind = "Deployment" + name = kubernetes_deployment.app.metadata[0].name + } + + min_replicas = 1 + max_replicas = 100 + + metrics { + resource { + name = "events_v3_current_connections" + target { + type = "Value" + value = var.connection_limit + } + } + } + } +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 00000000..ff9a85e1 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + backend "remote" { + hostname = "app.terraform.io" + organization = "7tv" + + workspaces { + prefix = "seventv-eventapi-" + } + } +} + +locals { + infra_workspace_name = replace(terraform.workspace, "eventapi", "infra") + infra = var.terraform_remote_state.infra.outputs +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 00000000..44afa341 --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,40 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.7.0" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.18.1" + } + + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 4.0" + } + } +} + +provider "aws" { + region = var.region +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + token = data.aws_eks_cluster_auth.cluster.token + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +data "aws_eks_cluster" "cluster" { + name = infra_workspace_name +} + +data "aws_eks_cluster_auth" "cluster" { + name = local.infra_workspace_name +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 00000000..eb4338bb --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,51 @@ +resource "terraform_remote_state" "infra" { + backend = "remote" + + config = { + organization = "7tv" + workspaces = { + name = "seventv-infra-${trimprefix(terraform.workspace, "eventapi")}" + } + } +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-2" +} + +variable "cloudflare_api_token" { + description = "Cloudflare API Token" + sensitive = true +} + +variable "namespace" { + type = string + default = "eventapi" +} + +variable "app_docker_image" { + type = string + default = "ghcr.io/seventv/eventapi:latest" +} + +variable "heartbeat_interval" { + type = number + default = 28 +} + +variable "subscription_limit" { + type = number + default = 500 +} + +variable "connection_limit" { + type = number + default = 10000 +} + +variable "ttl" { + type = number + default = 60 +} From 6c9f3e62389d9f74105337485b1410006c13edb8 Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 02:06:51 +0200 Subject: [PATCH 02/27] format --- .github/workflows/ci.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7c22c4ea..ccdaa3b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -168,7 +168,7 @@ jobs: - name: Terraform Validate id: validate run: terraform validate -no-color - + - name: "Terraform Plan" id: plan run: terraform plan @@ -191,7 +191,7 @@ jobs: const botComment = comments.find(comment => { return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style') }) - + // 2. Prepare format of the comment const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` @@ -201,21 +201,21 @@ jobs: \`\`\`\n ${{ steps.validate.outputs.stdout }} \`\`\` - + - + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` - +
Show Plan - + \`\`\`\n ${process.env.PLAN} \`\`\` - +
- + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`; - + // 3. If we have a comment, update it, otherwise create a new one if (botComment) { github.rest.issues.updateComment({ From 5d5340f181972a7b912546b3d4c5a58ee1177d60 Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 02:08:47 +0200 Subject: [PATCH 03/27] fix env name --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ccdaa3b2..d04d4ebd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,8 +131,8 @@ jobs: context: . file: docker/partial.Dockerfile tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', inputs.deploy) }}/eventapi:latest - ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', inputs.deploy) }}/eventapi:${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', env.DEPLOY) }}/eventapi:latest + ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', env.DEPLOY) }}/eventapi:${{ github.sha }} push: true deploy: From 06be38e8de50d8f67f6783507774bcabb290f20b Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 03:22:03 +0200 Subject: [PATCH 04/27] use ghcr --- .github/workflows/ci.yaml | 29 +++++++++++------------------ terraform/deployment.tf | 2 +- terraform/variables.tf | 3 +-- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d04d4ebd..ebae8cb4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ on: default: "none" type: choice options: - - production + - prod - test - none @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: true env: - DEPLOY: ${{ (inputs.deploy != 'none' && inputs.deploy) || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'production') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') && 'production') || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) && 'test') || 'none' }} + DEPLOY: ${{ (inputs.deploy != 'none' && inputs.deploy) || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'prod') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') && 'prod') || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) && 'test') || 'none' }} jobs: ci: @@ -99,19 +99,6 @@ jobs: - name: Build App run: make build - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - if: ${{ env.DEPLOY != 'none' }} - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - if: ${{ env.DEPLOY != 'none' }} - uses: aws-actions/amazon-ecr-login@v1 - - name: Make build context if: ${{ env.DEPLOY != 'none' }} run: | @@ -124,6 +111,13 @@ jobs: install: true endpoint: builders + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build docker image uses: docker/build-push-action@v3 if: inputs.deploy != 'none' @@ -131,8 +125,7 @@ jobs: context: . file: docker/partial.Dockerfile tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', env.DEPLOY) }}/eventapi:latest - ${{ steps.login-ecr.outputs.registry }}/${{ format('seventv-{0}', env.DEPLOY) }}/eventapi:${{ github.sha }} + ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }} push: true deploy: @@ -171,7 +164,7 @@ jobs: - name: "Terraform Plan" id: plan - run: terraform plan + run: terraform plan -var "image=ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" continue-on-error: true - name: "Update" diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 83b9e5a5..f4e85939 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -55,7 +55,7 @@ resource "kubernetes_deployment" "app" { spec { container { name = "eventapi" - image = var.app_docker_image + image = var.image_url port "http" { name = "http" diff --git a/terraform/variables.tf b/terraform/variables.tf index eb4338bb..13e21359 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -25,9 +25,8 @@ variable "namespace" { default = "eventapi" } -variable "app_docker_image" { +variable "image_url" { type = string - default = "ghcr.io/seventv/eventapi:latest" } variable "heartbeat_interval" { From 68517c024ec73ad6595f0c6fb95d1c2a1b44f2bd Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 03:32:42 +0200 Subject: [PATCH 05/27] terraform setup order --- .github/workflows/ci.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ebae8cb4..d98d87ab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,7 +137,7 @@ jobs: pull-requests: write defaults: run: - working-directory: terraform + working-directory: ./terraform steps: - name: "Setup Terraform" @@ -146,6 +146,11 @@ jobs: with: cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} + - name: "Terraform Init" + id: init + run: terraform init + continue-on-error: true + - name: "Terraform Workspace" run: terraform workspace select -or-create=true ${{ env.DEPLOY }} @@ -154,10 +159,6 @@ jobs: run: terraform fmt -check continue-on-error: true - - name: "Terraform Init" - id: init - run: terraform init -upgrade - - name: Terraform Validate id: validate run: terraform validate -no-color From 5d25c11b00f54891d74d694cd3742273b3e21380 Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 03:39:52 +0200 Subject: [PATCH 06/27] checkout at deploy --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d98d87ab..d9ec6410 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -140,6 +140,9 @@ jobs: working-directory: ./terraform steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: "Setup Terraform" uses: hashicorp/setup-terraform@v1 if: ${{ env.DEPLOY != 'none' }} From 64fae2a3793e4d1daa554776dbb507189a72590b Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 03:43:00 +0200 Subject: [PATCH 07/27] run on shared runner --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d9ec6410..d4ab008c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,7 +131,7 @@ jobs: deploy: name: EventAPI Deploy needs: ci - runs-on: aws-runner + runs-on: ubuntu-latest if: ${{ inputs.deploy != 'none' }} permissions: pull-requests: write From 3a2d8c1af67a3637a0c99fd1d1e3f37699face38 Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 03:59:09 +0200 Subject: [PATCH 08/27] fix terraform validation --- terraform/.terraform.lock.hcl | 68 +++++++++++++++++++++++++++++++++++ terraform/deployment.tf | 26 +++++++------- terraform/main.tf | 2 +- terraform/providers.tf | 2 +- terraform/variables.tf | 2 +- 5 files changed, 85 insertions(+), 15 deletions(-) create mode 100644 terraform/.terraform.lock.hcl diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 00000000..45f7de4a --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,68 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/cloudflare/cloudflare" { + version = "4.10.0" + constraints = "~> 4.0" + hashes = [ + "h1:Jv2Q4SIsK2NWi4J/pgnNuw6nLdvkBGjfeN5NB+nYtMA=", + "zh:3911fac103acce8b285b4f019381ec692b2504a3636dfd27515f7218bf64f7af", + "zh:3d50a5e52de8d061fbb9ada6d92c29540f0911cc1fac2f62c2396ff0a89337c7", + "zh:49e5a414e2e008319d1b34b3acebce569dfaad201f516ff8057f9d1d4977366d", + "zh:4eedb3a25587d8ec2fb868334db5bae38e20adaa998ec7f2818b53748a3763ab", + "zh:513b9aceae35905437cba31f7caf75526f4444e6d7848f81913fe16c8154441a", + "zh:62278c32b5a9386aa520185e2d6791bf12eaa5d3cbe6a2f86de8b73a4f7198a8", + "zh:7a278b276cd2347be67825ed4a11392f705cad57a477b0c46e09e99e4ca406e2", + "zh:829de094e3f4218ddadfd7e7257b16c9f826d545381815ea87fd9f5d95a55ad6", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:8d8a44570473e5bb690d33dd680670b3e814e530ac8554ec459d37154f3da7c8", + "zh:b13e7990ce3dc6be5b9832df127affb686800d847085576f4648d4c9aae11111", + "zh:c3b8671fd7ec3010e58d960617fa7afd27f8e733e145da7dad9a51c32d985768", + "zh:c69159ebac2bb0e1c7e68ad4aaa36681eb0ecf960dfe4b0f9e9e985ea7606f66", + "zh:cce849b19ea43c14f389f17da61ad0ca41a3320330f0bdf62fecc5a4646cb5f1", + "zh:f0d5f830a42eba80b582b4a6a651b8efb2cb222ba4c0c2a3587b932f614009da", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.7.0" + constraints = "~> 5.7.0" + hashes = [ + "h1:A7p0npQ+UHlnapVuikOzhmgchAq8agtfZGkYiiEOnp0=", + "zh:03240d7fc041d5331db7fd5f2ca4fe031321d07d2a6ca27085c5020dae13f211", + "zh:0b5252b14c354636fe0348823195dd901b457de1a033015f4a7d11cfe998c766", + "zh:2bfb62325b0487be8d1850a964f09cca0d45148faec577459c2a24334ec9977b", + "zh:2f9e317ffc57d2b5117cfe8dc266f88aa139b760bc93d8adeed7ad533a78b5a3", + "zh:36512725c9d7c559927b98fead04be58494a3a997e5270b905a75a468e307427", + "zh:5483e696d3ea764f746d3fe439f7dcc49001c3c774122d7baa51ce01011f0075", + "zh:5967635cc14f969ea26622863a2e3f9d6a7ddd3e7d35a29a7275c5e10579ac8c", + "zh:7e63c94a64af5b7aeb36ea6e3719962f65a7c28074532c02549a67212d410bb8", + "zh:8a7d5f33b11a3f5c7281413b431fa85de149ed8493ec1eea73d50d2d80a475e6", + "zh:8e2ed2d986aaf590975a79a2f6b5e60e0dc7d804ab01a8c03ab181e41cfe9b0f", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c7b8ca1b17489f16a6d0f1fc2aa9c130978ea74c9c861d8435410567a0a888f", + "zh:a54385896a70524063f0c5420be26ff6f88909bd8e6902dd3e922577b21fd546", + "zh:aecd3a8fb70b938b58d93459bfb311540fd6aaf981924bf34abd48f953b4be0d", + "zh:f3de076fa3402768d27af0187c6a677777b47691d1f0f84c9b259ff66e65953e", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.18.1" + constraints = "2.18.1" + hashes = [ + "h1:h4ezMuMNyKRMRhlrErph7QOUToc77U+rVKdR48w6tr8=", + "zh:09d69d244f5e688d9b1582112aa5d151c5336278e43d39c88ae920c26536b753", + "zh:0df4c988056f7d84d9161c6c955ad7346364c261d100ef510a6cc7fa4a235197", + "zh:2d3d0cb2931b6153a7971ce8c6fae92722b1116e16f42abbaef115dba895c8d8", + "zh:47830e8fc1760860bfa4aaf418627ff3c6ffcac6cebbbc490e5e0e6b31287d80", + "zh:49467177b514bada0fb3b6982897a347498af8ef9ef8d9fd611fe21dfded2e25", + "zh:5c7eae2c51ba175822730a63ad59cf41604c76c46c5c97332506ab42023525ce", + "zh:6efae755f02df8ab65ce7a831f33bd4817359db205652fd4bc4b969302072b15", + "zh:7e6e97b79fecd25aaf0f4fb91da945a65c36fe2ba2a4313288a60ede55506aad", + "zh:b75f2c9dd24b355ffe73e7b2fcd3145fc32735068f0ec2eba2df63f792dd16e8", + "zh:dbef9698d842eb49a846db6d7694f159ae5154ffbb7a753a9d4cab88c462a6d4", + "zh:f1b1fd580d92eedd9c8224d463997ccff1a62851fea65106aac299efe9ab622a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/deployment.tf b/terraform/deployment.tf index f4e85939..ca29a1a1 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -11,14 +11,15 @@ resource "kubernetes_secret" "app" { } data = { - "config.yaml" = templatefile("${path.module}/config.yaml", { - redis_address = var.infra.redis_host, - redis_password = var.infra.redis_password, + "config.yaml" = templatefile("${path.module}/config.template.yaml", { + redis_address = local.infra.redis_host, + redis_password = local.infra.redis_password, bind = "0.0.0.0:3000", - heartbeat_interval = var.heartbeat_interval, - subscription_limit = var.subscription_limit, - connection_limit = var.connection_limit, - ttl = var.ttl, + heartbeat_interval = tostring(var.heartbeat_interval), + subscription_limit = tostring(var.subscription_limit), + connection_limit = tostring(var.connection_limit), + ttl = tostring(var.ttl), + bridge_url = "", }) } } @@ -57,25 +58,25 @@ resource "kubernetes_deployment" "app" { name = "eventapi" image = var.image_url - port "http" { + port { name = "http" container_port = 3000 protocol = "TCP" } - port "metrics" { + port { name = "metrics" container_port = 9100 protocol = "TCP" } - port "health" { + port { name = "health" container_port = 9200 protocol = "TCP" } - port "pprof" { + port { name = "pprof" container_port = 9300 protocol = "TCP" @@ -239,7 +240,8 @@ resource "kubernetes_horizontal_pod_autoscaler_v2" "eventapil" { min_replicas = 1 max_replicas = 100 - metrics { + metric { + type = "External" resource { name = "events_v3_current_connections" target { diff --git a/terraform/main.tf b/terraform/main.tf index ff9a85e1..f5bc36f4 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -11,5 +11,5 @@ terraform { locals { infra_workspace_name = replace(terraform.workspace, "eventapi", "infra") - infra = var.terraform_remote_state.infra.outputs + infra = data.terraform_remote_state.infra.outputs } diff --git a/terraform/providers.tf b/terraform/providers.tf index 44afa341..7d25dd33 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -32,7 +32,7 @@ provider "cloudflare" { } data "aws_eks_cluster" "cluster" { - name = infra_workspace_name + name = local.infra_workspace_name } data "aws_eks_cluster_auth" "cluster" { diff --git a/terraform/variables.tf b/terraform/variables.tf index 13e21359..23b13b02 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,4 +1,4 @@ -resource "terraform_remote_state" "infra" { +data "terraform_remote_state" "infra" { backend = "remote" config = { From 73399e2424f407693fc273a865b7139b184a6d2d Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 09:25:18 +0200 Subject: [PATCH 09/27] apply --- .github/workflows/ci.yaml | 16 ++++++++++------ terraform/test.txt | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 terraform/test.txt diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d4ab008c..a9318922 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -166,10 +166,15 @@ jobs: id: validate run: terraform validate -no-color - - name: "Terraform Plan" - id: plan - run: terraform plan -var "image=ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" - continue-on-error: true + - name: Terraform Variables + run: | + echo "image_url = ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" \ + " " \ + >> terraform/eventapi.auto.tfvars + + - name: "Terraform Apply" + id: apply + run: terraform apply -var-file=eventapi.auto.tfvars - name: "Update" uses: actions/github-script@v6 @@ -177,7 +182,6 @@ jobs: env: PLAN: "terraform" with: - github-token: ${{ secrets.PAT }} script: | // 1. Retrieve existing bot comments for the PR const { data: comments } = await github.rest.issues.listComments({ @@ -201,7 +205,7 @@ jobs: - #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + #### Terraform Plan 📖\`${{ steps.apply.outcome }}\`
Show Plan diff --git a/terraform/test.txt b/terraform/test.txt new file mode 100644 index 00000000..9daeafb9 --- /dev/null +++ b/terraform/test.txt @@ -0,0 +1 @@ +test From 4d9bb5f76a9dc2924e7a246d4009f1323a680a3b Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 09:59:55 +0200 Subject: [PATCH 10/27] fix issues --- .github/workflows/ci.yaml | 9 +++++---- terraform/providers.tf | 9 --------- terraform/variables.tf | 17 ++++++----------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a9318922..68fe9c29 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -168,13 +168,14 @@ jobs: - name: Terraform Variables run: | - echo "image_url = ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" \ - " " \ - >> terraform/eventapi.auto.tfvars + cat < *.auto.tfvars + image_url="ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" + + EOF - name: "Terraform Apply" id: apply - run: terraform apply -var-file=eventapi.auto.tfvars + run: terraform apply - name: "Update" uses: actions/github-script@v6 diff --git a/terraform/providers.tf b/terraform/providers.tf index 7d25dd33..51c57cbb 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -9,11 +9,6 @@ terraform { source = "hashicorp/kubernetes" version = "2.18.1" } - - cloudflare = { - source = "cloudflare/cloudflare" - version = "~> 4.0" - } } } @@ -27,10 +22,6 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) } -provider "cloudflare" { - api_token = var.cloudflare_api_token -} - data "aws_eks_cluster" "cluster" { name = local.infra_workspace_name } diff --git a/terraform/variables.tf b/terraform/variables.tf index 23b13b02..ada285d9 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,12 +1,12 @@ data "terraform_remote_state" "infra" { - backend = "remote" + backend = "remote" - config = { - organization = "7tv" - workspaces = { - name = "seventv-infra-${trimprefix(terraform.workspace, "eventapi")}" - } + config = { + organization = "7tv" + workspaces = { + name = local.infra_workspace_name } + } } variable "region" { @@ -15,11 +15,6 @@ variable "region" { default = "us-east-2" } -variable "cloudflare_api_token" { - description = "Cloudflare API Token" - sensitive = true -} - variable "namespace" { type = string default = "eventapi" From f124b44eae6b4b3e4501f30b4197a967cb85b12d Mon Sep 17 00:00:00 2001 From: Anatole Date: Wed, 19 Jul 2023 10:15:02 +0200 Subject: [PATCH 11/27] fix: missing redis auth in cfg template --- terraform/config.template.yaml | 2 ++ terraform/deployment.tf | 1 + 2 files changed, 3 insertions(+) diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml index aa592355..167282e5 100644 --- a/terraform/config.template.yaml +++ b/terraform/config.template.yaml @@ -4,6 +4,8 @@ redis: - ${redis_address} sentinel: true master_name: "mymaster" + username: ${redis_username} + password: ${redis_password} api: enabled: true diff --git a/terraform/deployment.tf b/terraform/deployment.tf index ca29a1a1..eafa6af7 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -13,6 +13,7 @@ resource "kubernetes_secret" "app" { data = { "config.yaml" = templatefile("${path.module}/config.template.yaml", { redis_address = local.infra.redis_host, + redis_username = "default", redis_password = local.infra.redis_password, bind = "0.0.0.0:3000", heartbeat_interval = tostring(var.heartbeat_interval), From 5661a6084c110aa00fc78b0483f19587545bf269 Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 02:40:29 +0200 Subject: [PATCH 12/27] fix heartbeat default value --- .github/workflows/ci.yaml | 57 +++++++++++++-------------------------- Makefile | 3 +++ terraform/variables.tf | 2 +- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 68fe9c29..e2a34eae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -189,48 +189,27 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - }) + }); const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style') - }) - - // 2. Prepare format of the comment - const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` - #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` - #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` -
Validation Output - - \`\`\`\n - ${{ steps.validate.outputs.stdout }} - \`\`\` - -
- - #### Terraform Plan 📖\`${{ steps.apply.outcome }}\` - -
Show Plan - - \`\`\`\n - ${process.env.PLAN} - \`\`\` - -
- - *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`; - - // 3. If we have a comment, update it, otherwise create a new one + return comment.user.type === 'Bot' && comment.body.includes('Terraform Cloud Plan Output') + }); + const output = `#### Terraform Cloud Plan Output + \`\`\` + Plan: ${{ steps.plan.outputs.add }} to add, ${{ steps.plan.outputs.change }} to change, ${{ steps.plan.outputs.destroy }} to destroy. + \`\`\` + [Terraform Cloud Plan](${{ steps.plan.outputs.run_link }}) + `; + // 3. Delete previous comment so PR timeline makes sense if (botComment) { - github.rest.issues.updateComment({ + github.rest.issues.deleteComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: botComment.id, - body: output - }) - } else { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }) + }); } + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }); diff --git a/Makefile b/Makefile index cc3dbf47..3a58958f 100644 --- a/Makefile +++ b/Makefile @@ -43,3 +43,6 @@ work: dev: go run cmd/main.go + +deploy: + terraform -chdir=./terraform apply \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index ada285d9..3345cbef 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -26,7 +26,7 @@ variable "image_url" { variable "heartbeat_interval" { type = number - default = 28 + default = 28000 } variable "subscription_limit" { From eba5a87e682ca18159837983e4597524900d9753 Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 02:49:48 +0200 Subject: [PATCH 13/27] plan --- .github/workflows/ci.yaml | 39 +-------------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2a34eae..2b7cac40 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -175,41 +175,4 @@ jobs: - name: "Terraform Apply" id: apply - run: terraform apply - - - name: "Update" - uses: actions/github-script@v6 - if: github.event_name == 'pull_request' - env: - PLAN: "terraform" - with: - script: | - // 1. Retrieve existing bot comments for the PR - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('Terraform Cloud Plan Output') - }); - const output = `#### Terraform Cloud Plan Output - \`\`\` - Plan: ${{ steps.plan.outputs.add }} to add, ${{ steps.plan.outputs.change }} to change, ${{ steps.plan.outputs.destroy }} to destroy. - \`\`\` - [Terraform Cloud Plan](${{ steps.plan.outputs.run_link }}) - `; - // 3. Delete previous comment so PR timeline makes sense - if (botComment) { - github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - }); - } - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }); + run: terraform plan From 9c1eae18fbf294dbcb7232b42ac338c2e19777f9 Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 02:58:02 +0200 Subject: [PATCH 14/27] comment --- .github/workflows/ci.yaml | 62 +++++++++++++++++++++++++++++++++++++-- terraform/main.tf | 2 +- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b7cac40..817fa661 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,6 +173,64 @@ jobs: EOF - - name: "Terraform Apply" - id: apply + - name: "Terraform Plan" + id: plan run: terraform plan + + - uses: actions/github-script@v6 + if: github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + // 1. Retrieve existing bot comments for the PR + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }) + const botComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style') + }) + + // 2. Prepare format of the comment + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` +
Validation Output + + \`\`\`\n + ${{ steps.validate.outputs.stdout }} + \`\`\` + +
+ + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + +
Show Plan + + \`\`\`\n + ${process.env.PLAN} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`; + + // 3. If we have a comment, update it, otherwise create a new one + if (botComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: output + }) + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + } diff --git a/terraform/main.tf b/terraform/main.tf index f5bc36f4..393d1ec8 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -11,5 +11,5 @@ terraform { locals { infra_workspace_name = replace(terraform.workspace, "eventapi", "infra") - infra = data.terraform_remote_state.infra.outputs + infra = data.terraform_remote_state.infra.outputs } From 2cb678b0f7fcaedfec922e1f5c733e8f9186d5db Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 03:23:46 +0200 Subject: [PATCH 15/27] formatting --- .github/workflows/ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 817fa661..39b643cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -175,8 +175,7 @@ jobs: - name: "Terraform Plan" id: plan - run: terraform plan - + run: terraform plan -no-color - uses: actions/github-script@v6 if: github.event_name == 'pull_request' env: @@ -216,7 +215,7 @@ jobs:
- *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Working Directory: \`${{ env.tf_actions_working_dir }}\`, Workflow: \`${{ github.workflow }}\`*`; + *Actor: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Workflow: \`${{ github.workflow }}\`*`; // 3. If we have a comment, update it, otherwise create a new one if (botComment) { From 4014d2667bf0f7b28166eaf7ac97e4b9cb266a3e Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 04:07:11 +0200 Subject: [PATCH 16/27] move up deploy --- .github/workflows/ci.yaml | 11 +++++++++-- Makefile | 4 ++-- terraform/.terraform.lock.hcl | 23 ----------------------- terraform/deployment.tf | 2 +- 4 files changed, 12 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 39b643cf..06e8571b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -126,10 +126,11 @@ jobs: file: docker/partial.Dockerfile tags: | ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }} + ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-latest push: true - deploy: - name: EventAPI Deploy + validate: + name: EventAPI Deploy Validation needs: ci runs-on: ubuntu-latest if: ${{ inputs.deploy != 'none' }} @@ -176,6 +177,7 @@ jobs: - name: "Terraform Plan" id: plan run: terraform plan -no-color + - uses: actions/github-script@v6 if: github.event_name == 'pull_request' env: @@ -233,3 +235,8 @@ jobs: body: output }) } + + - name: "Terraform Apply" + id: apply + run: terraform apply -no-color + continue-on-error: true diff --git a/Makefile b/Makefile index 3a58958f..5af5a5a4 100644 --- a/Makefile +++ b/Makefile @@ -44,5 +44,5 @@ work: dev: go run cmd/main.go -deploy: - terraform -chdir=./terraform apply \ No newline at end of file +terraform: + terraform -chdir=./terraform init diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 45f7de4a..8c8b71c2 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,29 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/cloudflare/cloudflare" { - version = "4.10.0" - constraints = "~> 4.0" - hashes = [ - "h1:Jv2Q4SIsK2NWi4J/pgnNuw6nLdvkBGjfeN5NB+nYtMA=", - "zh:3911fac103acce8b285b4f019381ec692b2504a3636dfd27515f7218bf64f7af", - "zh:3d50a5e52de8d061fbb9ada6d92c29540f0911cc1fac2f62c2396ff0a89337c7", - "zh:49e5a414e2e008319d1b34b3acebce569dfaad201f516ff8057f9d1d4977366d", - "zh:4eedb3a25587d8ec2fb868334db5bae38e20adaa998ec7f2818b53748a3763ab", - "zh:513b9aceae35905437cba31f7caf75526f4444e6d7848f81913fe16c8154441a", - "zh:62278c32b5a9386aa520185e2d6791bf12eaa5d3cbe6a2f86de8b73a4f7198a8", - "zh:7a278b276cd2347be67825ed4a11392f705cad57a477b0c46e09e99e4ca406e2", - "zh:829de094e3f4218ddadfd7e7257b16c9f826d545381815ea87fd9f5d95a55ad6", - "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:8d8a44570473e5bb690d33dd680670b3e814e530ac8554ec459d37154f3da7c8", - "zh:b13e7990ce3dc6be5b9832df127affb686800d847085576f4648d4c9aae11111", - "zh:c3b8671fd7ec3010e58d960617fa7afd27f8e733e145da7dad9a51c32d985768", - "zh:c69159ebac2bb0e1c7e68ad4aaa36681eb0ecf960dfe4b0f9e9e985ea7606f66", - "zh:cce849b19ea43c14f389f17da61ad0ca41a3320330f0bdf62fecc5a4646cb5f1", - "zh:f0d5f830a42eba80b582b4a6a651b8efb2cb222ba4c0c2a3587b932f614009da", - ] -} - provider "registry.terraform.io/hashicorp/aws" { version = "5.7.0" constraints = "~> 5.7.0" diff --git a/terraform/deployment.tf b/terraform/deployment.tf index eafa6af7..5a72045c 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -206,7 +206,7 @@ resource "kubernetes_ingress_v1" "app" { spec { rule { - host = local.infra.secondary_zone + host = join(".", ["events", local.infra.secondary_zone]) http { path { path = "/" From 6243e9031c4a7900a51b3c18315f661ea4b712d0 Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 20 Jul 2023 06:54:01 +0200 Subject: [PATCH 17/27] variable image pull policy --- .github/workflows/ci.yaml | 1 + terraform/deployment.tf | 3 +-- terraform/variables.tf | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 06e8571b..b7d736ff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,6 +171,7 @@ jobs: run: | cat < *.auto.tfvars image_url="ghcr.io/seventv/eventapi:${{ env.DEPLOY }}-${{ github.sha }}" + image_pull_policy="IfNotPresent" EOF diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 5a72045c..0994a3af 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -141,8 +141,7 @@ resource "kubernetes_deployment" "app" { failure_threshold = 6 } - // TODO: This should be "IfNotPresent", but first requires the image url to use hashes - image_pull_policy = "Always" + image_pull_policy = var.image_pull_policy } volume { diff --git a/terraform/variables.tf b/terraform/variables.tf index 3345cbef..30bbe7d5 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -24,6 +24,11 @@ variable "image_url" { type = string } +variable "image_pull_policy" { + type = string + default = "Always" +} + variable "heartbeat_interval" { type = number default = 28000 From 115796a3ae1cc677cf5d64bcf39807f4405211db Mon Sep 17 00:00:00 2001 From: Anatole Date: Fri, 21 Jul 2023 12:10:08 +0200 Subject: [PATCH 18/27] ingress wrong annotation --- .github/workflows/ci.yaml | 2 ++ Makefile | 3 +++ terraform/deployment.tf | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b7d736ff..d6093611 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,6 +152,8 @@ jobs: - name: "Terraform Init" id: init + env: + TF_WORKSPACE: ${{ env.DEPLOY }} run: terraform init continue-on-error: true diff --git a/Makefile b/Makefile index 5af5a5a4..6a6c0210 100644 --- a/Makefile +++ b/Makefile @@ -46,3 +46,6 @@ dev: terraform: terraform -chdir=./terraform init + +deploy: + terraform -chdir=./terraform apply -auto-approve diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 0994a3af..c173327f 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -198,7 +198,7 @@ resource "kubernetes_ingress_v1" "app" { namespace = kubernetes_namespace.app.metadata[0].name annotations = { "kubernetes.io/ingress.class" = "nginx" - "external-dns.alpha.kubernetes.io/hostname" = local.infra.cloudflare_tunnel_hostname + "external-dns.alpha.kubernetes.io/target" = local.infra.cloudflare_tunnel_hostname "external-dns.alpha.kubernetes.io/cloudflare-proxied" = "true" } } From 8b8be75378171aa4d39702d54a28a170ab776741 Mon Sep 17 00:00:00 2001 From: Anatole Date: Sat, 22 Jul 2023 03:59:21 +0200 Subject: [PATCH 19/27] adjust hpa --- .github/workflows/ci.yaml | 2 +- terraform/deployment.tf | 13 ++++++++----- terraform/test.txt | 1 - 3 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 terraform/test.txt diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d6093611..e3a975f4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -241,5 +241,5 @@ jobs: - name: "Terraform Apply" id: apply - run: terraform apply -no-color + run: terraform apply -no-color -auto-approve continue-on-error: true diff --git a/terraform/deployment.tf b/terraform/deployment.tf index c173327f..838955cb 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -241,12 +241,15 @@ resource "kubernetes_horizontal_pod_autoscaler_v2" "eventapil" { max_replicas = 100 metric { - type = "External" - resource { - name = "events_v3_current_connections" + type = "Pods" + pods { + metric { + name = "events_v3_current_connections" + } + target { - type = "Value" - value = var.connection_limit + type = "Value" + average_value = "10000" } } } diff --git a/terraform/test.txt b/terraform/test.txt deleted file mode 100644 index 9daeafb9..00000000 --- a/terraform/test.txt +++ /dev/null @@ -1 +0,0 @@ -test From 4c02bba47d1b4b65b2125fef789abfd342ec2ace Mon Sep 17 00:00:00 2001 From: Anatole Date: Mon, 24 Jul 2023 19:11:25 +0200 Subject: [PATCH 20/27] default image url, service monitor & tolerations --- terraform/deployment.tf | 44 +++++++++++++++++++++++++++++++++++++---- terraform/main.tf | 1 + terraform/providers.tf | 12 +++++++++++ terraform/variables.tf | 4 +++- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 838955cb..9078879a 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -55,9 +55,20 @@ resource "kubernetes_deployment" "app" { } spec { + node_selector = { + "node-group" = "events" + } + + toleration { + key = "seventv-pool" + operator = "Equal" + value = "events" + effect = "NoSchedule" + } + container { name = "eventapi" - image = var.image_url + image = local.image_url port { name = "http" @@ -159,6 +170,9 @@ resource "kubernetes_service" "app" { metadata { name = "eventapi" namespace = kubernetes_namespace.app.metadata[0].name + labels = { + app = "eventapi" + } } spec { @@ -192,6 +206,28 @@ resource "kubernetes_service" "app" { } } +resource "kubectl_manifest" "app_monitor" { + depends_on = [kubernetes_deployment.app] + + yaml_body = < Date: Wed, 26 Jul 2023 01:19:21 +0200 Subject: [PATCH 21/27] update affinity and toleration --- terraform/deployment.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 9078879a..29f49570 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -56,11 +56,11 @@ resource "kubernetes_deployment" "app" { spec { node_selector = { - "node-group" = "events" + "7tv.io/node-pool" = "events" } toleration { - key = "seventv-pool" + key = "7tv.io/node-pool" operator = "Equal" value = "events" effect = "NoSchedule" From caebe6dbf0345992d9b37574ac7f38c64867d506 Mon Sep 17 00:00:00 2001 From: Anatole Date: Fri, 28 Jul 2023 10:09:35 +0200 Subject: [PATCH 22/27] health: handle concurrency limit --- cmd/main.go | 11 +++++++---- internal/app/server.go | 8 ++++++-- internal/health/health.go | 19 +++++++++++++++---- terraform/deployment.tf | 8 ++++---- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e5727c4e..ea3d1f5a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -86,16 +86,19 @@ func main() { dones := []<-chan struct{}{} + var srv *app.Server + if gctx.Config().API.Enabled { - _, done := app.New(gctx) + var done <-chan struct{} + + srv, done = app.New(gctx) dones = append(dones, done) } if gctx.Config().PProf.Enabled { - done := pprof.New(gctx) - dones = append(dones, done) + dones = append(dones, pprof.New(gctx)) } if gctx.Config().Health.Enabled { - dones = append(dones, health.New(gctx)) + dones = append(dones, health.New(gctx, srv)) } if gctx.Config().Monitoring.Enabled { dones = append(dones, monitoring.New(gctx)) diff --git a/internal/app/server.go b/internal/app/server.go index 0ffc6ede..7e7f7ac1 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -23,7 +23,7 @@ type Server struct { activeConns *int32 } -func New(gctx global.Context) (Server, <-chan struct{}) { +func New(gctx global.Context) (*Server, <-chan struct{}) { upgrader := websocket.FastHTTPUpgrader{ CheckOrigin: func(ctx *fasthttp.RequestCtx) bool { return true @@ -140,7 +140,11 @@ func New(gctx global.Context) (Server, <-chan struct{}) { close(done) }() - return srv, done + return &srv, done +} + +func (s Server) GetConcurrentCinnections() int32 { + return atomic.LoadInt32(s.activeConns) } type ErrorResponse struct { diff --git a/internal/health/health.go b/internal/health/health.go index 2554729a..06576e87 100644 --- a/internal/health/health.go +++ b/internal/health/health.go @@ -4,12 +4,13 @@ import ( "context" "time" + "github.com/seventv/eventapi/internal/app" "github.com/seventv/eventapi/internal/global" "github.com/valyala/fasthttp" "go.uber.org/zap" ) -func New(gCtx global.Context) <-chan struct{} { +func New(gctx global.Context, srv *app.Server) <-chan struct{} { server := fasthttp.Server{ Handler: func(ctx *fasthttp.RequestCtx) { start := time.Now() @@ -26,12 +27,22 @@ func New(gCtx global.Context) <-chan struct{} { } }() + ctx.Response.Header.Set("Cache-Control", "no-cache, no-store, must-revalidate") ctx.SetStatusCode(200) redisCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() - if err := gCtx.Inst().Redis.Ping(redisCtx); err != nil { + if err := gctx.Inst().Redis.Ping(redisCtx); err != nil { zap.S().Error("redis down: ", err) + + ctx.SetBodyString("Redis Down") + ctx.SetStatusCode(503) + } + + if srv != nil && srv.GetConcurrentCinnections() >= (gctx.Config().API.ConnectionLimit) { + zap.S().Warnw("connection limit reached") + + ctx.SetBodyString("Maximum Concurrency") ctx.SetStatusCode(503) } }, @@ -40,14 +51,14 @@ func New(gCtx global.Context) <-chan struct{} { } go func() { - if err := server.ListenAndServe(gCtx.Config().Health.Bind); err != nil { + if err := server.ListenAndServe(gctx.Config().Health.Bind); err != nil { zap.S().Fatal("failed to start health bind: ", err) } }() done := make(chan struct{}) go func() { - <-gCtx.Done() + <-gctx.Done() _ = server.Shutdown() close(done) }() diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 29f49570..93cb3d46 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -35,7 +35,7 @@ resource "kubernetes_deployment" "app" { } timeouts { - create = "2m" + create = "4m" update = "2m" delete = "2m" } @@ -147,9 +147,9 @@ resource "kubernetes_deployment" "app" { } initial_delay_seconds = 3 timeout_seconds = 5 - period_seconds = 5 - success_threshold = 1 - failure_threshold = 6 + period_seconds = 1 + success_threshold = 3 + failure_threshold = 1 } image_pull_policy = var.image_pull_policy From fe95b08667b785f5d2231a924580953060f668cf Mon Sep 17 00:00:00 2001 From: Anatole Date: Fri, 28 Jul 2023 10:53:50 +0200 Subject: [PATCH 23/27] correct probe for concurency health check --- go.mod | 4 ++-- go.sum | 10 ++++------ internal/app/connection/connection.go | 2 +- .../app/connection/eventstream/eventstream.go | 6 +++++- .../eventstream/eventstream_read.go | 2 +- .../app/connection/websocket/websocket.go | 2 +- .../connection/websocket/websocket_read.go | 2 +- internal/app/server.go | 20 ++++++++++++++++++- internal/global/instances.go | 5 +++-- internal/health/health.go | 6 ++++-- terraform/deployment.tf | 10 ++++++---- 11 files changed, 47 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index fe414433..91ab2b94 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.12 github.com/prometheus/client_golang v1.14.0 - github.com/seventv/api v0.0.0-20230328043606-720366e31af1 - github.com/seventv/common v0.0.0-20230402095344-527b0414019e + github.com/seventv/api v0.0.0-20230728083849-9b49a7905591 + github.com/seventv/common v0.0.0-20230528214454-1a842fd909aa github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/valyala/fasthttp v1.44.0 diff --git a/go.sum b/go.sum index e4c0e265..671767ff 100644 --- a/go.sum +++ b/go.sum @@ -233,12 +233,10 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= -github.com/seventv/api v0.0.0-20230328043606-720366e31af1 h1:UaHUcEeALbiBXAHwv5lKPwImQShXTq4kX2Q0o2awGWI= -github.com/seventv/api v0.0.0-20230328043606-720366e31af1/go.mod h1:o1EMa6sXZH5SxgrVno0pTOaMU8G5Q2LLa2TASm7e5cs= -github.com/seventv/common v0.0.0-20230321183435-350a6acc3175 h1:D0Op+jkNGUYCptieoHuiVROVupx+IQERHbyceLtjepI= -github.com/seventv/common v0.0.0-20230321183435-350a6acc3175/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw= -github.com/seventv/common v0.0.0-20230402095344-527b0414019e h1:8mChwg4qO8yi+XmSp7848Jmskz+7mPhSOExmgruiwSA= -github.com/seventv/common v0.0.0-20230402095344-527b0414019e/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw= +github.com/seventv/api v0.0.0-20230728083849-9b49a7905591 h1:/uUa/gPQkGj7xFUA7VBNFuEUGifk3S3yZApwQp8Erf8= +github.com/seventv/api v0.0.0-20230728083849-9b49a7905591/go.mod h1:P9sY6WXnMOgGwTxNj4m9pWrFCyMfTyTa2JFw+HwG8Pg= +github.com/seventv/common v0.0.0-20230528214454-1a842fd909aa h1:kIYvE8Ii1VmbTNQzI22QMiufnQZeG3vWURrCHd4ZH2I= +github.com/seventv/common v0.0.0-20230528214454-1a842fd909aa/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= diff --git a/internal/app/connection/connection.go b/internal/app/connection/connection.go index 6dbb7c04..4ee2609d 100644 --- a/internal/app/connection/connection.go +++ b/internal/app/connection/connection.go @@ -20,7 +20,7 @@ type Connection interface { // Retrieve the hex-encoded ID of this session SessionID() string // Greet sends an Hello message to the client - Greet() error + Greet(gctx global.Context) error // Listen for incoming and outgoing events Read(gctx global.Context) // SendHeartbeat lets the client know that the connection is healthy diff --git a/internal/app/connection/eventstream/eventstream.go b/internal/app/connection/eventstream/eventstream.go index 1f5a4afe..fabd9bab 100644 --- a/internal/app/connection/eventstream/eventstream.go +++ b/internal/app/connection/eventstream/eventstream.go @@ -127,11 +127,15 @@ func (es *EventStream) Buffer() client.EventBuffer { return es.evbuf } -func (es *EventStream) Greet() error { +func (es *EventStream) Greet(gctx global.Context) error { msg := events.NewMessage(events.OpcodeHello, events.HelloPayload{ HeartbeatInterval: uint32(es.heartbeatInterval), SessionID: hex.EncodeToString(es.sessionID), SubscriptionLimit: es.subscriptionLimit, + Instance: events.HelloPayloadInstanceInfo{ + Name: gctx.Config().Pod.Name, + Population: gctx.Inst().ConcurrencyValue, + }, }) return es.Write(msg.ToRaw()) diff --git a/internal/app/connection/eventstream/eventstream_read.go b/internal/app/connection/eventstream/eventstream_read.go index 5640e59e..55d3c046 100644 --- a/internal/app/connection/eventstream/eventstream_read.go +++ b/internal/app/connection/eventstream/eventstream_read.go @@ -23,7 +23,7 @@ func (es *EventStream) Read(gctx global.Context) { es.Destroy() }() - if err := es.Greet(); err != nil { + if err := es.Greet(gctx); err != nil { return } diff --git a/internal/app/connection/websocket/websocket.go b/internal/app/connection/websocket/websocket.go index 3e80add8..6f3d349b 100644 --- a/internal/app/connection/websocket/websocket.go +++ b/internal/app/connection/websocket/websocket.go @@ -74,7 +74,7 @@ func (w *WebSocket) SessionID() string { return hex.EncodeToString(w.sessionID) } -func (w *WebSocket) Greet() error { +func (w *WebSocket) Greet(gctx global.Context) error { msg := events.NewMessage(events.OpcodeHello, events.HelloPayload{ HeartbeatInterval: uint32(w.heartbeatInterval), SessionID: hex.EncodeToString(w.sessionID), diff --git a/internal/app/connection/websocket/websocket_read.go b/internal/app/connection/websocket/websocket_read.go index ecc1bdcf..b11af930 100644 --- a/internal/app/connection/websocket/websocket_read.go +++ b/internal/app/connection/websocket/websocket_read.go @@ -126,7 +126,7 @@ func (w *WebSocket) Read(gctx global.Context) { } }() - if err := w.Greet(); err != nil { + if err := w.Greet(gctx); err != nil { return } diff --git a/internal/app/server.go b/internal/app/server.go index 7e7f7ac1..2933e516 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -140,10 +140,28 @@ func New(gctx global.Context) (*Server, <-chan struct{}) { close(done) }() + go func() { + ticker := time.NewTicker(time.Second * 10) + + for { + select { + case <-gctx.Done(): + ticker.Stop() + return + case <-ticker.C: + v := atomic.LoadInt32(srv.activeConns) + + gctx.Inst().ConcurrencyValue = v + + zap.S().Infof("concurrency: %d", v) + } + } + }() + return &srv, done } -func (s Server) GetConcurrentCinnections() int32 { +func (s Server) GetConcurrentConnections() int32 { return atomic.LoadInt32(s.activeConns) } diff --git a/internal/global/instances.go b/internal/global/instances.go index 8ea54c5f..1066ad43 100644 --- a/internal/global/instances.go +++ b/internal/global/instances.go @@ -3,6 +3,7 @@ package global import "github.com/seventv/eventapi/internal/instance" type Instances struct { - Redis instance.Redis - Monitoring instance.Monitoring + Redis instance.Redis + Monitoring instance.Monitoring + ConcurrencyValue int32 } diff --git a/internal/health/health.go b/internal/health/health.go index 06576e87..33058cec 100644 --- a/internal/health/health.go +++ b/internal/health/health.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/seventv/common/utils" "github.com/seventv/eventapi/internal/app" "github.com/seventv/eventapi/internal/global" "github.com/valyala/fasthttp" @@ -39,11 +40,12 @@ func New(gctx global.Context, srv *app.Server) <-chan struct{} { ctx.SetStatusCode(503) } - if srv != nil && srv.GetConcurrentCinnections() >= (gctx.Config().API.ConnectionLimit) { + // check if path is /concurrency + if !ctx.IsGet() || utils.B2S(ctx.URI().Path()) == "/concurrency" && srv != nil && srv.GetConcurrentConnections() >= (gctx.Config().API.ConnectionLimit) { zap.S().Warnw("connection limit reached") ctx.SetBodyString("Maximum Concurrency") - ctx.SetStatusCode(503) + ctx.SetStatusCode(410) } }, GetOnly: true, diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 93cb3d46..876e065c 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -131,24 +131,26 @@ resource "kubernetes_deployment" "app" { } liveness_probe { - tcp_socket { + http_get { + path = "/" port = "health" } initial_delay_seconds = 3 timeout_seconds = 5 period_seconds = 5 success_threshold = 1 - failure_threshold = 6 + failure_threshold = 3 } readiness_probe { - tcp_socket { + http_get { + path = "/concurrency" port = "health" } initial_delay_seconds = 3 timeout_seconds = 5 period_seconds = 1 - success_threshold = 3 + success_threshold = 1 failure_threshold = 1 } From e70e976f950fc8a9d9eb51f7a74956fdbb23ac2c Mon Sep 17 00:00:00 2001 From: Anatole Date: Fri, 28 Jul 2023 15:46:22 +0200 Subject: [PATCH 24/27] eventstream connection check frequency --- internal/app/connection/eventstream/eventstream_read.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/app/connection/eventstream/eventstream_read.go b/internal/app/connection/eventstream/eventstream_read.go index 55d3c046..7c7968dd 100644 --- a/internal/app/connection/eventstream/eventstream_read.go +++ b/internal/app/connection/eventstream/eventstream_read.go @@ -18,9 +18,12 @@ func (es *EventStream) Read(gctx global.Context) { heartbeat := time.NewTicker(time.Duration(es.heartbeatInterval) * time.Millisecond) + liveness := time.NewTicker(time.Second * 1) + defer func() { heartbeat.Stop() es.Destroy() + liveness.Stop() }() if err := es.Greet(gctx); err != nil { @@ -49,6 +52,7 @@ func (es *EventStream) Read(gctx global.Context) { if err := es.SendHeartbeat(); err != nil { return } + case <-liveness.C: // Connection liveness check case s = <-es.evm.DispatchChannel(): if s == nil { // channel closed return From e7939fd1e160d66d2ec7aa518f8cf99f222a93ce Mon Sep 17 00:00:00 2001 From: Anatole Date: Fri, 28 Jul 2023 15:49:51 +0200 Subject: [PATCH 25/27] adjust hpa values --- internal/health/health.go | 2 +- terraform/deployment.tf | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/internal/health/health.go b/internal/health/health.go index 33058cec..883594dc 100644 --- a/internal/health/health.go +++ b/internal/health/health.go @@ -41,7 +41,7 @@ func New(gctx global.Context, srv *app.Server) <-chan struct{} { } // check if path is /concurrency - if !ctx.IsGet() || utils.B2S(ctx.URI().Path()) == "/concurrency" && srv != nil && srv.GetConcurrentConnections() >= (gctx.Config().API.ConnectionLimit) { + if !ctx.IsGet() || utils.B2S(ctx.URI().Path()) == "/concurrency" && srv != nil && srv.GetConcurrentConnections() >= (gctx.Config().API.ConnectionLimit-1) { zap.S().Warnw("connection limit reached") ctx.SetBodyString("Maximum Concurrency") diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 876e065c..9aefe9a0 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -150,7 +150,7 @@ resource "kubernetes_deployment" "app" { initial_delay_seconds = 3 timeout_seconds = 5 period_seconds = 1 - success_threshold = 1 + success_threshold = 3 failure_threshold = 1 } @@ -225,8 +225,8 @@ spec: app: eventapi endpoints: - port: metrics - interval: 10s - scrapeTimeout: 10s + interval: 3s + scrapeTimeout: 2s YAML } @@ -287,7 +287,38 @@ resource "kubernetes_horizontal_pod_autoscaler_v2" "app" { target { type = "AverageValue" - average_value = var.connection_limit * 0.75 + average_value = var.connection_limit * 0.85 + } + } + } + + behavior { + scale_up { + stabilization_window_seconds = 10 + select_policy = "Min" + policy { + period_seconds = 5 + type = "Percent" + value = 25 + } + policy { + period_seconds = 5 + type = "Pods" + value = 6 + } + } + scale_down { + stabilization_window_seconds = 60 + select_policy = "Min" + policy { + period_seconds = 15 + type = "Percent" + value = 10 + } + policy { + period_seconds = 15 + type = "Pods" + value = 2 } } } From aac1cad7c9e645e6ddeb8eebe4c74b79aef77d7c Mon Sep 17 00:00:00 2001 From: Anatole Date: Thu, 3 Aug 2023 11:40:15 +0200 Subject: [PATCH 26/27] stage scaling --- .github/workflows/ci.yaml | 9 ++++----- terraform/deployment.tf | 9 ++++----- terraform/variables.tf | 6 ++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e3a975f4..2646d044 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,13 +100,13 @@ jobs: run: make build - name: Make build context - if: ${{ env.DEPLOY != 'none' }} + if: env.DEPLOY != 'none' run: | docker context create builders - name: Setup buildx uses: docker/setup-buildx-action@v2 - if: ${{ env.DEPLOY != 'none' }} + if: env.DEPLOY != 'none' with: install: true endpoint: builders @@ -120,7 +120,7 @@ jobs: - name: Build docker image uses: docker/build-push-action@v3 - if: inputs.deploy != 'none' + if: env.DEPLOY != 'none' with: context: . file: docker/partial.Dockerfile @@ -133,7 +133,6 @@ jobs: name: EventAPI Deploy Validation needs: ci runs-on: ubuntu-latest - if: ${{ inputs.deploy != 'none' }} permissions: pull-requests: write defaults: @@ -146,7 +145,6 @@ jobs: - name: "Setup Terraform" uses: hashicorp/setup-terraform@v1 - if: ${{ env.DEPLOY != 'none' }} with: cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} @@ -240,6 +238,7 @@ jobs: } - name: "Terraform Apply" + if: env.deploy != 'none' id: apply run: terraform apply -no-color -auto-approve continue-on-error: true diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 9aefe9a0..ab8255a5 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -115,12 +115,12 @@ resource "kubernetes_deployment" "app" { resources { requests = { - cpu = "350m" - memory = "3000Mi" + cpu = var.production ? "350m" : "100m" + memory = var.production ? "3Gi" : "500Mi" } limits = { - cpu = "1000m" - memory = "3225Mi" + cpu = var.production ? "500m" : "150m" + memory = var.production ? "3.25Gi" : "550Mi" } } @@ -235,7 +235,6 @@ resource "kubernetes_ingress_v1" "app" { name = "eventapi" namespace = kubernetes_namespace.app.metadata[0].name annotations = { - "kubernetes.io/ingress.class" = "nginx" "external-dns.alpha.kubernetes.io/target" = local.infra.cloudflare_tunnel_hostname "external-dns.alpha.kubernetes.io/cloudflare-proxied" = "true" } diff --git a/terraform/variables.tf b/terraform/variables.tf index 8e547ee9..d09472db 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -20,6 +20,12 @@ variable "namespace" { default = "eventapi" } +variable "production" { + description = "Whether or not to scale resources to a production state" + type = bool + default = false +} + variable "image_url" { type = string nullable = true From cbf384824d9511fa7f5509d5ad476d2c70c08cc0 Mon Sep 17 00:00:00 2001 From: Anatole Date: Tue, 15 Aug 2023 20:19:35 +0200 Subject: [PATCH 27/27] remove recover() --- internal/app/connection/websocket/websocket_read.go | 3 --- internal/app/server.go | 7 ++----- internal/instance/redis.go | 5 ----- terraform/deployment.tf | 10 +++++----- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/internal/app/connection/websocket/websocket_read.go b/internal/app/connection/websocket/websocket_read.go index b11af930..0126aad0 100644 --- a/internal/app/connection/websocket/websocket_read.go +++ b/internal/app/connection/websocket/websocket_read.go @@ -53,9 +53,6 @@ func (w *WebSocket) Read(gctx global.Context) { defer func() { deferred = true - // Ignore panics, they're caused by fasthttp - _ = recover() - buf := w.Buffer() if buf != nil { <-buf.Context().Done() diff --git a/internal/app/server.go b/internal/app/server.go index 2933e516..a4be6e20 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -59,11 +59,8 @@ func New(gctx global.Context) (*Server, <-chan struct{}) { "method", utils.B2S(ctx.Method()), "entrypoint", "api", ) - if err := recover(); err != nil { - l.Error("panic in handler: ", err) - } else { - l.Debug("") - } + + l.Debug("") }() ctx.Response.Header.Set("X-Pod-Name", gctx.Config().Pod.Name) diff --git a/internal/instance/redis.go b/internal/instance/redis.go index bc8ab8b7..87089a0c 100644 --- a/internal/instance/redis.go +++ b/internal/instance/redis.go @@ -28,11 +28,6 @@ func WrapRedis(r redis.Instance) Redis { subs: map[string][]*redisSub{}, } go func() { - defer func() { - if err := recover(); err != nil { - zap.S().Fatalw("panic in subs", "error", err) - } - }() ch := inst.sub.Channel() var msg *goRedis.Message for { diff --git a/terraform/deployment.tf b/terraform/deployment.tf index ab8255a5..e0042bcf 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -115,12 +115,12 @@ resource "kubernetes_deployment" "app" { resources { requests = { - cpu = var.production ? "350m" : "100m" - memory = var.production ? "3Gi" : "500Mi" + cpu = local.infra.production ? "250m" : "150m" + memory = local.infra.production ? "3Gi" : "500Mi" } limits = { - cpu = var.production ? "500m" : "150m" - memory = var.production ? "3.25Gi" : "550Mi" + cpu = local.infra.production ? "250m" : "150m" + memory = local.infra.production ? "3Gi" : "500Mi" } } @@ -150,7 +150,7 @@ resource "kubernetes_deployment" "app" { initial_delay_seconds = 3 timeout_seconds = 5 period_seconds = 1 - success_threshold = 3 + success_threshold = 10 failure_threshold = 1 }