From 6fb537e8012808314c1885b7c980dc4d554452bf Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 19 Jul 2023 14:37:08 +1000 Subject: [PATCH 01/38] feat: migrate postdeployment to data http --- app/init/Dockerfile | 17 +++ app/init/README.md | 10 ++ app/init/init-execute.sh | 56 +++++++ app/init/init-image.cloudbuild.yaml | 36 +++++ app/placeholder/Dockerfile | 2 +- app/placeholder/placeholder-deploy.sh | 23 ++- infra/containers.tf | 3 +- infra/iam.tf | 14 +- infra/jobs.tf | 29 ---- infra/postdeployment.tf | 202 +++++++++++++------------- 10 files changed, 253 insertions(+), 139 deletions(-) create mode 100644 app/init/Dockerfile create mode 100644 app/init/README.md create mode 100755 app/init/init-execute.sh create mode 100644 app/init/init-image.cloudbuild.yaml diff --git a/app/init/Dockerfile b/app/init/Dockerfile new file mode 100644 index 00000000..bbe83037 --- /dev/null +++ b/app/init/Dockerfile @@ -0,0 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/google.com/cloudsdktool/cloud-sdk:slim +COPY init-execute.sh . +ENTRYPOINT ./init-execute.sh diff --git a/app/init/README.md b/app/init/README.md new file mode 100644 index 00000000..0082b0df --- /dev/null +++ b/app/init/README.md @@ -0,0 +1,10 @@ +# Init + +This folder contains the source for a container image that runs the tasks required to setup the application on first deployment. + +The container is designed to be executed as a Cloud Run job, with the `roles/run.developer` role, to run the `init-execute.sh` script: + + * execute the `setup` job, [primes the the database](https://github.com/GoogleCloudPlatform/avocano/blob/main/server/scripts/prime_database.sh) script + * execute the `client` job, that runs the [client deployment](https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh) + * purges cache and warms API. + diff --git a/app/init/init-execute.sh b/app/init/init-execute.sh new file mode 100755 index 00000000..ed2f9bf2 --- /dev/null +++ b/app/init/init-execute.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to assist in Dockerfile-based deployments. +# any errors? exit immediately. +set -e + +# escape if project_id not defined (mandatory, required later) +if [[ -z $PROJECT_ID ]]; then + echo "PROJECT_ID not defined. Cannot deploy. Exiting." + exit 1 +fi + +# escape if firebase_url not defined (mandatory, required later) +if [[ -z $FIREBASE_URL ]]; then + echo "FIREBASE_URL not defined. Cannot deploy. Exiting." + exit 1 +fi + +# Define common defaults, all overrideable. +REGION="${REGION:-us-central1}" +SETUP_JOB="${SETUP_JOB:-setup}" +CLIENT_JOB="${CLIENT_JOB:-client}" + +echo "*** Executing initization job ***" +echo "PROJECT_ID: $PROJECT_ID" +echo "REGION: $REGION" +echo "SETUP JOB: $SETUP_JOB" +echo "CLIENT JOB: $CLIENT_JOB" +echo "FIREBASE URL: $FIREBASE_URL" +echo "SERVER URL: $SERVER_URL" +echo "" + +echo "Running init database migration..." +gcloud run jobs execute $SETUP_JOB --wait --project $PROJECT_ID --region $REGION + +echo "Running client deploy..." +gcloud run jobs execute $CLIENT_JOB --wait --project $PROJECT_ID --region $REGION + +echo "Purge Firebase cache" +echo curl -X PURGE "${FIREBASE_URL}/" + +echo "Warm up API" +curl "${SERVER_URL}/api/products/?warmup" diff --git a/app/init/init-image.cloudbuild.yaml b/app/init/init-image.cloudbuild.yaml new file mode 100644 index 00000000..7b854213 --- /dev/null +++ b/app/init/init-image.cloudbuild.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build placeholder site code into container + +steps: + - id: build + name: "gcr.io/cloud-builders/docker" + dir: app/init + args: + [ + "build", + "-t", + "gcr.io/$PROJECT_ID/$_IMAGE_NAME", + ".", + ] + +images: + - gcr.io/$PROJECT_ID/$_IMAGE_NAME + +substitutions: + _IMAGE_NAME: avocano-init + +options: + dynamic_substitutions: true diff --git a/app/placeholder/Dockerfile b/app/placeholder/Dockerfile index 81870466..5bc8abe6 100644 --- a/app/placeholder/Dockerfile +++ b/app/placeholder/Dockerfile @@ -17,7 +17,7 @@ ARG PROJECT_ID=YOURPROJECTID FROM gcr.io/$PROJECT_ID/firebase -RUN apk add gettext +RUN apk add gettext curl RUN npm install -g json COPY . ./ diff --git a/app/placeholder/placeholder-deploy.sh b/app/placeholder/placeholder-deploy.sh index c3ce9941..09005ada 100755 --- a/app/placeholder/placeholder-deploy.sh +++ b/app/placeholder/placeholder-deploy.sh @@ -17,6 +17,21 @@ # any errors? exit immediately. set -e +# escape if firebase_url not defined (mandatory, required later) +if [[ -z $FIREBASE_URL ]]; then + echo "FIREBASE_URL not defined. Cannot deploy. Exiting." + exit 1 +fi + +# Only run the placeholder script if the site has been deployed before. +# Check if the firebase url has "Site Not Found" (the pre-deployment state) +if curl $FIREBASE_URL | grep -q "Site Not Found"; then + echo "Firebase site $FIREBASE_URL hasn't been deployed before, so it needs a placeholder." +else + echo "Firebase site $FIREBASE_URL has been deployed before. Not going to deploy placeholder. Exiting." + exit 0 +fi + # if deploying with a suffix (from infra/jobs.tf), adjust the config to suit the custom site # https://firebase.google.com/docs/hosting/multisites#set_up_deploy_targets if [[ -n $SUFFIX ]]; then @@ -24,7 +39,7 @@ if [[ -n $SUFFIX ]]; then UPDATED=true # Use template file to generate configuration - envsubst < firebaserc.tmpl > .firebaserc + envsubst .firebaserc echo "Customised .firebaserc created to support site." cat .firebaserc fi @@ -35,6 +50,10 @@ if [[ -n $UPDATED ]]; then cat firebase.json fi +# Finally, deploy the application echo "Deploying placeholder to Firebase..." - firebase deploy --project "$PROJECT_ID" --only hosting + +# Setup for greater chances of success by explicitly purging cache +echo "Purging firebase cache" +curl -X PURGE "${FIREBASE_URL}/" diff --git a/infra/containers.tf b/infra/containers.tf index 1fefdade..92056d66 100644 --- a/infra/containers.tf +++ b/infra/containers.tf @@ -19,5 +19,6 @@ locals { server_image = "gcr.io/${var.server_image_host}/server:${var.image_version}" client_image = "gcr.io/${var.client_image_host}/client:${var.image_version}" - placeholder_image = "gcr.io/hsa-public/avocano-placeholder:latest" + placeholder_image = "gcr.io/hsa-public/avocano-placeholder:postjsscurl" # TODO(glasnt): revert to tag "latest" + init_image = "gcr.io/hsa-public/avocano-init:postjsscurl" # TODO(glasnt): revert to tag "latest" } diff --git a/infra/iam.tf b/infra/iam.tf index 2513e9eb..7dd2f0fd 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -41,9 +41,9 @@ resource "google_service_account" "automation" { depends_on = [module.project_services] } -resource "google_service_account" "compute" { - account_id = var.random_suffix ? "compute-startup-${random_id.suffix.hex}" : "compute-startup" - display_name = "Head Start App Compute Instance SA" +resource "google_service_account" "init" { + account_id = var.random_suffix ? "init-startup-${random_id.suffix.hex}" : "init-startup" + display_name = "Jump Start App Init SA" depends_on = [module.project_services] count = var.init ? 1 : 0 } @@ -88,12 +88,12 @@ resource "google_project_iam_member" "client_permissions" { depends_on = [google_service_account.client] } -# GCE instance needs access to start Jobs -resource "google_project_iam_member" "computestartup_permissions" { +# Init process needs access to start Jobs +resource "google_project_iam_member" "initstartup_permissions" { project = var.project_id role = "roles/run.developer" - member = "serviceAccount:${google_service_account.compute[0].email}" - depends_on = [google_service_account.compute] + member = "serviceAccount:${google_service_account.init[0].email}" + depends_on = [google_service_account.init] count = var.init ? 1 : 0 } diff --git a/infra/jobs.tf b/infra/jobs.tf index 74583cdc..bed42a35 100644 --- a/infra/jobs.tf +++ b/infra/jobs.tf @@ -144,32 +144,3 @@ resource "google_cloud_run_v2_job" "client" { ] } - -resource "google_cloud_run_v2_job" "placeholder" { - name = var.random_suffix ? "placeholder-${random_id.suffix.hex}" : "placeholder" - location = var.region - - labels = var.labels - - template { - template { - service_account = google_service_account.client.email - containers { - image = local.placeholder_image - env { - name = "PROJECT_ID" - value = var.project_id - } - - env { - name = "SUFFIX" - value = var.random_suffix ? random_id.suffix.hex : "" - } - } - } - } - - depends_on = [ - module.project_services - ] -} diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 0ca6b1dd..3a181276 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -14,126 +14,130 @@ * limitations under the License. */ -resource "google_compute_network" "gce_init" { - count = var.init ? 1 : 0 - - name = var.random_suffix ? "gce-init-network-${random_id.suffix.hex}" : "gce-init-network" - auto_create_subnetworks = false - routing_mode = "GLOBAL" - project = var.project_id - delete_default_routes_on_create = false - mtu = 0 - depends_on = [module.project_services] +# used to collect access token, for authenticated POST commands +data "google_client_config" "current" { } -resource "google_compute_subnetwork" "gce_init" { - count = var.init ? 1 : 0 - - name = var.random_suffix ? "subnet-gce-init-${random_id.suffix.hex}" : "subnet-gce-init" - network = google_compute_network.gce_init[0].id - ip_cidr_range = "10.10.10.0/24" - region = var.region +# Job that uses pre-built docker image to deploy a placeholder website. +resource "google_cloud_run_v2_job" "placeholder" { + name = var.random_suffix ? "placeholder-${random_id.suffix.hex}" : "placeholder" + location = var.region + + labels = var.labels + + template { + template { + service_account = google_service_account.client.email + max_retries = 1 + containers { + image = local.placeholder_image + + # Variables consumed by /app/placeholder/placeholder-deploy.sh + env { + name = "PROJECT_ID" + value = var.project_id + } + env { + name = "SUFFIX" + value = var.random_suffix ? random_id.suffix.hex : "" + } + env { + name = "FIREBASE_URL" + value = local.firebase_url + } + } + } + } - depends_on = [module.project_services] + depends_on = [ + module.project_services + ] } -resource "google_compute_instance" "gce_init" { - count = var.init ? 1 : 0 + +# execute the job by calling the API directly. +data "http" "execute_placeholder_job" { + url = "https://${var.region}-run.googleapis.com/v2/projects/${var.project_id}/locations/${var.region}/jobs/${google_cloud_run_v2_job.placeholder.name}:run" + method = "POST" + request_headers = { + Accept = "application/json" + Authorization = "Bearer ${data.google_client_config.current.access_token}" } depends_on = [ module.project_services, - google_sql_database_instance.postgres, - google_cloud_run_v2_job.setup, - google_cloud_run_v2_job.client, + google_cloud_run_v2_job.placeholder, ] +} - name = var.random_suffix ? "head-start-initialize-${random_id.suffix.hex}" : "head-start-initialize" - machine_type = "n1-standard-1" - zone = var.zone - desired_status = "RUNNING" # https://github.com/GoogleCloudPlatform/terraform-dynamic-python-webapp/pull/75#issuecomment-1547198414 - - allow_stopping_for_update = true - - boot_disk { - initialize_params { - image = "debian-cloud/debian-11" - } - } - - network_interface { - network = google_compute_network.gce_init[0].self_link - subnetwork = google_compute_subnetwork.gce_init[0].self_link - access_config { - // Ephemeral public IP +resource "google_cloud_run_v2_job" "init" { + name = var.random_suffix ? "init-${random_id.suffix.hex}" : "init" + location = var.region + + labels = var.labels + + template { + template { + service_account = google_service_account.init[0].email + max_retries = 1 + containers { + image = local.init_image + + # Variables consumed by /app/init/init-execute.sh + env { + name = "PROJECT_ID" + value = var.project_id + } + env { + name = "SUFFIX" + value = var.random_suffix ? random_id.suffix.hex : "" + } + env { + name = "REGION" + value = var.region + } + env { + name = "SETUP_JOB" + value = google_cloud_run_v2_job.setup.name + } + env { + name = "CLIENT_JOB" + value = google_cloud_run_v2_job.client.name + } + env { + name = "FIREBASE_URL" + value = local.firebase_url + } + env { + name = "SERVER_URL" + value = google_cloud_run_v2_service.server.uri + } + + } } } - service_account { - email = google_service_account.compute[0].email - scopes = ["cloud-platform"] # TODO: Restrict?? - } - - metadata_startup_script = < Date: Wed, 19 Jul 2023 15:11:20 +1000 Subject: [PATCH 02/38] lint --- app/init/README.md | 6 +++--- app/init/init-execute.sh | 4 ++-- app/placeholder/placeholder-deploy.sh | 2 +- infra/containers.tf | 2 +- infra/postdeployment.tf | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/init/README.md b/app/init/README.md index 0082b0df..72de895a 100644 --- a/app/init/README.md +++ b/app/init/README.md @@ -1,10 +1,10 @@ # Init -This folder contains the source for a container image that runs the tasks required to setup the application on first deployment. +This folder contains the source for a container image that runs the tasks required to setup the application on first deployment. -The container is designed to be executed as a Cloud Run job, with the `roles/run.developer` role, to run the `init-execute.sh` script: +The container is designed to be executed as a Cloud Run job, with the `roles/run.developer` role, to run the `init-execute.sh` script: * execute the `setup` job, [primes the the database](https://github.com/GoogleCloudPlatform/avocano/blob/main/server/scripts/prime_database.sh) script * execute the `client` job, that runs the [client deployment](https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh) - * purges cache and warms API. + * purges cache and warms API. diff --git a/app/init/init-execute.sh b/app/init/init-execute.sh index ed2f9bf2..6c99e9dc 100755 --- a/app/init/init-execute.sh +++ b/app/init/init-execute.sh @@ -44,10 +44,10 @@ echo "SERVER URL: $SERVER_URL" echo "" echo "Running init database migration..." -gcloud run jobs execute $SETUP_JOB --wait --project $PROJECT_ID --region $REGION +gcloud run jobs execute "$SETUP_JOB" --wait --project "$PROJECT_ID" --region "$REGION" echo "Running client deploy..." -gcloud run jobs execute $CLIENT_JOB --wait --project $PROJECT_ID --region $REGION +gcloud run jobs execute "$CLIENT_JOB" --wait --project "$PROJECT_ID" --region "$REGION" echo "Purge Firebase cache" echo curl -X PURGE "${FIREBASE_URL}/" diff --git a/app/placeholder/placeholder-deploy.sh b/app/placeholder/placeholder-deploy.sh index 09005ada..46cc3118 100755 --- a/app/placeholder/placeholder-deploy.sh +++ b/app/placeholder/placeholder-deploy.sh @@ -25,7 +25,7 @@ fi # Only run the placeholder script if the site has been deployed before. # Check if the firebase url has "Site Not Found" (the pre-deployment state) -if curl $FIREBASE_URL | grep -q "Site Not Found"; then +if curl "$FIREBASE_URL" | grep -q "Site Not Found"; then echo "Firebase site $FIREBASE_URL hasn't been deployed before, so it needs a placeholder." else echo "Firebase site $FIREBASE_URL has been deployed before. Not going to deploy placeholder. Exiting." diff --git a/infra/containers.tf b/infra/containers.tf index 92056d66..ee70766c 100644 --- a/infra/containers.tf +++ b/infra/containers.tf @@ -20,5 +20,5 @@ locals { server_image = "gcr.io/${var.server_image_host}/server:${var.image_version}" client_image = "gcr.io/${var.client_image_host}/client:${var.image_version}" placeholder_image = "gcr.io/hsa-public/avocano-placeholder:postjsscurl" # TODO(glasnt): revert to tag "latest" - init_image = "gcr.io/hsa-public/avocano-init:postjsscurl" # TODO(glasnt): revert to tag "latest" + init_image = "gcr.io/hsa-public/avocano-init:postjsscurl" # TODO(glasnt): revert to tag "latest" } diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 3a181276..b3ee3ba2 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -19,7 +19,7 @@ data "google_client_config" "current" { } -# Job that uses pre-built docker image to deploy a placeholder website. +# Job that uses pre-built docker image to deploy a placeholder website. resource "google_cloud_run_v2_job" "placeholder" { name = var.random_suffix ? "placeholder-${random_id.suffix.hex}" : "placeholder" location = var.region @@ -56,7 +56,7 @@ resource "google_cloud_run_v2_job" "placeholder" { } -# execute the job by calling the API directly. +# execute the job by calling the API directly. data "http" "execute_placeholder_job" { url = "https://${var.region}-run.googleapis.com/v2/projects/${var.project_id}/locations/${var.region}/jobs/${google_cloud_run_v2_job.placeholder.name}:run" method = "POST" @@ -124,7 +124,7 @@ resource "google_cloud_run_v2_job" "init" { } -# execute the job, once it and other dependencies exit. +# execute the job, once it and other dependencies exit. data "http" "execute_init_job" { count = var.init ? 1 : 0 From 2e552502dc0301a12a343bfe13e2c7fad5cc7144 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 19 Jul 2023 16:02:51 +1000 Subject: [PATCH 03/38] lint: add ignore to side-effect 'unused' data sources --- infra/postdeployment.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index b3ee3ba2..adc8bd9e 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -56,7 +56,8 @@ resource "google_cloud_run_v2_job" "placeholder" { } -# execute the job by calling the API directly. +# execute the job by calling the API directly. Intended side-effect +# tflint-ignore: terraform_unused_declarations data "http" "execute_placeholder_job" { url = "https://${var.region}-run.googleapis.com/v2/projects/${var.project_id}/locations/${var.region}/jobs/${google_cloud_run_v2_job.placeholder.name}:run" method = "POST" @@ -124,7 +125,8 @@ resource "google_cloud_run_v2_job" "init" { } -# execute the job, once it and other dependencies exit. +# execute the job, once it and other dependencies exit. Intended side-effect. +# tflint-ignore: terraform_unused_declarations data "http" "execute_init_job" { count = var.init ? 1 : 0 From 03741e2d6e3884d622655ff75492385dea2b50e7 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 20 Jul 2023 17:19:31 +1000 Subject: [PATCH 04/38] WIP NO MERGE: progress on cloudbuild-based deployments --- app/init/init-execute.sh | 2 +- app/placeholder/Dockerfile | 6 +- infra/iam.tf | 30 ++++- infra/postdeployment.tf | 183 ++++++++++++++---------------- infra/server-step.cloudbuild.yaml | 51 +++++++++ 5 files changed, 172 insertions(+), 100 deletions(-) create mode 100644 infra/server-step.cloudbuild.yaml diff --git a/app/init/init-execute.sh b/app/init/init-execute.sh index 6c99e9dc..267d057a 100755 --- a/app/init/init-execute.sh +++ b/app/init/init-execute.sh @@ -50,7 +50,7 @@ echo "Running client deploy..." gcloud run jobs execute "$CLIENT_JOB" --wait --project "$PROJECT_ID" --region "$REGION" echo "Purge Firebase cache" -echo curl -X PURGE "${FIREBASE_URL}/" +curl -X PURGE "${FIREBASE_URL}/" echo "Warm up API" curl "${SERVER_URL}/api/products/?warmup" diff --git a/app/placeholder/Dockerfile b/app/placeholder/Dockerfile index 5bc8abe6..a64436d3 100644 --- a/app/placeholder/Dockerfile +++ b/app/placeholder/Dockerfile @@ -15,10 +15,12 @@ # Execute with "docker run --build-arg PROJECT_ID=$PROJECT_ID ..." ARG PROJECT_ID=YOURPROJECTID -FROM gcr.io/$PROJECT_ID/firebase +#FROM gcr.io/$PROJECT_ID/firebase # TODO(glasnt): revert +FROM gcr.io/hsa-public/firebase +WORKDIR /app RUN apk add gettext curl RUN npm install -g json COPY . ./ -ENTRYPOINT ./placeholder-deploy.sh +ENTRYPOINT ["/app/placeholder-deploy.sh"] diff --git a/infra/iam.tf b/infra/iam.tf index 7dd2f0fd..ff59b733 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -21,6 +21,7 @@ locals { automation_SA = "serviceAccount:${google_service_account.automation.email}" server_SA = "serviceAccount:${google_service_account.server.email}" client_SA = "serviceAccount:${google_service_account.client.email}" + init_SA = "serviceAccount:${google_service_account.init[0].email}" } resource "google_service_account" "server" { @@ -88,11 +89,38 @@ resource "google_project_iam_member" "client_permissions" { depends_on = [google_service_account.client] } + +locals { + cloudbuild_roles = ["roles/logging.logWriter","roles/cloudbuild.builds.builder", + "roles/iam.serviceAccountUser","roles/run.developer"] +} +# client account needs permissions to invoke cloud build triggers +resource "google_project_iam_member" "client_cloudbuild" { + project = var.project_id + + for_each = toset(local.cloudbuild_roles) + + role = each.key + member = local.client_SA + depends_on = [google_service_account.client] +} + +# init account needs permissions to invoke cloud build triggers +resource "google_project_iam_member" "init_cloudbuild" { + project = var.project_id + + for_each = toset(local.cloudbuild_roles) + + role = each.key + member = local.init_SA + depends_on = [google_service_account.client] +} + # Init process needs access to start Jobs resource "google_project_iam_member" "initstartup_permissions" { project = var.project_id role = "roles/run.developer" - member = "serviceAccount:${google_service_account.init[0].email}" + member = local.init_SA depends_on = [google_service_account.init] count = var.init ? 1 : 0 } diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index adc8bd9e..da5c18a6 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -19,127 +19,118 @@ data "google_client_config" "current" { } -# Job that uses pre-built docker image to deploy a placeholder website. -resource "google_cloud_run_v2_job" "placeholder" { - name = var.random_suffix ? "placeholder-${random_id.suffix.hex}" : "placeholder" - location = var.region - - labels = var.labels - - template { - template { - service_account = google_service_account.client.email - max_retries = 1 - containers { - image = local.placeholder_image - - # Variables consumed by /app/placeholder/placeholder-deploy.sh - env { - name = "PROJECT_ID" - value = var.project_id - } - env { - name = "SUFFIX" - value = var.random_suffix ? random_id.suffix.hex : "" - } - env { - name = "FIREBASE_URL" - value = local.firebase_url - } - } - } +# topic that is never used in practice, except as a configuration type for Cloud Build Triggers +resource "google_pubsub_topic" "faux" { + name = "faux-topic" +} + +## Placeholder - deploys a placeholder website - uses prebuilt image in /app/placeholder +locals { + random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" +} + +resource "google_cloudbuild_trigger" "placeholder" { + name = "placeholder" + location = "us-central1" + + description = "Deploy a placeholder Firebase website" + + service_account = google_service_account.client.id + + pubsub_config { + topic = google_pubsub_topic.faux.id } - depends_on = [ - module.project_services - ] + build { + step { + name = "gcr.io/${var.project_id}/placeholder" # TODO(glasnt) revert when fixed local.placeholder_image + env = [ + "PROJECT_ID=${var.project_id}", + "SUFFIX=${local.random_suffix_value}", + "FIREBASE_URL=${local.firebase_url}", + ] + } + + options { + logging = "CLOUD_LOGGING_ONLY" + } + } } -# execute the job by calling the API directly. Intended side-effect +# execute the trigger, once it and other dependencies exist. Intended side-effect. # tflint-ignore: terraform_unused_declarations -data "http" "execute_placeholder_job" { - url = "https://${var.region}-run.googleapis.com/v2/projects/${var.project_id}/locations/${var.region}/jobs/${google_cloud_run_v2_job.placeholder.name}:run" +data "http" "execute_placeholder_trigger" { + count = var.init ? 1 : 0 + + url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.placeholder.id}:run" method = "POST" request_headers = { - Accept = "application/json" - Authorization = "Bearer ${data.google_client_config.current.access_token}" } - + Authorization = "Bearer ${data.google_client_config.current.access_token}" + } depends_on = [ module.project_services, - google_cloud_run_v2_job.placeholder, + google_cloudbuild_trigger.placeholder ] } -resource "google_cloud_run_v2_job" "init" { - name = var.random_suffix ? "init-${random_id.suffix.hex}" : "init" - location = var.region - - labels = var.labels - - template { - template { - service_account = google_service_account.init[0].email - max_retries = 1 - containers { - image = local.init_image - - # Variables consumed by /app/init/init-execute.sh - env { - name = "PROJECT_ID" - value = var.project_id - } - env { - name = "SUFFIX" - value = var.random_suffix ? random_id.suffix.hex : "" - } - env { - name = "REGION" - value = var.region - } - env { - name = "SETUP_JOB" - value = google_cloud_run_v2_job.setup.name - } - env { - name = "CLIENT_JOB" - value = google_cloud_run_v2_job.client.name - } - env { - name = "FIREBASE_URL" - value = local.firebase_url - } - env { - name = "SERVER_URL" - value = google_cloud_run_v2_service.server.uri - } - - } - } +## Initalization trigger +resource "google_cloudbuild_trigger" "init" { + name = "init-application" + location = "us-central1" + + description = "Perform initialization setup for server and client" + + pubsub_config { + topic = google_pubsub_topic.faux.id } - depends_on = [ - module.project_services - ] -} + service_account = google_service_account.init[0].id + + build { + #step { + # id = "server-setup" + # name = local.server_image + # entrypoint = "setup" + #} + step { + id = "client-setup" + name = local.client_image + env = [ + "PROJECT_ID=${var.project_id}", + "SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""}", + "REGION=${var.region}", + "FIREBASE_URL=${local.firebase_url}", + ] + } + step { + id = "purge-firebase" + name = "gcr.io/distroless/static-debian11" + script = "curl -X PURGE \"${local.firebase_url}/\"" + } + step { + id = "warmup-api" + name = "gcr.io/distroless/static-debian11" + script = "curl \"${google_cloud_run_v2_service.server.uri}/api/products/?warmup\"" + } + } +} -# execute the job, once it and other dependencies exit. Intended side-effect. +# execute the trigger, once it and other dependencies exist. Intended side-effect. # tflint-ignore: terraform_unused_declarations -data "http" "execute_init_job" { +data "http" "execute_init_trigger" { count = var.init ? 1 : 0 - url = "https://${var.region}-run.googleapis.com/v2/projects/${var.project_id}/locations/${var.region}/jobs/${google_cloud_run_v2_job.init.name}:run" + url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.init.id}:run" method = "POST" request_headers = { - Accept = "application/json" - Authorization = "Bearer ${data.google_client_config.current.access_token}" } + Authorization = "Bearer ${data.google_client_config.current.access_token}" + } depends_on = [ - google_cloud_run_v2_job.init, - google_cloud_run_v2_job.setup, - google_cloud_run_v2_job.client, module.project_services, google_sql_database_instance.postgres, + google_cloudbuild_trigger.init, ] } diff --git a/infra/server-step.cloudbuild.yaml b/infra/server-step.cloudbuild.yaml new file mode 100644 index 00000000..7a312b4f --- /dev/null +++ b/infra/server-step.cloudbuild.yaml @@ -0,0 +1,51 @@ +## Iterating on what the server setup would look like +## view to include in google_cloudbuild_trigger.init + +steps: + + - id: "docker-layer" + name: "gcr.io/cloud-builders/docker" + entrypoint: /bin/bash + args: + - '-c' + - | + echo "FROM $_SERVER_NAME + COPY --from=gcr.io/cloud-sql-connectors/cloud-sql-proxy /cloud-sql-proxy /cloudsql/cloud-sql-proxy" > Dockerfile-proxy; + + docker build -f Dockerfile-proxy -t $_PROXY_NAME . + + - id: "migrate" + name: $_PROXY_NAME + secretEnv: + - DJANGO_ENV + - DJANGO_SUPERUSER_PASSWORD + env: + - USE_CLOUD_SQL_AUTH_PROXY=true + entrypoint: /bin/bash + args: + - -c + - | + /cloudsql/cloud-sql-proxy ${_INSTANCE_CONNECTION_NAME} & sleep 2 + find / -name prime_database.sh + bash /workspace/scripts/prime_database.sh + + +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/django_settings-${_SUFFIX}/versions/latest + env: DJANGO_ENV + + - versionName: projects/$PROJECT_ID/secrets/django_admin_password-${_SUFFIX}/versions/latest + env: DJANGO_SUPERUSER_PASSWORD + +options: + dynamic_substitutions: true + #logging: CLOUD_LOGGING_ONLY + +#serviceAccount: projects/${PROJECT_ID}/serviceAccounts/automation-6f21@glasnt-postjsscurl-6728.iam.gserviceaccount.com + +substitutions: + _SERVER_NAME: gcr.io/hsa-public/terraform-python-dynamic-webapp/server:v1.8.2 + _PROXY_NAME: gcr.io/${PROJECT_ID}/server-proxy + _SUFFIX: "6f21" + _INSTANCE_CONNECTION_NAME: glasnt-postjsscurl-6728:us-central1:psql-6f21 \ No newline at end of file From 751f8c3c28e87be7b4d99c58d1dd4ed1d3fe0d72 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 10:51:00 +1000 Subject: [PATCH 05/38] WIP save --- infra/iam.tf | 1 + infra/jobs.tf | 52 ++---------------------------- infra/postdeployment.tf | 71 +++++++++++++++++++++++++++++++---------- infra/secrets.tf | 8 ++++- 4 files changed, 65 insertions(+), 67 deletions(-) diff --git a/infra/iam.tf b/infra/iam.tf index ff59b733..80b52bb0 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -94,6 +94,7 @@ locals { cloudbuild_roles = ["roles/logging.logWriter","roles/cloudbuild.builds.builder", "roles/iam.serviceAccountUser","roles/run.developer"] } + # client account needs permissions to invoke cloud build triggers resource "google_project_iam_member" "client_cloudbuild" { project = var.project_id diff --git a/infra/jobs.tf b/infra/jobs.tf index bed42a35..f9ee638f 100644 --- a/infra/jobs.tf +++ b/infra/jobs.tf @@ -14,54 +14,7 @@ * limitations under the License. */ -resource "google_cloud_run_v2_job" "setup" { - name = var.random_suffix ? "setup-${random_id.suffix.hex}" : "setup" - location = var.region - - labels = var.labels - - template { - template { - service_account = google_service_account.automation.email - containers { - image = local.server_image - command = ["setup"] - env { - name = "DJANGO_ENV" - value_source { - secret_key_ref { - secret = google_secret_manager_secret.django_settings.secret_id - version = "latest" - } - } - } - env { - name = "DJANGO_SUPERUSER_PASSWORD" - value_source { - secret_key_ref { - secret = google_secret_manager_secret.django_admin_password.secret_id - version = "latest" - } - } - } - volume_mounts { - name = "cloudsql" - mount_path = "/cloudsql" - } - } - volumes { - name = "cloudsql" - cloud_sql_instance { - instances = [google_sql_database_instance.postgres.connection_name] - } - } - } - } - depends_on = [ - google_secret_manager_secret_version.django_settings - ] -} - +# Job to apply server database updates resource "google_cloud_run_v2_job" "migrate" { name = var.random_suffix ? "migrate-${random_id.suffix.hex}" : "migrate" location = var.region @@ -101,8 +54,7 @@ resource "google_cloud_run_v2_job" "migrate" { ] } - - +# Job to apply client application updates resource "google_cloud_run_v2_job" "client" { name = var.random_suffix ? "client-${random_id.suffix.hex}" : "client" diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index da5c18a6..79d84587 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -14,23 +14,28 @@ * limitations under the License. */ +locals { + random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" # literal value (NNNN) + random_suffix_append = var.random_suffix ? "-${random_id.suffix.hex}" : "" # appended value (-NNNN) + + setup_job_name = "setup${local.random_suffix_append}" + + gcloud_step_container = "gcr.io/google.com/cloudsdktool/cloud-sdk:slim" + curl_step_container = "gcr.io/distroless/static-debian11" +} # used to collect access token, for authenticated POST commands data "google_client_config" "current" { } -# topic that is never used in practice, except as a configuration type for Cloud Build Triggers +# topic that is never used, except as a configuration type for Cloud Build Triggers resource "google_pubsub_topic" "faux" { - name = "faux-topic" + name = "faux-topic${local.random_suffix_append}" } ## Placeholder - deploys a placeholder website - uses prebuilt image in /app/placeholder -locals { - random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" -} - resource "google_cloudbuild_trigger" "placeholder" { - name = "placeholder" + name = "placeholder${local.random_suffix_append}" location = "us-central1" description = "Deploy a placeholder Firebase website" @@ -77,8 +82,8 @@ data "http" "execute_placeholder_trigger" { ## Initalization trigger resource "google_cloudbuild_trigger" "init" { - name = "init-application" - location = "us-central1" + name = "init-application${local.random_suffix_append}" + location = var.region description = "Perform initialization setup for server and client" @@ -89,11 +94,38 @@ resource "google_cloudbuild_trigger" "init" { service_account = google_service_account.init[0].id build { - #step { - # id = "server-setup" - # name = local.server_image - # entrypoint = "setup" - #} + step { + # Check if a job already exists under the exact name. If it doesn't, create it. + id = "create-setup-job" + name = local.gcloud_step_container + script = < Date: Fri, 21 Jul 2023 11:01:01 +1000 Subject: [PATCH 06/38] remove unused code --- app/init/Dockerfile | 17 --------- app/init/README.md | 10 ------ app/init/init-execute.sh | 56 ----------------------------- app/init/init-image.cloudbuild.yaml | 36 ------------------- infra/server-step.cloudbuild.yaml | 51 -------------------------- 5 files changed, 170 deletions(-) delete mode 100644 app/init/Dockerfile delete mode 100644 app/init/README.md delete mode 100755 app/init/init-execute.sh delete mode 100644 app/init/init-image.cloudbuild.yaml delete mode 100644 infra/server-step.cloudbuild.yaml diff --git a/app/init/Dockerfile b/app/init/Dockerfile deleted file mode 100644 index bbe83037..00000000 --- a/app/init/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM gcr.io/google.com/cloudsdktool/cloud-sdk:slim -COPY init-execute.sh . -ENTRYPOINT ./init-execute.sh diff --git a/app/init/README.md b/app/init/README.md deleted file mode 100644 index 72de895a..00000000 --- a/app/init/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Init - -This folder contains the source for a container image that runs the tasks required to setup the application on first deployment. - -The container is designed to be executed as a Cloud Run job, with the `roles/run.developer` role, to run the `init-execute.sh` script: - - * execute the `setup` job, [primes the the database](https://github.com/GoogleCloudPlatform/avocano/blob/main/server/scripts/prime_database.sh) script - * execute the `client` job, that runs the [client deployment](https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh) - * purges cache and warms API. - diff --git a/app/init/init-execute.sh b/app/init/init-execute.sh deleted file mode 100755 index 267d057a..00000000 --- a/app/init/init-execute.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script to assist in Dockerfile-based deployments. -# any errors? exit immediately. -set -e - -# escape if project_id not defined (mandatory, required later) -if [[ -z $PROJECT_ID ]]; then - echo "PROJECT_ID not defined. Cannot deploy. Exiting." - exit 1 -fi - -# escape if firebase_url not defined (mandatory, required later) -if [[ -z $FIREBASE_URL ]]; then - echo "FIREBASE_URL not defined. Cannot deploy. Exiting." - exit 1 -fi - -# Define common defaults, all overrideable. -REGION="${REGION:-us-central1}" -SETUP_JOB="${SETUP_JOB:-setup}" -CLIENT_JOB="${CLIENT_JOB:-client}" - -echo "*** Executing initization job ***" -echo "PROJECT_ID: $PROJECT_ID" -echo "REGION: $REGION" -echo "SETUP JOB: $SETUP_JOB" -echo "CLIENT JOB: $CLIENT_JOB" -echo "FIREBASE URL: $FIREBASE_URL" -echo "SERVER URL: $SERVER_URL" -echo "" - -echo "Running init database migration..." -gcloud run jobs execute "$SETUP_JOB" --wait --project "$PROJECT_ID" --region "$REGION" - -echo "Running client deploy..." -gcloud run jobs execute "$CLIENT_JOB" --wait --project "$PROJECT_ID" --region "$REGION" - -echo "Purge Firebase cache" -curl -X PURGE "${FIREBASE_URL}/" - -echo "Warm up API" -curl "${SERVER_URL}/api/products/?warmup" diff --git a/app/init/init-image.cloudbuild.yaml b/app/init/init-image.cloudbuild.yaml deleted file mode 100644 index 7b854213..00000000 --- a/app/init/init-image.cloudbuild.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build placeholder site code into container - -steps: - - id: build - name: "gcr.io/cloud-builders/docker" - dir: app/init - args: - [ - "build", - "-t", - "gcr.io/$PROJECT_ID/$_IMAGE_NAME", - ".", - ] - -images: - - gcr.io/$PROJECT_ID/$_IMAGE_NAME - -substitutions: - _IMAGE_NAME: avocano-init - -options: - dynamic_substitutions: true diff --git a/infra/server-step.cloudbuild.yaml b/infra/server-step.cloudbuild.yaml deleted file mode 100644 index 7a312b4f..00000000 --- a/infra/server-step.cloudbuild.yaml +++ /dev/null @@ -1,51 +0,0 @@ -## Iterating on what the server setup would look like -## view to include in google_cloudbuild_trigger.init - -steps: - - - id: "docker-layer" - name: "gcr.io/cloud-builders/docker" - entrypoint: /bin/bash - args: - - '-c' - - | - echo "FROM $_SERVER_NAME - COPY --from=gcr.io/cloud-sql-connectors/cloud-sql-proxy /cloud-sql-proxy /cloudsql/cloud-sql-proxy" > Dockerfile-proxy; - - docker build -f Dockerfile-proxy -t $_PROXY_NAME . - - - id: "migrate" - name: $_PROXY_NAME - secretEnv: - - DJANGO_ENV - - DJANGO_SUPERUSER_PASSWORD - env: - - USE_CLOUD_SQL_AUTH_PROXY=true - entrypoint: /bin/bash - args: - - -c - - | - /cloudsql/cloud-sql-proxy ${_INSTANCE_CONNECTION_NAME} & sleep 2 - find / -name prime_database.sh - bash /workspace/scripts/prime_database.sh - - -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/django_settings-${_SUFFIX}/versions/latest - env: DJANGO_ENV - - - versionName: projects/$PROJECT_ID/secrets/django_admin_password-${_SUFFIX}/versions/latest - env: DJANGO_SUPERUSER_PASSWORD - -options: - dynamic_substitutions: true - #logging: CLOUD_LOGGING_ONLY - -#serviceAccount: projects/${PROJECT_ID}/serviceAccounts/automation-6f21@glasnt-postjsscurl-6728.iam.gserviceaccount.com - -substitutions: - _SERVER_NAME: gcr.io/hsa-public/terraform-python-dynamic-webapp/server:v1.8.2 - _PROXY_NAME: gcr.io/${PROJECT_ID}/server-proxy - _SUFFIX: "6f21" - _INSTANCE_CONNECTION_NAME: glasnt-postjsscurl-6728:us-central1:psql-6f21 \ No newline at end of file From ba1d223b800c3f31517773088ec41521f000ada9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 11:03:13 +1000 Subject: [PATCH 07/38] cleanup todos --- app/placeholder/Dockerfile | 3 +-- infra/containers.tf | 1 - infra/postdeployment.tf | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/placeholder/Dockerfile b/app/placeholder/Dockerfile index a64436d3..212b16a0 100644 --- a/app/placeholder/Dockerfile +++ b/app/placeholder/Dockerfile @@ -15,8 +15,7 @@ # Execute with "docker run --build-arg PROJECT_ID=$PROJECT_ID ..." ARG PROJECT_ID=YOURPROJECTID -#FROM gcr.io/$PROJECT_ID/firebase # TODO(glasnt): revert -FROM gcr.io/hsa-public/firebase +FROM gcr.io/$PROJECT_ID/firebase WORKDIR /app RUN apk add gettext curl diff --git a/infra/containers.tf b/infra/containers.tf index ee70766c..970cc659 100644 --- a/infra/containers.tf +++ b/infra/containers.tf @@ -20,5 +20,4 @@ locals { server_image = "gcr.io/${var.server_image_host}/server:${var.image_version}" client_image = "gcr.io/${var.client_image_host}/client:${var.image_version}" placeholder_image = "gcr.io/hsa-public/avocano-placeholder:postjsscurl" # TODO(glasnt): revert to tag "latest" - init_image = "gcr.io/hsa-public/avocano-init:postjsscurl" # TODO(glasnt): revert to tag "latest" } diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 79d84587..1620537f 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -48,7 +48,7 @@ resource "google_cloudbuild_trigger" "placeholder" { build { step { - name = "gcr.io/${var.project_id}/placeholder" # TODO(glasnt) revert when fixed local.placeholder_image + name = local.placeholder_image env = [ "PROJECT_ID=${var.project_id}", "SUFFIX=${local.random_suffix_value}", From 126c684377fec3f20a1d9b8e506e0c78f3a0fc8b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 11:43:21 +1000 Subject: [PATCH 08/38] Add roles/pubsub.editor permissions --- README.md | 1 + infra/README.md | 1 + infra/apis.tf | 1 + infra/metadata.yaml | 2 ++ infra/test/setup/iam.tf | 1 + 5 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 0789bdd0..dfa813c6 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ the resources of this module: - roles/firebasehosting.admin - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser +- roles/pubsub.editor - roles/resourcemanager.projectIamAdmin - roles/run.admin - roles/secretmanager.admin diff --git a/infra/README.md b/infra/README.md index 977d1d65..c05f5072 100644 --- a/infra/README.md +++ b/infra/README.md @@ -96,6 +96,7 @@ the resources of this module: - roles/firebasehosting.admin - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser +- roles/pubsub.editor - roles/resourcemanager.projectIamAdmin - roles/run.admin - roles/secretmanager.admin diff --git a/infra/apis.tf b/infra/apis.tf index 583e47c4..2916801f 100644 --- a/infra/apis.tf +++ b/infra/apis.tf @@ -50,6 +50,7 @@ module "project_services" { "compute.googleapis.com", "firebase.googleapis.com", "firebasehosting.googleapis.com", + "pubsub.googleapis.com", "iam.googleapis.com", "run.googleapis.com", "secretmanager.googleapis.com", diff --git a/infra/metadata.yaml b/infra/metadata.yaml index 04c0cd4c..0a7d0665 100644 --- a/infra/metadata.yaml +++ b/infra/metadata.yaml @@ -132,6 +132,7 @@ spec: - roles/firebasehosting.admin - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser + - roles/pubsub.editor - roles/resourcemanager.projectIamAdmin - roles/run.admin - roles/secretmanager.admin @@ -158,6 +159,7 @@ spec: - firebase.googleapis.com - firebasehosting.googleapis.com - iam.googleapis.com + - pubsub.googleapis.com - run.googleapis.com - secretmanager.googleapis.com - sqladmin.googleapis.com diff --git a/infra/test/setup/iam.tf b/infra/test/setup/iam.tf index 8e4ff5c5..961c8453 100644 --- a/infra/test/setup/iam.tf +++ b/infra/test/setup/iam.tf @@ -23,6 +23,7 @@ locals { "roles/firebasehosting.admin", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", + "roles/pubsub.editor", "roles/resourcemanager.projectIamAdmin", "roles/run.admin", "roles/secretmanager.admin", From 9a95c4a38e69ef45c52003c54b3556cb353c1a8b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 11:44:17 +1000 Subject: [PATCH 09/38] terraform format --- infra/iam.tf | 10 +++++----- infra/postdeployment.tf | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/infra/iam.tf b/infra/iam.tf index 80b52bb0..f5a73c66 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -90,14 +90,14 @@ resource "google_project_iam_member" "client_permissions" { } -locals { - cloudbuild_roles = ["roles/logging.logWriter","roles/cloudbuild.builds.builder", - "roles/iam.serviceAccountUser","roles/run.developer"] +locals { + cloudbuild_roles = ["roles/logging.logWriter", "roles/cloudbuild.builds.builder", + "roles/iam.serviceAccountUser", "roles/run.developer"] } # client account needs permissions to invoke cloud build triggers resource "google_project_iam_member" "client_cloudbuild" { - project = var.project_id + project = var.project_id for_each = toset(local.cloudbuild_roles) @@ -108,7 +108,7 @@ resource "google_project_iam_member" "client_cloudbuild" { # init account needs permissions to invoke cloud build triggers resource "google_project_iam_member" "init_cloudbuild" { - project = var.project_id + project = var.project_id for_each = toset(local.cloudbuild_roles) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 1620537f..aafe4d4a 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -15,13 +15,13 @@ */ locals { - random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" # literal value (NNNN) - random_suffix_append = var.random_suffix ? "-${random_id.suffix.hex}" : "" # appended value (-NNNN) + random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" # literal value (NNNN) + random_suffix_append = var.random_suffix ? "-${random_id.suffix.hex}" : "" # appended value (-NNNN) setup_job_name = "setup${local.random_suffix_append}" gcloud_step_container = "gcr.io/google.com/cloudsdktool/cloud-sdk:slim" - curl_step_container = "gcr.io/distroless/static-debian11" + curl_step_container = "gcr.io/distroless/static-debian11" } # used to collect access token, for authenticated POST commands @@ -117,9 +117,9 @@ fi EOT } # Now that the job definitely exists, execute it. - step { - id = "execute-setup-job" - name = local.gcloud_step_container + step { + id = "execute-setup-job" + name = local.gcloud_step_container script = "gcloud run jobs execute ${local.setup_job_name} --wait --region ${var.region} --project ${var.project_id}" } From 821bfb1c24a1c883a23f936459e20b284a28fa99 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 11:50:36 +1000 Subject: [PATCH 10/38] lint: trim whitespace --- infra/postdeployment.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index aafe4d4a..9080b8b6 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -95,7 +95,7 @@ resource "google_cloudbuild_trigger" "init" { build { step { - # Check if a job already exists under the exact name. If it doesn't, create it. + # Check if a job already exists under the exact name. If it doesn't, create it. id = "create-setup-job" name = local.gcloud_step_container script = < Date: Fri, 21 Jul 2023 11:50:46 +1000 Subject: [PATCH 11/38] lint: ignore unused variable (for now) --- infra/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/variables.tf b/infra/variables.tf index b7845fc9..51794201 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -29,6 +29,7 @@ variable "region" { # HSA +# tflint-ignore: terraform_unused_declarations variable "zone" { type = string description = "GCP zone for provisioning zonal resources." From 604bda4e80d6cf4e4a172e4c37a970cc5df7115e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 12:11:04 +1000 Subject: [PATCH 12/38] Add roles/cloudbuild.builds.editor permissions --- README.md | 1 + infra/README.md | 1 + infra/metadata.yaml | 1 + infra/test/setup/iam.tf | 1 + 4 files changed, 4 insertions(+) diff --git a/README.md b/README.md index dfa813c6..01809cee 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The following dependencies must be available: A service account with the following roles must be used to provision the resources of this module: +- roles/cloudbuild.builds.editor - roles/cloudsql.admin - roles/compute.admin - roles/compute.networkAdmin diff --git a/infra/README.md b/infra/README.md index c05f5072..45cfb2e3 100644 --- a/infra/README.md +++ b/infra/README.md @@ -89,6 +89,7 @@ The following dependencies must be available: A service account with the following roles must be used to provision the resources of this module: +- roles/cloudbuild.builds.editor - roles/cloudsql.admin - roles/compute.admin - roles/compute.networkAdmin diff --git a/infra/metadata.yaml b/infra/metadata.yaml index 0a7d0665..8c4cc564 100644 --- a/infra/metadata.yaml +++ b/infra/metadata.yaml @@ -125,6 +125,7 @@ spec: roles: - level: Project roles: + - roles/cloudbuild.builds.editor - roles/cloudsql.admin - roles/compute.admin - roles/compute.networkAdmin diff --git a/infra/test/setup/iam.tf b/infra/test/setup/iam.tf index 961c8453..a7664cd6 100644 --- a/infra/test/setup/iam.tf +++ b/infra/test/setup/iam.tf @@ -16,6 +16,7 @@ locals { int_required_roles = [ + "roles/cloudbuild.builds.editor", "roles/cloudsql.admin", "roles/compute.admin", "roles/compute.networkAdmin", From d3dbb0788961102de05e4b66b8413689e5a0273c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 12:14:46 +1000 Subject: [PATCH 13/38] bump avocano version --- infra/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/variables.tf b/infra/variables.tf index 51794201..de655b9a 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -62,7 +62,7 @@ variable "init" { variable "image_version" { type = string - default = "v1.8.2" + default = "v1.9.0" description = "Version of the Container Registry image to use" } From 323ef0ed20626967d1447b2c49362c1ae1e32a4e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 13:19:09 +1000 Subject: [PATCH 14/38] bump avocano version --- infra/README.md | 2 +- infra/metadata.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/README.md b/infra/README.md index 45cfb2e3..cf4de0c1 100644 --- a/infra/README.md +++ b/infra/README.md @@ -48,7 +48,7 @@ Functional examples are included in the | database\_name | Cloud SQL database name | `string` | `"django"` | no | | database\_username | Cloud SQL database name | `string` | `"server"` | no | | enable\_apis | Whether or not to enable underlying apis in this solution. | `bool` | `true` | no | -| image\_version | Version of the Container Registry image to use | `string` | `"v1.8.2"` | no | +| image\_version | Version of the Container Registry image to use | `string` | `"v1.9.0"` | no | | init | Initialize database? | `bool` | `true` | no | | instance\_name | Cloud SQL Instance name | `string` | `"psql"` | no | | labels | A set of key/value label pairs to assign to the resources deployed by this blueprint. | `map(string)` | `{}` | no | diff --git a/infra/metadata.yaml b/infra/metadata.yaml index 8c4cc564..17bfe30d 100644 --- a/infra/metadata.yaml +++ b/infra/metadata.yaml @@ -69,7 +69,7 @@ spec: - name: image_version description: Version of the Container Registry image to use varType: string - defaultValue: v1.8.2 + defaultValue: v1.9.0 - name: init description: Initialize database? varType: bool From dc04522a55d8c12feef544fd8646413a99d421ae Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 13:19:37 +1000 Subject: [PATCH 15/38] use updated placeholder placeholder --- infra/containers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/containers.tf b/infra/containers.tf index 970cc659..111cadad 100644 --- a/infra/containers.tf +++ b/infra/containers.tf @@ -19,5 +19,5 @@ locals { server_image = "gcr.io/${var.server_image_host}/server:${var.image_version}" client_image = "gcr.io/${var.client_image_host}/client:${var.image_version}" - placeholder_image = "gcr.io/hsa-public/avocano-placeholder:postjsscurl" # TODO(glasnt): revert to tag "latest" + placeholder_image = "gcr.io/hsa-public/avocano-placeholder:postjsstrigger" # TODO(glasnt): revert to tag "latest" } From cfd9c3908ad067b6de150fe4409df389c50d4b4b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 13:19:52 +1000 Subject: [PATCH 16/38] still use a client job, rather than a step. --- infra/postdeployment.tf | 55 +++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 9080b8b6..36a866d4 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -19,6 +19,7 @@ locals { random_suffix_append = var.random_suffix ? "-${random_id.suffix.hex}" : "" # appended value (-NNNN) setup_job_name = "setup${local.random_suffix_append}" + client_job_name = "client${local.random_suffix_append}" gcloud_step_container = "gcr.io/google.com/cloudsdktool/cloud-sdk:slim" curl_step_container = "gcr.io/distroless/static-debian11" @@ -94,6 +95,7 @@ resource "google_cloudbuild_trigger" "init" { service_account = google_service_account.init[0].id build { + ## Server/API processing step { # Check if a job already exists under the exact name. If it doesn't, create it. id = "create-setup-job" @@ -104,13 +106,13 @@ SETUP_JOB=$(gcloud run jobs list --filter "metadata.name~${local.setup_job_name} if [[ -z $SETUP_JOB ]]; then echo "Creating ${local.setup_job_name} Cloud Run Job" -gcloud run jobs create ${local.setup_job_name} --region ${var.region} \ - --command migrate \ - --image ${local.server_image} \ - --service-account ${google_service_account.automation.email} \ - --set-secrets DJANGO_ENV=${google_secret_manager_secret.django_settings.secret_id}:latest \ - --set-secrets DJANGO_SUPERUSER_PASSWORD=${google_secret_manager_secret.django_admin_password.secret_id}:latest \ - --set-cloudsql-instances ${google_sql_database_instance.postgres.connection_name} + gcloud run jobs create ${local.setup_job_name} --region ${var.region} \ + --command setup \ + --image ${local.server_image} \ + --service-account ${google_service_account.automation.email} \ + --set-secrets DJANGO_ENV=${google_secret_manager_secret.django_settings.secret_id}:latest \ + --set-secrets DJANGO_SUPERUSER_PASSWORD=${google_secret_manager_secret.django_admin_password.secret_id}:latest \ + --set-cloudsql-instances ${google_sql_database_instance.postgres.connection_name} else echo "Cloud Run Job ${local.setup_job_name} already exists." fi @@ -123,20 +125,37 @@ EOT script = "gcloud run jobs execute ${local.setup_job_name} --wait --region ${var.region} --project ${var.project_id}" } - # Instead of creating a client job then executing it, execute the container itself. - # This container doesn't require additional Cloud SQL configurations like the server container, so it can be run directly - # rather than creating a job first. + ## Client/frontend processing step { - id = "client-setup" - name = local.client_image - env = [ - "PROJECT_ID=${var.project_id}", - "SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""}", - "REGION=${var.region}", - "FIREBASE_URL=${local.firebase_url}", - ] + # Check if a job already exists under the exact name. If it doesn't, create it. + id = "create-client-job" + name = local.gcloud_step_container + script = < Date: Fri, 21 Jul 2023 13:34:25 +1000 Subject: [PATCH 17/38] move from distroless to ubuntu these steps weren't tested; you can't do interactive steps in distroless ubuntu latest is sufficiently small, allows installing/executing utils --- infra/postdeployment.tf | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 36a866d4..cdf4642a 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -18,11 +18,10 @@ locals { random_suffix_value = var.random_suffix ? random_id.suffix.hex : "" # literal value (NNNN) random_suffix_append = var.random_suffix ? "-${random_id.suffix.hex}" : "" # appended value (-NNNN) - setup_job_name = "setup${local.random_suffix_append}" + setup_job_name = "setup${local.random_suffix_append}" client_job_name = "client${local.random_suffix_append}" gcloud_step_container = "gcr.io/google.com/cloudsdktool/cloud-sdk:slim" - curl_step_container = "gcr.io/distroless/static-debian11" } # used to collect access token, for authenticated POST commands @@ -155,19 +154,17 @@ EOT script = "gcloud run jobs execute ${local.client_job_name} --wait --region ${var.region} --project ${var.project_id}" } - ## Setup and priming tasks # Ensure any cached versions of the application are purged - step { - id = "purge-firebase" - name = local.curl_step_container - script = "curl -X PURGE \"${local.firebase_url}/\"" - } - # Preemptively warm up the server API step { - id = "warmup-api" - name = local.curl_step_container - script = "curl \"${google_cloud_run_v2_service.server.uri}/api/products/?warmup\"" + id = "purge-and-warmfirebase" + name = "ubuntu" + script = < Date: Fri, 21 Jul 2023 13:51:30 +1000 Subject: [PATCH 18/38] fix: ensure correct workdir used when using image in cloudbuild --- app/placeholder/Dockerfile | 2 +- app/placeholder/placeholder-deploy.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/placeholder/Dockerfile b/app/placeholder/Dockerfile index 212b16a0..45b23c26 100644 --- a/app/placeholder/Dockerfile +++ b/app/placeholder/Dockerfile @@ -20,6 +20,6 @@ WORKDIR /app RUN apk add gettext curl RUN npm install -g json -COPY . ./ +COPY . /app/ ENTRYPOINT ["/app/placeholder-deploy.sh"] diff --git a/app/placeholder/placeholder-deploy.sh b/app/placeholder/placeholder-deploy.sh index 46cc3118..d55df618 100755 --- a/app/placeholder/placeholder-deploy.sh +++ b/app/placeholder/placeholder-deploy.sh @@ -17,6 +17,9 @@ # any errors? exit immediately. set -e +# Ensure we got to the right directory. Cloud Build may start us in /workspace +cd /app + # escape if firebase_url not defined (mandatory, required later) if [[ -z $FIREBASE_URL ]]; then echo "FIREBASE_URL not defined. Cannot deploy. Exiting." From fdcd270d812ce27ecc3e3baee453ca7c320ab599 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 14:29:13 +1000 Subject: [PATCH 19/38] make all init jobs optional; (debug) use init sa for permissions --- infra/postdeployment.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index cdf4642a..b5fa86dd 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -35,12 +35,14 @@ resource "google_pubsub_topic" "faux" { ## Placeholder - deploys a placeholder website - uses prebuilt image in /app/placeholder resource "google_cloudbuild_trigger" "placeholder" { + count = var.init ? 1 : 0 + name = "placeholder${local.random_suffix_append}" location = "us-central1" description = "Deploy a placeholder Firebase website" - service_account = google_service_account.client.id + service_account = google_service_account.init[0].id pubsub_config { topic = google_pubsub_topic.faux.id @@ -82,6 +84,8 @@ data "http" "execute_placeholder_trigger" { ## Initalization trigger resource "google_cloudbuild_trigger" "init" { + count = var.init ? 1 : 0 + name = "init-application${local.random_suffix_append}" location = var.region From f4fffbed2f40a472c1c3039a0ca4e424a6c07df8 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 14:37:12 +1000 Subject: [PATCH 20/38] update optional resources --- infra/postdeployment.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index b5fa86dd..b2371065 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -70,14 +70,14 @@ resource "google_cloudbuild_trigger" "placeholder" { data "http" "execute_placeholder_trigger" { count = var.init ? 1 : 0 - url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.placeholder.id}:run" + url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.placeholder[0].id}:run" method = "POST" request_headers = { Authorization = "Bearer ${data.google_client_config.current.access_token}" } depends_on = [ module.project_services, - google_cloudbuild_trigger.placeholder + google_cloudbuild_trigger.placeholder[0] ] } @@ -182,7 +182,7 @@ EOT data "http" "execute_init_trigger" { count = var.init ? 1 : 0 - url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.init.id}:run" + url = "https://cloudbuild.googleapis.com/v1/${google_cloudbuild_trigger.init[0].id}:run" method = "POST" request_headers = { Authorization = "Bearer ${data.google_client_config.current.access_token}" @@ -190,6 +190,6 @@ data "http" "execute_init_trigger" { depends_on = [ module.project_services, google_sql_database_instance.postgres, - google_cloudbuild_trigger.init, + google_cloudbuild_trigger.init[0], ] } From 729b0a2d510ca25b9657c653362607b3f83579a0 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 15:00:13 +1000 Subject: [PATCH 21/38] (maybe?) ensure broken placeholder looks identical to working init trigger --- infra/postdeployment.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index b2371065..cc005c2a 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -38,18 +38,19 @@ resource "google_cloudbuild_trigger" "placeholder" { count = var.init ? 1 : 0 name = "placeholder${local.random_suffix_append}" - location = "us-central1" + location = var.region description = "Deploy a placeholder Firebase website" - service_account = google_service_account.init[0].id - pubsub_config { topic = google_pubsub_topic.faux.id } + service_account = google_service_account.init[0].id + build { step { + id = "deploy-placeholder" name = local.placeholder_image env = [ "PROJECT_ID=${var.project_id}", From ae3762218987c16a9aacea7395dd5dc6bc31677e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 15:01:23 +1000 Subject: [PATCH 22/38] lint --- infra/postdeployment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index cc005c2a..6677c8c3 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -50,7 +50,7 @@ resource "google_cloudbuild_trigger" "placeholder" { build { step { - id = "deploy-placeholder" + id = "deploy-placeholder" name = local.placeholder_image env = [ "PROJECT_ID=${var.project_id}", From 7ed09e4d12d7dc7bed0a76fd8809d1e676759533 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 15:10:33 +1000 Subject: [PATCH 23/38] the trigger execution needs to wait for the permissions to land --- infra/postdeployment.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 6677c8c3..4c8b4237 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -63,6 +63,10 @@ resource "google_cloudbuild_trigger" "placeholder" { logging = "CLOUD_LOGGING_ONLY" } } + + depends_on = [ + google_service_account.init[0] + ] } From c505dc2fea0f0fed3e674afc524430a070de2960 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 15:31:57 +1000 Subject: [PATCH 24/38] cleanup depends, don't give client perms it doesn't need anymore --- infra/iam.tf | 22 +++------------------- infra/postdeployment.tf | 3 ++- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/infra/iam.tf b/infra/iam.tf index f5a73c66..43b7880e 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -89,32 +89,16 @@ resource "google_project_iam_member" "client_permissions" { depends_on = [google_service_account.client] } - -locals { - cloudbuild_roles = ["roles/logging.logWriter", "roles/cloudbuild.builds.builder", - "roles/iam.serviceAccountUser", "roles/run.developer"] -} - -# client account needs permissions to invoke cloud build triggers -resource "google_project_iam_member" "client_cloudbuild" { - project = var.project_id - - for_each = toset(local.cloudbuild_roles) - - role = each.key - member = local.client_SA - depends_on = [google_service_account.client] -} - # init account needs permissions to invoke cloud build triggers resource "google_project_iam_member" "init_cloudbuild" { project = var.project_id - for_each = toset(local.cloudbuild_roles) + for_each = toset(["roles/logging.logWriter", "roles/cloudbuild.builds.builder", + "roles/iam.serviceAccountUser", "roles/run.developer"]) role = each.key member = local.init_SA - depends_on = [google_service_account.client] + depends_on = [google_service_account.init] } # Init process needs access to start Jobs diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 4c8b4237..ed8c6bc4 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -65,7 +65,8 @@ resource "google_cloudbuild_trigger" "placeholder" { } depends_on = [ - google_service_account.init[0] + google_service_account.init[0], # service account exists + google_project_iam_member.init_cloudbuild # permissions assigned ] } From 0475b3a21695b4cb8f5c290146ed0a463dd425fb Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 17:25:00 +1000 Subject: [PATCH 25/38] add an wait to let iam consist eventually, attempt to fix race conditions --- infra/postdeployment.tf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index ed8c6bc4..fd71e5e0 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -33,6 +33,17 @@ resource "google_pubsub_topic" "faux" { name = "faux-topic${local.random_suffix_append}" } +# wait for early IAM commands to settle before trying to create/execute the early placeholder +resource "time_sleep" "wait_for_iam" { + create_duration = "30s" + + depends_on = [ + module.project_services, # apis enabled + google_service_account.init[0], # service account exists + google_project_iam_member.init_cloudbuild # permissions assigned + ] +} + ## Placeholder - deploys a placeholder website - uses prebuilt image in /app/placeholder resource "google_cloudbuild_trigger" "placeholder" { count = var.init ? 1 : 0 @@ -65,8 +76,7 @@ resource "google_cloudbuild_trigger" "placeholder" { } depends_on = [ - google_service_account.init[0], # service account exists - google_project_iam_member.init_cloudbuild # permissions assigned + time_sleep.wait_for_iam ] } @@ -82,7 +92,6 @@ data "http" "execute_placeholder_trigger" { Authorization = "Bearer ${data.google_client_config.current.access_token}" } depends_on = [ - module.project_services, google_cloudbuild_trigger.placeholder[0] ] } From 469df4aa034aa2b95007bc23d213c68c14ff9e75 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 17:55:24 +1000 Subject: [PATCH 26/38] debug: wait longer, and for more debugs --- infra/postdeployment.tf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index fd71e5e0..37a28301 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -35,12 +35,14 @@ resource "google_pubsub_topic" "faux" { # wait for early IAM commands to settle before trying to create/execute the early placeholder resource "time_sleep" "wait_for_iam" { - create_duration = "30s" + create_duration = "60s" depends_on = [ - module.project_services, # apis enabled - google_service_account.init[0], # service account exists - google_project_iam_member.init_cloudbuild # permissions assigned + module.project_services, + google_service_account.init[0], + google_service_account.client, + google_project_iam_member.init_cloudbuild, + client_permissions.client_permissions ] } @@ -57,7 +59,7 @@ resource "google_cloudbuild_trigger" "placeholder" { topic = google_pubsub_topic.faux.id } - service_account = google_service_account.init[0].id + service_account = google_service_account.client.id build { step { From 33ef1dfb7e0df1319158885194a19918a70af53c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 18:27:01 +1000 Subject: [PATCH 27/38] that's not a resource --- infra/postdeployment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 37a28301..e31c395e 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -42,7 +42,7 @@ resource "time_sleep" "wait_for_iam" { google_service_account.init[0], google_service_account.client, google_project_iam_member.init_cloudbuild, - client_permissions.client_permissions + google_project_iam_member.google_project_iam_member.client_permissions ] } From 782f6b64f3776f345523ee3fbc4fee86c51e60c1 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 18:42:31 +1000 Subject: [PATCH 28/38] neither is that --- infra/postdeployment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index e31c395e..aa19d4b5 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -42,7 +42,7 @@ resource "time_sleep" "wait_for_iam" { google_service_account.init[0], google_service_account.client, google_project_iam_member.init_cloudbuild, - google_project_iam_member.google_project_iam_member.client_permissions + google_project_iam_member.client_permissions ] } From fc1c23615522d2f3048e72f36569d58d4af0aa7b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 19:16:58 +1000 Subject: [PATCH 29/38] debug: improve iam reliability by artificially staggering triggers --- infra/postdeployment.tf | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index aa19d4b5..51c2b047 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -33,19 +33,6 @@ resource "google_pubsub_topic" "faux" { name = "faux-topic${local.random_suffix_append}" } -# wait for early IAM commands to settle before trying to create/execute the early placeholder -resource "time_sleep" "wait_for_iam" { - create_duration = "60s" - - depends_on = [ - module.project_services, - google_service_account.init[0], - google_service_account.client, - google_project_iam_member.init_cloudbuild, - google_project_iam_member.client_permissions - ] -} - ## Placeholder - deploys a placeholder website - uses prebuilt image in /app/placeholder resource "google_cloudbuild_trigger" "placeholder" { count = var.init ? 1 : 0 @@ -78,7 +65,10 @@ resource "google_cloudbuild_trigger" "placeholder" { } depends_on = [ - time_sleep.wait_for_iam + # This trigger doesn't depend on this service + # However, IAM eventual consistency takes a while, and as the other trigger + # which has to fire later has more dependencies, this will always fire and finish first. + google_sql_database_instance.postgres, ] } @@ -192,6 +182,12 @@ EOT logging = "CLOUD_LOGGING_ONLY" } } + + depends_on = [ + google_sql_database_instance.postgres, + google_cloud_run_v2_job.client, + google_cloud_run_v2_job.migrate + ] } # execute the trigger, once it and other dependencies exist. Intended side-effect. @@ -205,8 +201,6 @@ data "http" "execute_init_trigger" { Authorization = "Bearer ${data.google_client_config.current.access_token}" } depends_on = [ - module.project_services, - google_sql_database_instance.postgres, google_cloudbuild_trigger.init[0], ] } From c2d094f4abb205323bb223d27bc4fad96bd9ff03 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 19:18:31 +1000 Subject: [PATCH 30/38] remove duplicate cloud run job declaration --- infra/jobs.tf | 43 +---------------------------------------- infra/postdeployment.tf | 2 ++ 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/infra/jobs.tf b/infra/jobs.tf index f9ee638f..4cf808fa 100644 --- a/infra/jobs.tf +++ b/infra/jobs.tf @@ -54,45 +54,4 @@ resource "google_cloud_run_v2_job" "migrate" { ] } -# Job to apply client application updates -resource "google_cloud_run_v2_job" "client" { - - name = var.random_suffix ? "client-${random_id.suffix.hex}" : "client" - location = var.region - - labels = var.labels - - template { - template { - service_account = google_service_account.client.email - containers { - image = local.client_image - - # Variables used to customise Firebase configuration on deployment - # https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh - env { - name = "SUFFIX" - value = var.random_suffix ? random_id.suffix.hex : "" - } - env { - name = "SERVICE_NAME" - value = google_cloud_run_v2_service.server.name - } - env { - name = "REGION" - value = var.region - } - env { - name = "PROJECT_ID" - value = var.project_id - } - - } - } - } - - depends_on = [ - module.project_services - ] -} - +# Other jobs defined in postdeployment.tf diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 51c2b047..1922743e 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -138,6 +138,8 @@ EOT ## Client/frontend processing step { # Check if a job already exists under the exact name. If it doesn't, create it. + # Environment variables used to customise Firebase configuration on deployment + # https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh id = "create-client-job" name = local.gcloud_step_container script = < Date: Fri, 21 Jul 2023 19:31:22 +1000 Subject: [PATCH 31/38] correct removal of tf-based client job --- infra/outputs.tf | 2 +- infra/postdeployment.tf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/infra/outputs.tf b/infra/outputs.tf index a6790c00..ab48df7f 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -71,5 +71,5 @@ output "server_service_name" { output "client_job_name" { description = "Name of the Cloud Run Job, deploying the front end" - value = google_cloud_run_v2_job.client.name + value = local.client_job_name } diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 1922743e..87fa91d0 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -187,7 +187,6 @@ EOT depends_on = [ google_sql_database_instance.postgres, - google_cloud_run_v2_job.client, google_cloud_run_v2_job.migrate ] } From 75af1dcf3224e8ab3de271790c2b233f5a95a31b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 19:48:54 +1000 Subject: [PATCH 32/38] correct syntax --- infra/postdeployment.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 87fa91d0..bd49e719 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -151,10 +151,10 @@ if [[ -z $SETUP_JOB ]]; then gcloud run jobs create ${local.client_job_name} --region ${var.region} \ --image ${local.client_image} \ --service-account ${google_service_account.client.email} \ - --set-env_var PROJECT_ID=${var.project_id} \ - --set-env_var SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""} \ - --set-env_var REGION=${var.region} \ - --set-env_var FIREBASE_URL=${local.firebase_url} + --set-env-var PROJECT_ID=${var.project_id} \ + --set-env-var SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""} \ + --set-env-var REGION=${var.region} \ + --set-env-var FIREBASE_URL=${local.firebase_url} else echo "Cloud Run Job ${local.client_job_name} already exists." fi From 057953bb58f6bc428f2cd08448fb8fa1df22d651 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 20:12:48 +1000 Subject: [PATCH 33/38] cleanup IAM, attempt to reduce concurrent policy changes --- infra/iam.tf | 98 ++++++++++++++++++----------------------- infra/postdeployment.tf | 2 +- 2 files changed, 44 insertions(+), 56 deletions(-) diff --git a/infra/iam.tf b/infra/iam.tf index 43b7880e..5ec632d5 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -22,8 +22,30 @@ locals { server_SA = "serviceAccount:${google_service_account.server.email}" client_SA = "serviceAccount:${google_service_account.client.email}" init_SA = "serviceAccount:${google_service_account.init[0].email}" + + # Lists of required roles + server_iam_members = [ + "roles/cloudsql.client", + "roles/run.viewer", + "roles/cloudtrace.agent" + ] + client_iam_members = [ + "roles/run.viewer", + "roles/firebasehosting.admin", + ] + automation_iam_members = [ + "roles/cloudsql.client" + ] + init_iam_members = [ + "roles/logging.logWriter", + "roles/cloudbuild.builds.builder", + "roles/iam.serviceAccountUser", + "roles/run.developer" + ] } +# Accounts + resource "google_service_account" "server" { account_id = var.random_suffix ? "api-backend-${random_id.suffix.hex}" : "api-backend" display_name = "API Backend service account" @@ -42,6 +64,7 @@ resource "google_service_account" "automation" { depends_on = [module.project_services] } + resource "google_service_account" "init" { account_id = var.random_suffix ? "init-startup-${random_id.suffix.hex}" : "init-startup" display_name = "Jump Start App Init SA" @@ -49,71 +72,36 @@ resource "google_service_account" "init" { count = var.init ? 1 : 0 } -# The Cloud Run server can access the database -resource "google_project_iam_member" "server_permissions" { - project = var.project_id - role = "roles/cloudsql.client" - member = local.server_SA - depends_on = [google_service_account.server] -} - -# Cloud Build can access the database -resource "google_project_iam_member" "build_permissions" { - project = var.project_id - role = "roles/cloudsql.client" - member = local.automation_SA - depends_on = [google_service_account.automation] -} +# Permissions -# Server needs introspection permissions -resource "google_project_iam_member" "server_introspection" { - project = var.project_id - role = "roles/run.viewer" - member = local.server_SA - depends_on = [google_service_account.server] -} +resource "google_project_iam_member" "server_permissions" { + count = length(local.server_iam_members) -# Client needs introspection permissions -resource "google_project_iam_member" "client_introspection" { - project = var.project_id - role = "roles/run.viewer" - member = local.client_SA - depends_on = [google_service_account.client] + project = var.project_id + role = local.server_iam_members[count.index] + member = "serviceAccount:${google_service_account.server.email}" } -# Client may need permission to deploy the front end resource "google_project_iam_member" "client_permissions" { - project = var.project_id - role = "roles/firebasehosting.admin" - member = local.client_SA - depends_on = [google_service_account.client] -} + count = length(local.client_iam_members) -# init account needs permissions to invoke cloud build triggers -resource "google_project_iam_member" "init_cloudbuild" { project = var.project_id + role = local.client_iam_members[count.index] + member = "serviceAccount:${google_service_account.client.email}" +} - for_each = toset(["roles/logging.logWriter", "roles/cloudbuild.builds.builder", - "roles/iam.serviceAccountUser", "roles/run.developer"]) +resource "google_project_iam_member" "automation_permissions" { + count = length(local.automation_iam_members) - role = each.key - member = local.init_SA - depends_on = [google_service_account.init] + project = var.project_id + role = local.automation_iam_members[count.index] + member = "serviceAccount:${google_service_account.automation.email}" } -# Init process needs access to start Jobs -resource "google_project_iam_member" "initstartup_permissions" { - project = var.project_id - role = "roles/run.developer" - member = local.init_SA - depends_on = [google_service_account.init] - count = var.init ? 1 : 0 -} +resource "google_project_iam_member" "init_permissions" { + count = length(local.init_iam_members) -# Server needs to write to Cloud Trace -resource "google_project_iam_member" "server_traceagent" { - project = var.project_id - role = "roles/cloudtrace.agent" - member = local.server_SA - depends_on = [google_service_account.server] + project = var.project_id + role = local.init_iam_members[count.index] + member = "serviceAccount:${google_service_account.init[0].email}" } diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index bd49e719..390a69a4 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -46,7 +46,7 @@ resource "google_cloudbuild_trigger" "placeholder" { topic = google_pubsub_topic.faux.id } - service_account = google_service_account.client.id + service_account = google_service_account.init[0].id build { step { From e74695c9b380ddd52b74bccca32c5f94a4ff821d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 20:24:52 +1000 Subject: [PATCH 34/38] lint, remove unused helpers --- infra/iam.tf | 8 -------- infra/secrets.tf | 7 +++++-- infra/storage.tf | 4 ++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/infra/iam.tf b/infra/iam.tf index 5ec632d5..37f797b0 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -14,15 +14,7 @@ * limitations under the License. */ -# Service Accounts - locals { - # Helpers for the clunky formatting of these values - automation_SA = "serviceAccount:${google_service_account.automation.email}" - server_SA = "serviceAccount:${google_service_account.server.email}" - client_SA = "serviceAccount:${google_service_account.client.email}" - init_SA = "serviceAccount:${google_service_account.init[0].email}" - # Lists of required roles server_iam_members = [ "roles/cloudsql.client", diff --git a/infra/secrets.tf b/infra/secrets.tf index 88d55331..bf5b7fdf 100644 --- a/infra/secrets.tf +++ b/infra/secrets.tf @@ -39,7 +39,7 @@ resource "google_secret_manager_secret" "django_admin_password" { resource "google_secret_manager_secret_iam_binding" "django_admin_password" { secret_id = google_secret_manager_secret.django_admin_password.id role = "roles/secretmanager.secretAccessor" - members = [local.automation_SA] + members = ["serviceAccount:${google_service_account.automation.email}"] } resource "google_secret_manager_secret_version" "django_admin_password" { @@ -80,5 +80,8 @@ EOF resource "google_secret_manager_secret_iam_binding" "django_settings" { secret_id = google_secret_manager_secret.django_settings.id role = "roles/secretmanager.secretAccessor" - members = [local.server_SA, local.automation_SA] + members = [ + "serviceAccount:${google_service_account.server.email}", + "serviceAccount:${google_service_account.automation.email}" + ] } diff --git a/infra/storage.tf b/infra/storage.tf index 6215479a..3b148d14 100644 --- a/infra/storage.tf +++ b/infra/storage.tf @@ -26,12 +26,12 @@ resource "google_storage_bucket" "media" { resource "google_storage_bucket_iam_member" "server" { bucket = google_storage_bucket.media.name - member = local.server_SA + member = "serviceAccount:${google_service_account.server.email}" role = "roles/storage.admin" } resource "google_storage_bucket_iam_member" "automation" { bucket = google_storage_bucket.media.name - member = local.automation_SA + member = "serviceAccount:${google_service_account.automation.email}" role = "roles/storage.admin" } From a75787e81338d83c041749a2b0583c2d7c8e7702 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 20:47:09 +1000 Subject: [PATCH 35/38] fix syntax --- infra/postdeployment.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 390a69a4..d8c9010c 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -151,10 +151,10 @@ if [[ -z $SETUP_JOB ]]; then gcloud run jobs create ${local.client_job_name} --region ${var.region} \ --image ${local.client_image} \ --service-account ${google_service_account.client.email} \ - --set-env-var PROJECT_ID=${var.project_id} \ - --set-env-var SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""} \ - --set-env-var REGION=${var.region} \ - --set-env-var FIREBASE_URL=${local.firebase_url} + --set-env-vars PROJECT_ID=${var.project_id} \ + --set-env-vars SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""} \ + --set-env-vars REGION=${var.region} \ + --set-env-vars FIREBASE_URL=${local.firebase_url} else echo "Cloud Run Job ${local.client_job_name} already exists." fi From 1bca6da408d6dc1247e92cfda956c31f0658ec26 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 21:01:09 +1000 Subject: [PATCH 36/38] cleanup client job --- infra/postdeployment.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index d8c9010c..2838de68 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -152,9 +152,9 @@ if [[ -z $SETUP_JOB ]]; then --image ${local.client_image} \ --service-account ${google_service_account.client.email} \ --set-env-vars PROJECT_ID=${var.project_id} \ - --set-env-vars SUFFIX=${var.random_suffix ? random_id.suffix.hex : ""} \ + --set-env-vars SUFFIX=${local.random_suffix_value} \ --set-env-vars REGION=${var.region} \ - --set-env-vars FIREBASE_URL=${local.firebase_url} + --set-env-vars SERVICE_NAME=${google_cloud_run_v2_service.server.name} else echo "Cloud Run Job ${local.client_job_name} already exists." fi From f153dbce81747d3856cf31952305253feb8a844c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 21:32:17 +1000 Subject: [PATCH 37/38] init needs firebase admin --- infra/iam.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/iam.tf b/infra/iam.tf index 37f797b0..74e545e4 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -32,7 +32,8 @@ locals { "roles/logging.logWriter", "roles/cloudbuild.builds.builder", "roles/iam.serviceAccountUser", - "roles/run.developer" + "roles/run.developer", + "roles/firebasehosting.admin" ] } From fecf35469d73494a740da179ceba36a92e0a7b3d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 21 Jul 2023 21:32:25 +1000 Subject: [PATCH 38/38] create the client job as early as possible, hopefully helping tests --- infra/postdeployment.tf | 55 +++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/infra/postdeployment.tf b/infra/postdeployment.tf index 2838de68..6a0383b1 100644 --- a/infra/postdeployment.tf +++ b/infra/postdeployment.tf @@ -105,6 +105,32 @@ resource "google_cloudbuild_trigger" "init" { service_account = google_service_account.init[0].id build { + ## Client/frontend processing + step { + # Check if a job already exists under the exact name. If it doesn't, create it. + # Environment variables used to customise Firebase configuration on deployment + # https://github.com/GoogleCloudPlatform/avocano/blob/main/client/docker-deploy.sh + id = "create-client-job" + name = local.gcloud_step_container + script = <