Custom auto-setup temporal docker image (#7681)

This is a custom auto-setup script for the temporal environment. Unfortunately there is no other way properly update the DB without copy pasting parts of the temporal auto-setup script. Ideally temporal would provide a dedicated container for it DB but it is not the case right now.

airbyte-temporal/Dockerfile

@@ -0,0 +1,8 @@
+# A test describe in the README is available to test a version update 
+FROM temporalio/auto-setup:1.13.0
+
+ENV TEMPORAL_HOME /etc/temporal
+
+COPY bin/scripts/update-and-start-temporal.sh update-and-start-temporal.sh
+
+ENTRYPOINT ["./update-and-start-temporal.sh"]

airbyte-temporal/README.md

@@ -0,0 +1,14 @@
+## Testing a temporal migration
+
+`tools/bin/test_temporal_migration.sh` is available to test that a bump of the temporal version won't break the docker compose build. Here is what 
+the script does:
+- checkout master
+- build the docker image
+- run docker compose up in the background
+- Sleep for 75 secondes
+- shutdown docker compose
+- checkout the commit being tested
+- build the docker image
+- run docker compose up.
+
+At the end of the script you should be able to access a local airbyte in `localhost:8000`.

airbyte-temporal/build.gradle

@@ -0,0 +1,10 @@
+task copyScripts(type: Copy) {
+    dependsOn copyDocker
+
+    from('scripts')
+    into 'build/docker/bin/scripts'
+}
+
+Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir")
+dockerBuildTask.dependsOn(copyScripts)
+assemble.dependsOn(dockerBuildTask)

airbyte-temporal/scripts/update-and-start-temporal.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+set -x
+
+DBNAME="${DBNAME:-temporal}"
+VISIBILITY_DBNAME="${VISIBILITY_DBNAME:-temporal_visibility}"
+DB_PORT="${DB_PORT:-3306}"
+
+POSTGRES_SEEDS="${POSTGRES_SEEDS:-}"
+POSTGRES_USER="${POSTGRES_USER:-}"
+POSTGRES_PWD="${POSTGRES_PWD:-}"
+
+SCHEMA_DIR=${TEMPORAL_HOME}/schema/postgresql/v96/temporal/versioned
+VISIBILITY_SCHEMA_DIR=${TEMPORAL_HOME}/schema/postgresql/v96/visibility/versioned
+
+SKIP_DEFAULT_NAMESPACE_CREATION="${SKIP_DEFAULT_NAMESPACE_CREATION:-false}"
+DEFAULT_NAMESPACE="${DEFAULT_NAMESPACE:-default}"
+DEFAULT_NAMESPACE_RETENTION=${DEFAULT_NAMESPACE_RETENTION:-1}
+
+# See https://github.com/temporalio/temporal/blob/release/v1.13.x/docker/entrypoint.sh
+init_entry_point() {
+  echo "Start init"
+  export BIND_ON_IP="${BIND_ON_IP:-$(hostname -i)}"
+
+  if [[ "${BIND_ON_IP}" =~ ":" ]]; then
+      # ipv6
+      export TEMPORAL_CLI_ADDRESS="[${BIND_ON_IP}]:7233"
+  else
+      # ipv4
+      export TEMPORAL_CLI_ADDRESS="${BIND_ON_IP}:7233"
+  fi
+
+  dockerize -template ./config/config_template.yaml:./config/docker.yaml
+  echo "Done init"
+}
+
+wait_for_postgres() {
+  until nc -z "${POSTGRES_SEEDS%%,*}" "${DB_PORT}"; do
+      echo 'Waiting for PostgreSQL to startup.'
+      sleep 1
+  done
+
+  echo 'PostgreSQL started.'
+}
+
+update_postgres_schema() {
+  { export SQL_PASSWORD=${POSTGRES_PWD}; } 2> /dev/null
+
+  CONTAINER_ALREADY_STARTED="CONTAINER_ALREADY_STARTED_PLACEHOLDER"
+  if [ ! -e $CONTAINER_ALREADY_STARTED ]; then
+      touch $CONTAINER_ALREADY_STARTED
+      temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" create --db "${DBNAME}"
+      temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" --db "${DBNAME}" setup-schema -v 0.0
+
+
+      temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" create --db "${VISIBILITY_DBNAME}"
+      temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" --db "${VISIBILITY_DBNAME}" setup-schema -v 0.0
+  fi
+  echo "Starting to update the temporal DB"
+  temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" --db "${DBNAME}" update-schema -d "${SCHEMA_DIR}"
+  echo "Update the temporal DB is done"
+
+  echo "Starting to update the temporal visibility DB"
+  temporal-sql-tool --plugin postgres --ep "${POSTGRES_SEEDS}" -u "${POSTGRES_USER}" -p "${DB_PORT}" --db "${VISIBILITY_DBNAME}" update-schema -d "${VISIBILITY_SCHEMA_DIR}"
+  echo "Update the temporal visibility DB is done"
+
+}
+
+setup_server(){
+    echo "Temporal CLI address: ${TEMPORAL_CLI_ADDRESS}."
+
+    until tctl cluster health | grep SERVING; do
+        echo "Waiting for Temporal server to start..."
+        sleep 1
+    done
+    echo "Temporal server started."
+
+    if [ "${SKIP_DEFAULT_NAMESPACE_CREATION}" != true ]; then
+        register_default_namespace
+    fi
+
+    if [ "${SKIP_ADD_CUSTOM_SEARCH_ATTRIBUTES}" != true ]; then
+        add_custom_search_attributes
+    fi
+}
+register_default_namespace() {
+    echo "Registering default namespace: ${DEFAULT_NAMESPACE}."
+    if ! tctl --ns "${DEFAULT_NAMESPACE}" namespace describe; then
+        echo "Default namespace ${DEFAULT_NAMESPACE} not found. Creating..."
+        tctl --ns "${DEFAULT_NAMESPACE}" namespace register --rd "${DEFAULT_NAMESPACE_RETENTION}" --desc "Default namespace for Temporal Server."
+        echo "Default namespace ${DEFAULT_NAMESPACE} registration complete."
+    else
+        echo "Default namespace ${DEFAULT_NAMESPACE} already registered."
+    fi
+}
+
+add_custom_search_attributes() {
+      echo "Adding Custom*Field search attributes."
+      # TODO: Remove CustomStringField
+# @@@SNIPSTART add-custom-search-attributes-for-testing-command
+      tctl --auto_confirm admin cluster add-search-attributes \
+          --name CustomKeywordField --type Keyword \
+          --name CustomStringField --type Text \
+          --name CustomTextField --type Text \
+          --name CustomIntField --type Int \
+          --name CustomDatetimeField --type Datetime \
+          --name CustomDoubleField --type Double \
+          --name CustomBoolField --type Bool
+# @@@SNIPEND
+}
+
+setup_server(){
+    echo "Temporal CLI address: ${TEMPORAL_CLI_ADDRESS}."
+
+    until tctl cluster health | grep SERVING; do
+        echo "Waiting for Temporal server to start..."
+        sleep 1
+    done
+    echo "Temporal server started."
+
+    if [ "${SKIP_DEFAULT_NAMESPACE_CREATION}" != true ]; then
+        register_default_namespace
+    fi
+
+    if [ "${SKIP_ADD_CUSTOM_SEARCH_ATTRIBUTES}" != true ]; then
+        add_custom_search_attributes
+    fi
+}
+
+init_entry_point
+wait_for_postgres
+update_postgres_schema
+
+echo "starting temporal server"
+setup_server &
+./start-temporal.sh

build.gradle

@@ -316,6 +316,7 @@ task('generate-docker') {
     dependsOn(':airbyte-migration:assemble')
     dependsOn(':airbyte-db:lib:assemble')
     dependsOn(':airbyte-config:init:assemble')
+    dependsOn(':airbyte-temporal:assemble')
 }
 
 // produce reproducible archives

docker-compose.debug.yaml

@@ -12,7 +12,7 @@ services:
     ports:
       - 8011:5432
   airbyte-temporal-ui:
-    image: temporalio/web:latest
+    image: temporalio/web:1.13.0
     logging: *default-logging
     container_name: airbyte-temporal-ui
     restart: unless-stopped

docker-compose.yaml

@@ -176,7 +176,7 @@ services:
       - OPENREPLAY=${OPENREPLAY:-}
       - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-}
   airbyte-temporal:
-    image: temporalio/auto-setup:1.7.0
+    image: airbyte/temporal:${VERSION}
     logging: *default-logging
     container_name: airbyte-temporal
     restart: unless-stopped

settings.gradle

@@ -61,6 +61,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
     include ':airbyte-scheduler:app'
     include ':airbyte-scheduler:client'
     include ':airbyte-server'
+    include 'airbyte-temporal'
     include ':airbyte-tests'
     include ':airbyte-webapp'
     include ':airbyte-config:init'

tools/bin/test-temporal-migration.sh

@@ -0,0 +1,30 @@
+#! /bin/bash
+
+set -ex
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+if ! command -v timeout &> /dev/null
+then
+    echo "timeout could not be found, installing it"
+    brew install coreutils
+fi
+
+NEW_HASH="$( git rev-parse HEAD )"
+
+git checkout master
+git pull --no-rebase
+
+SUB_BUILD=PLATFORM "$SCRIPT_DIR"/../../gradlew -p "$SCRIPT_DIR"/../.. generate-docker
+
+cd "$SCRIPT_DIR"/../..
+VERSION=dev docker-compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up &
+
+sleep 75
+VERSION=dev docker-compose down
+
+git stash
+git checkout $NEW_HASH
+SUB_BUILD=PLATFORM "$SCRIPT_DIR"/../../gradlew -p "$SCRIPT_DIR"/../.. generate-docker
+
+VERSION=dev docker-compose -f "$SCRIPT_DIR"/../../docker-compose.yaml up