Tweak .env in kube/overlays to connect to external db (#4403)

* Mirror .env to connect to external db
airbytehq · Jul 1, 2021 · 53b02e0 · 53b02e0
1 parent 586eeb7
commit 53b02e0
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 19 deletions.
diff --git a/.env b/.env
@@ -6,14 +6,15 @@ DATABASE_PASSWORD=docker
 DATABASE_HOST=db
 DATABASE_PORT=5432
 DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
 DATABASE_URL=jdbc:postgresql://db:5432/airbyte
 
 # When using the airbyte-db via default docker image:
 CONFIG_ROOT=/data
 DATA_DOCKER_MOUNT=airbyte_data
 DB_DOCKER_MOUNT=airbyte_db
 
-# Internal Temporal.io deployment
+# Temporal.io worker configuration
 TEMPORAL_HOST=airbyte-temporal:7233
 
 # Workspace storage for running jobs (logs, etc)

diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env
@@ -1,24 +1,39 @@
 AIRBYTE_VERSION=dev
+
+# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker
-DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+DATABASE_HOST=airbyte-db-svc
+DATABASE_PORT=5432
 DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
+DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+
+# When using the airbyte-db via default docker image:
 CONFIG_ROOT=/configs
-WORKSPACE_ROOT=/workspace
 DATA_DOCKER_MOUNT=airbyte_data
 DB_DOCKER_MOUNT=airbyte_db
+
+# Temporal.io worker configuration
+TEMPORAL_HOST=airbyte-temporal-svc:7233
+TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030
+
+# Workspace storage for running jobs (logs, etc)
+WORKSPACE_ROOT=/workspace
 WORKSPACE_DOCKER_MOUNT=airbyte_workspace
-API_URL=/api/v1/
-WEBAPP_URL=airbyte-webapp-svc:80
+
+LOCAL_ROOT=/tmp/airbyte_local
+
+# Miscellaneous
 TRACKING_STRATEGY=logging
+WEBAPP_URL=airbyte-webapp-svc:80
+API_URL=/api/v1/
+INTERNAL_API_HOST=airbyte-server-svc:8001
+
 WORKER_ENVIRONMENT=kubernetes
-LOCAL_ROOT=/tmp/airbyte_local
 PAPERCUPS_STORYTIME=disabled
 FULLSTORY=disabled
 IS_DEMO=false
-TEMPORAL_HOST=airbyte-temporal-svc:7233
-INTERNAL_API_HOST=airbyte-server-svc:8001
-TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030
 LOG_LEVEL=INFO
 
 S3_LOG_BUCKET=airbyte-dev-logs
@@ -28,6 +43,7 @@ AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
+# Docker Resource Limits
 RESOURCE_CPU_REQUEST=
 RESOURCE_CPU_LIMIT=
 RESOURCE_MEMORY_REQUEST=

diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env
@@ -1,24 +1,39 @@
 AIRBYTE_VERSION=0.27.0-alpha
+
+# Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker
-DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+DATABASE_HOST=airbyte-db-svc
+DATABASE_PORT=5432
 DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB}
+DATABASE_URL=jdbc:postgresql://airbyte-db-svc:5432/airbyte
+
+# When using the airbyte-db via default docker image:
 CONFIG_ROOT=/configs
-WORKSPACE_ROOT=/workspace
 DATA_DOCKER_MOUNT=airbyte_data
 DB_DOCKER_MOUNT=airbyte_db
+
+# Temporal.io worker configuration
+TEMPORAL_HOST=airbyte-temporal-svc:7233
+TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030
+
+# Workspace storage for running jobs (logs, etc)
+WORKSPACE_ROOT=/workspace
 WORKSPACE_DOCKER_MOUNT=airbyte_workspace
-API_URL=/api/v1/
-WEBAPP_URL=airbyte-webapp-svc:80
+
+LOCAL_ROOT=/tmp/airbyte_local
+
+# Miscellaneous
 TRACKING_STRATEGY=segment
+WEBAPP_URL=airbyte-webapp-svc:80
+API_URL=/api/v1/
+INTERNAL_API_HOST=airbyte-server-svc:8001
+
 WORKER_ENVIRONMENT=kubernetes
-LOCAL_ROOT=/tmp/airbyte_local
 PAPERCUPS_STORYTIME=enabled
 FULLSTORY=enabled
 IS_DEMO=false
-TEMPORAL_HOST=airbyte-temporal-svc:7233
-INTERNAL_API_HOST=airbyte-server-svc:8001
-TEMPORAL_WORKER_PORTS=9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030
 LOG_LEVEL=INFO
 
 S3_LOG_BUCKET=airbyte-dev-logs
@@ -28,6 +43,7 @@ AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
+# Docker Resource Limits
 RESOURCE_CPU_REQUEST=
 RESOURCE_CPU_LIMIT=
 RESOURCE_MEMORY_REQUEST=

diff --git a/kube/resources/scheduler.yaml b/kube/resources/scheduler.yaml
@@ -28,6 +28,16 @@ spec:
                 configMapKeyRef:
                   name: airbyte-env
                   key: CONFIG_ROOT
+            - name: DATABASE_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_HOST
+            - name: DATABASE_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_PORT
             - name: DATABASE_PASSWORD
               valueFrom:
                 configMapKeyRef:
@@ -51,6 +61,7 @@ spec:
             - name: WAIT_BEFORE_HOSTS
               value: "5"
             - name: WAIT_HOSTS
+              #Translate value: $(DATABASE_HOST):$(DATABASE_PORT)
               value: airbyte-db-svc:5432
             - name: WAIT_HOSTS_TIMEOUT
               value: "45"

diff --git a/kube/resources/temporal.yaml b/kube/resources/temporal.yaml
@@ -92,9 +92,15 @@ spec:
             - name: DB
               value: "postgresql"
             - name: DB_PORT
-              value: "5432"
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_PORT
             - name: POSTGRES_SEEDS
-              value: "airbyte-db-svc"
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_HOST
 
           ports:
             - containerPort: 7233