restore containerOrchestrator image value (#20313)

* Revert "fix: remove unused CONTAINER_ORCHESTRATOR_ENABLED var (#20261)" This reverts commit ce29361. * docs: add additional commentary on flag usage
airbytehq · Dec 13, 2022 · e7c3fbf · e7c3fbf
1 parent 0fac8c8
commit e7c3fbf
Show file tree

Hide file tree

Showing 11 changed files with 32 additions and 1 deletion.
diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java
@@ -671,6 +671,12 @@ public interface Configs {
 
   // Container Orchestrator
 
+  /**
+   * Define if Airbyte should use the container orchestrator. Internal-use only.
+   * Should always be set to true - otherwise causes syncs to be run on workers instead.
+   */
+  boolean getContainerOrchestratorEnabled();
+
   /**
    * Get the name of the container orchestrator secret. Internal-use only.
    */

diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java
@@ -115,6 +115,7 @@ public class EnvConfigs implements Configs {
   private static final String CONFIGS_DATABASE_INITIALIZATION_TIMEOUT_MS = "CONFIGS_DATABASE_INITIALIZATION_TIMEOUT_MS";
   private static final String JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION = "JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION";
   private static final String JOBS_DATABASE_INITIALIZATION_TIMEOUT_MS = "JOBS_DATABASE_INITIALIZATION_TIMEOUT_MS";
+  private static final String CONTAINER_ORCHESTRATOR_ENABLED = "CONTAINER_ORCHESTRATOR_ENABLED";
   private static final String CONTAINER_ORCHESTRATOR_SECRET_NAME = "CONTAINER_ORCHESTRATOR_SECRET_NAME";
   private static final String CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH = "CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH";
   private static final String CONTAINER_ORCHESTRATOR_IMAGE = "CONTAINER_ORCHESTRATOR_IMAGE";
@@ -1057,6 +1058,11 @@ public Set<Integer> getTemporalWorkerPorts() {
     return Arrays.stream(ports.split(",")).map(Integer::valueOf).collect(Collectors.toSet());
   }
 
+  @Override
+  public boolean getContainerOrchestratorEnabled() {
+    return getEnvOrDefault(CONTAINER_ORCHESTRATOR_ENABLED, false, Boolean::valueOf);
+  }
+
   @Override
   public String getContainerOrchestratorSecretName() {
     return getEnvOrDefault(CONTAINER_ORCHESTRATOR_SECRET_NAME, null);

diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml
@@ -52,6 +52,7 @@ airbyte:
     specific-resource-defaults-enabled: ${CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED:false}
   container:
     orchestrator:
+      enabled: ${CONTAINER_ORCHESTRATOR_ENABLED:false}
       image: ${CONTAINER_ORCHESTRATOR_IMAGE:}
       java-opts: ${CONTAINER_ORCHESTRATOR_JAVA_OPTS:}
       secret-mount-path: ${CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH:}

diff --git a/charts/airbyte-worker/values.yaml b/charts/airbyte-worker/values.yaml
@@ -347,6 +347,7 @@ secrets: {}
 #   JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY:
 
 #   # Launch a separate pod to orchestrate sync steps
+#   CONTAINER_ORCHESTRATOR_ENABLED: true
 #   CONTAINER_ORCHESTRATOR_IMAGE:
 
 #   # Open Telemetry Configuration

diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml
@@ -64,6 +64,7 @@ data:
   WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: ""
   USE_STREAM_CAPABLE_STATE: "true"
   AUTO_DETECT_SCHEMA: "false"
+  CONTAINER_ORCHESTRATOR_ENABLED: {{ .Values.worker.containerOrchestrator.enabled | quote }}
   CONTAINER_ORCHESTRATOR_IMAGE: {{ .Values.worker.containerOrchestrator.image | quote }}
   WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane"
   CRON_MICRONAUT_ENVIRONMENTS: "control-plane"

diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml
@@ -795,7 +795,14 @@ worker:
 
   hpa:
     enabled: false
-
+  ## Container Orchestrator
+  ## enabled -- Enable or disable Orchestrator
+  ## image -- Orchestrator image
+
+  containerOrchestrator:
+    enabled: true
+    image: ""
+
 ## @section Metrics parameters
 metrics:
   enabled: false

diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env
@@ -74,6 +74,7 @@ JOB_KUBE_NODE_SELECTORS=
 JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
 
 # Launch a separate pod to orchestrate sync steps
+CONTAINER_ORCHESTRATOR_ENABLED=true
 CONTAINER_ORCHESTRATOR_IMAGE=
 
 # Cloud Storage Configuration

diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env
@@ -76,6 +76,7 @@ JOB_KUBE_NODE_SELECTORS=
 JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
 
 # Launch a separate pod to orchestrate sync steps
+CONTAINER_ORCHESTRATOR_ENABLED=true
 CONTAINER_ORCHESTRATOR_IMAGE=
 
 # Cloud Storage Configuration

diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env
@@ -76,6 +76,7 @@ JOB_KUBE_NODE_SELECTORS=
 JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
 
 # Launch a separate pod to orchestrate sync steps
+CONTAINER_ORCHESTRATOR_ENABLED=true
 CONTAINER_ORCHESTRATOR_IMAGE=
 
 CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED=true

diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env
@@ -76,6 +76,7 @@ JOB_KUBE_NODE_SELECTORS=
 JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY=
 
 # Launch a separate pod to orchestrate sync steps
+CONTAINER_ORCHESTRATOR_ENABLED=true
 CONTAINER_ORCHESTRATOR_IMAGE=
 
 # Cloud Storage Configuration

diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml
@@ -220,6 +220,11 @@ spec:
                 secretKeyRef:
                   name: airbyte-secrets
                   key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+            - name: CONTAINER_ORCHESTRATOR_ENABLED
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: CONTAINER_ORCHESTRATOR_ENABLED
             - name: CONTAINER_ORCHESTRATOR_IMAGE
               valueFrom:
                 configMapKeyRef: