fix(docker) Add platform to docker-compose command

docker/dev-with-cassandra.sh

@@ -23,13 +23,13 @@ fi
 # YOU MUST BUILD VIA GRADLE BEFORE RUNNING THIS.
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd $DIR && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose \
     -f docker-compose-with-cassandra.yml \
     -f docker-compose.dev.yml \
     $CONSUMERS_COMPOSE $MONITORING_COMPOSE $M1_COMPOSE \
     pull \
 && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose -p datahub \
     -f docker-compose-with-cassandra.yml \
     -f docker-compose.dev.yml \
     $CONSUMERS_COMPOSE $MONITORING_COMPOSE $M1_COMPOSE \

docker/dev-without-neo4j.sh

@@ -22,13 +22,13 @@ fi
 # YOU MUST BUILD VIA GRADLE BEFORE RUNNING THIS.
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd $DIR && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose \
     -f docker-compose-without-neo4j.yml \
     -f docker-compose-without-neo4j.override.yml \
     -f docker-compose.dev.yml \
     $CONSUMERS_COMPOSE $MONITORING_COMPOSE $M1_COMPOSE pull \
 && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose -p datahub \
     -f docker-compose-without-neo4j.yml \
     -f docker-compose-without-neo4j.override.yml \
     -f docker-compose.dev.yml \

docker/dev.sh

@@ -23,13 +23,13 @@ fi
 # YOU MUST BUILD VIA GRADLE BEFORE RUNNING THIS.
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd $DIR && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose \
     -f docker-compose.yml \
     -f docker-compose.override.yml \
     -f docker-compose.dev.yml \
     $CONSUMERS_COMPOSE $MONITORING_COMPOSE $M1_COMPOSE pull \
 && \
-  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -p datahub \
+  COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose -p datahub \
     -f docker-compose.yml \
     -f docker-compose.override.yml \
     -f docker-compose.dev.yml \

docker/quickstart.sh

@@ -37,7 +37,7 @@ then
 else
   echo "No Datahub Neo4j volume found, starting with elasticsearch as graph service"
   cd $DIR && \
-  docker-compose -p datahub \
+  DOCKER_DEFAULT_PLATFORM="$(uname -m)" docker-compose -p datahub \
     -f quickstart/docker-compose-without-neo4j.quickstart.yml \
     $MONITORING_COMPOSE $CONSUMERS_COMPOSE $M1_COMPOSE up $@
 fi

metadata-ingestion/src/datahub/cli/docker.py → metadata-ingestion/src/datahub/cli/docker_cli.py

@@ -56,6 +56,12 @@
 GITHUB_M1_QUICKSTART_COMPOSE_URL = f"{GITHUB_BASE_URL}/{M1_QUICKSTART_COMPOSE_FILE}"
 GITHUB_BOOTSTRAP_MCES_URL = f"{GITHUB_BASE_URL}/{BOOTSTRAP_MCES_FILE}"
 
+DOCKER_COMPOSE_PLATFORM = (
+    subprocess.run(["uname", "-m"], stdout=subprocess.PIPE)
+    .stdout.decode("utf-8")
+    .rstrip()
+)
+
 
 @click.group()
 def docker() -> None:
@@ -208,6 +214,10 @@ def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None:
             subprocess.run(
                 [*base_command, "stop"],
                 check=True,
+                env={
+                    **os.environ,
+                    "DOCKER_DEFAULT_PLATFORM": DOCKER_COMPOSE_PLATFORM,
+                },
             )
             click.secho("Stopped datahub successfully.", fg="green")
         except subprocess.CalledProcessError:
@@ -638,6 +648,10 @@ def quickstart(
         subprocess.run(
             [*base_command, "pull", "-q"],
             check=True,
+            env={
+                **os.environ,
+                "DOCKER_DEFAULT_PLATFORM": DOCKER_COMPOSE_PLATFORM,
+            },
         )
         click.secho("Finished pulling docker images!")
     except subprocess.CalledProcessError:
@@ -659,6 +673,7 @@ def quickstart(
             check=True,
             env={
                 **os.environ,
+                "DOCKER_DEFAULT_PLATFORM": DOCKER_COMPOSE_PLATFORM,
                 "DOCKER_BUILDKIT": "1",
             },
         )
@@ -674,7 +689,13 @@ def quickstart(
         # Attempt to run docker compose up every minute.
         if (datetime.datetime.now() - start_time) > up_attempts * up_interval:
             click.echo()
-            subprocess.run(base_command + ["up", "-d", "--remove-orphans"])
+            subprocess.run(
+                base_command + ["up", "-d", "--remove-orphans"],
+                env={
+                    **os.environ,
+                    "DOCKER_DEFAULT_PLATFORM": DOCKER_COMPOSE_PLATFORM,
+                },
+            )
             up_attempts += 1
 
         # Check docker health every few seconds.
@@ -694,6 +715,10 @@ def quickstart(
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 check=True,
+                env={
+                    **os.environ,
+                    "DOCKER_DEFAULT_PLATFORM": DOCKER_COMPOSE_PLATFORM,
+                },
             )
             log_file.write(ret.stdout)
 

metadata-ingestion/src/datahub/entrypoints.py

@@ -11,7 +11,7 @@
 from datahub.cli.check_cli import check
 from datahub.cli.cli_utils import DATAHUB_CONFIG_PATH, write_datahub_config
 from datahub.cli.delete_cli import delete
-from datahub.cli.docker import docker
+from datahub.cli.docker_cli import docker
 from datahub.cli.get_cli import get
 from datahub.cli.ingest_cli import ingest
 from datahub.cli.migrate import migrate

smoke-test/tests/utils.py

@@ -6,7 +6,7 @@
 import requests
 
 from datahub.cli import cli_utils
-from datahub.cli.docker import check_local_docker_containers
+from datahub.cli.docker_cli import check_local_docker_containers
 from datahub.ingestion.run.pipeline import Pipeline