Update prefect server start command to specify and disable port mappings

CHANGELOG.md

@@ -10,7 +10,8 @@ These changes are available in the [master branch](https://github.com/PrefectHQ/
 
 ### Enhancements
 
-- None
+- Add flags to `prefect server start` for disabling service port mapping - [#2228](https://github.com/PrefectHQ/prefect/pull/2228)
+- Add options to `prefect server start` for mapping to host ports - [#2228](https://github.com/PrefectHQ/prefect/pull/2228)
 
 ### Task Library
 

src/prefect/cli/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   postgres:
     image: "postgres:11"
     ports:
-      - "5432:5432"
+      - "${POSTGRES_HOST_PORT:-5432}:5432"
     environment:
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
@@ -20,7 +20,7 @@ services:
   hasura:
     image: "hasura/graphql-engine:v1.1.0"
     ports:
-      - "3000:3000"
+      - "${HASURA_HOST_PORT:-3000}:3000"
     command: "graphql-engine serve"
     environment:
       HASURA_GRAPHQL_DATABASE_URL: ${DB_CONNECTION_URL}
@@ -35,7 +35,7 @@ services:
   graphql:
     image: "prefecthq/server:${PREFECT_SERVER_TAG:-latest}"
     ports:
-      - "4201:4201"
+      - "${GRAPHQL_HOST_PORT:-4201}:4201"
     command: bash -c "${PREFECT_SERVER_DB_CMD} && python src/prefect_server/services/graphql/server.py"
     environment:
       PREFECT_SERVER_DB_CMD: ${PREFECT_SERVER_DB_CMD:-"echo 'DATABASE MIGRATIONS SKIPPED'"}
@@ -61,7 +61,7 @@ services:
   apollo:
     image: "prefecthq/apollo:${PREFECT_SERVER_TAG:-latest}"
     ports:
-      - "4200:4200"
+      - "${APOLLO_HOST_PORT:-4200}:4200"
     command: "npm run serve"
     environment:
       HASURA_API_URL: ${HASURA_API_URL:-http://hasura:3000/v1alpha1/graphql}
@@ -75,7 +75,7 @@ services:
   ui:
     image: "prefecthq/ui:${PREFECT_SERVER_TAG:-latest}"
     ports:
-      - "8080:8080"
+      - "${UI_HOST_PORT:-8080}:8080"
     command: "/intercept.sh"
     networks:
       - prefect-server

src/prefect/cli/server.py

@@ -1,10 +1,15 @@
 import click
 import os
+from pathlib import Path
+import shutil
 import subprocess
+import tempfile
 import time
 
-from pathlib import Path
+import yaml
+
 from prefect import config
+from prefect.utilities.configuration import set_temporary_config
 
 
 def make_env(fname=None):
@@ -13,7 +18,9 @@ def make_env(fname=None):
     PREFECT_ENV = dict(
         DB_CONNECTION_URL=config.server.database.connection_url.replace(
             "localhost", "postgres"
-        )
+        ),
+        GRAPHQL_HOST_PORT=config.server.graphql.host_port,
+        UI_HOST_PORT=config.server.ui.host_port,
     )
 
     APOLLO_ENV = dict(
@@ -29,15 +36,17 @@ def make_env(fname=None):
         PREFECT_API_HEALTH_URL="http://graphql:{port}/health".format(
             port=config.server.graphql.port
         ),
+        APOLLO_HOST_PORT=config.server.host_port,
     )
 
     POSTGRES_ENV = dict(
+        POSTGRES_HOST_PORT=config.server.database.host_port,
         POSTGRES_USER=config.server.database.username,
         POSTGRES_PASSWORD=config.server.database.password,
         POSTGRES_DB=config.server.database.name,
     )
 
-    HASURA_ENV = dict()
+    HASURA_ENV = dict(HASURA_HOST_PORT=config.server.hasura.host_port)
 
     ENV = os.environ.copy()
     ENV.update(**PREFECT_ENV, **APOLLO_ENV, **POSTGRES_ENV, **HASURA_ENV)
@@ -57,7 +66,7 @@ def make_env(fname=None):
 @click.group(hidden=True)
 def server():
     """
-    Commands for interacting with the Prefect Server
+    Commands for interacting with the Prefect Core server
 
     \b
     Usage:
@@ -81,28 +90,181 @@ def server():
     help="The server image versions to use (for example, '0.10.0' or 'master')",
     # TODO: update this default to use prefect.__version__ logic
     default="latest",
+    hidden=True,
 )
 @click.option(
     "--skip-pull",
     help="Pass this flag to skip pulling new images (if available)",
     is_flag=True,
+    hidden=True,
 )
 @click.option(
     "--no-upgrade",
     "-n",
     help="Pass this flag to avoid running a database upgrade when the database spins up",
     is_flag=True,
+    hidden=True,
+)
+@click.option(
+    "--no-ui",
+    "-u",
+    help="Pass this flag to avoid starting the UI",
+    is_flag=True,
+    hidden=True,
+)
+@click.option(
+    "--postgres-port",
+    help="The port used to serve Postgres",
+    default=config.server.database.host_port,
+    type=str,
+    hidden=True,
+)
+@click.option(
+    "--hasura-port",
+    help="The port used to serve Hasura",
+    default=config.server.hasura.host_port,
+    type=str,
+    hidden=True,
+)
+@click.option(
+    "--graphql-port",
+    help="The port used to serve the GraphQL API",
+    default=config.server.graphql.host_port,
+    type=str,
+    hidden=True,
+)
+@click.option(
+    "--ui-port",
+    help="The port used to serve the UI",
+    default=config.server.ui.host_port,
+    type=str,
+    hidden=True,
+)
+@click.option(
+    "--server-port",
+    help="The port used to serve the Core server",
+    default=config.server.host_port,
+    type=str,
+    hidden=True,
+)
+@click.option(
+    "--no-postgres-port",
+    help="Disable port map of Postgres to host",
+    is_flag=True,
+    hidden=True,
 )
 @click.option(
-    "--no-ui", "-u", help="Pass this flag to avoid starting the UI", is_flag=True,
+    "--no-hasura-port",
+    help="Disable port map of Hasura to host",
+    is_flag=True,
+    hidden=True,
 )
-def start(version, skip_pull, no_upgrade, no_ui):
+@click.option(
+    "--no-graphql-port",
+    help="Disable port map of the GraphqlAPI to host",
+    is_flag=True,
+    hidden=True,
+)
+@click.option(
+    "--no-ui-port", help="Disable port map of the UI to host", is_flag=True, hidden=True
+)
+@click.option(
+    "--no-server-port",
+    help="Disable port map of the Core server to host",
+    is_flag=True,
+    hidden=True,
+)
+def start(
+    version,
+    skip_pull,
+    no_upgrade,
+    no_ui,
+    postgres_port,
+    hasura_port,
+    graphql_port,
+    ui_port,
+    server_port,
+    no_postgres_port,
+    no_hasura_port,
+    no_graphql_port,
+    no_ui_port,
+    no_server_port,
+):
     """
-    This command spins up all infrastructure and services for Prefect Server
+    This command spins up all infrastructure and services for the Prefect Core server
+
+    \b
+    Options:
+        --version, -v   TEXT    The server image versions to use (for example, '0.10.0' or 'master')
+                                Defaults to 'latest'
+        --skip-pull             Flag to skip pulling new images (if available)
+        --no-upgrade, -n        Flag to avoid running a database upgrade when the database spins up
+        --no-ui, -u             Flag to avoid starting the UI
+
+    \b
+        --postgres-port TEXT    Port used to serve Postgres, defaults to '5432'
+        --hasura-port   TEXT    Port used to serve Hasura, defaults to '3001'
+        --graphql-port  TEXT    Port used to serve the GraphQL API, defaults to '4001'
+        --ui-port       TEXT    Port used to serve the UI, defaults to '8080'
+        --server-port   TEXT    Port used to serve the Core server, defaults to '4200'
+
+    \b
+        --no-postgres-port      Disable port map of Postgres to host
+        --no-hasura-port        Disable port map of Hasura to host
+        --no-graphql-port       Disable port map of the GraphQL API to host
+        --no-ui-port            Disable port map of the UI to host
+        --no-server-port        Disable port map of the Core server to host
     """
+
     docker_dir = Path(__file__).parents[0]
+    compose_dir_path = docker_dir
+
+    # Remove port mappings if specified
+    if (
+        no_postgres_port
+        or no_hasura_port
+        or no_graphql_port
+        or no_ui_port
+        or no_server_port
+    ):
+        temp_dir = tempfile.gettempdir()
+        temp_path = os.path.join(temp_dir, "docker-compose.yml")
+        shutil.copy2(os.path.join(docker_dir, "docker-compose.yml"), temp_path)
+
+        with open(temp_path, "r") as file:
+            y = yaml.load(file)
+
+            if no_postgres_port:
+                del y["services"]["postgres"]["ports"]
+
+            if no_hasura_port:
+                del y["services"]["hasura"]["ports"]
+
+            if no_graphql_port:
+                del y["services"]["graphql"]["ports"]
 
-    env = make_env()
+            if no_ui_port:
+                del y["services"]["ui"]["ports"]
+
+            if no_server_port:
+                del y["services"]["apollo"]["ports"]
+
+        with open(temp_path, "w") as f:
+            y = yaml.dump(y, f)
+
+        compose_dir_path = temp_dir
+
+    # Temporary config set for port allocation
+    with set_temporary_config(
+        {
+            "server.database.host_port": postgres_port,
+            "server.hasura.host_port": hasura_port,
+            "server.graphql.host_port": graphql_port,
+            "server.ui.host_port": ui_port,
+            "server.host_port": server_port,
+        }
+    ):
+        env = make_env()
 
     if "PREFECT_SERVER_TAG" not in env:
         env.update(PREFECT_SERVER_TAG=version)
@@ -117,12 +279,14 @@ def start(version, skip_pull, no_upgrade, no_ui):
     proc = None
     try:
         if not skip_pull:
-            subprocess.check_call(["docker-compose", "pull"], cwd=docker_dir, env=env)
+            subprocess.check_call(
+                ["docker-compose", "pull"], cwd=compose_dir_path, env=env
+            )
 
         cmd = ["docker-compose", "up"]
         if no_ui:
             cmd += ["--scale", "ui=0"]
-        proc = subprocess.Popen(cmd, cwd=docker_dir, env=env)
+        proc = subprocess.Popen(cmd, cwd=compose_dir_path, env=env)
         while True:
             time.sleep(0.5)
     except:
@@ -131,7 +295,9 @@ def start(version, skip_pull, no_upgrade, no_ui):
             fg="white",
             bg="red",
         )
-        subprocess.check_output(["docker-compose", "down"], cwd=docker_dir, env=env)
+        subprocess.check_output(
+            ["docker-compose", "down"], cwd=compose_dir_path, env=env
+        )
         if proc:
             proc.kill()
         raise

src/prefect/config.toml

@@ -9,12 +9,13 @@ backend = "cloud"
 [server]
 host = "http://localhost"
 port = "4200"
+host_port = "4200"
 endpoint = "${server.host}:${server.port}"
 
     [server.database]
-
     host = "localhost"
-    port = 5432
+    port = "5432"
+    host_port = "5432"
     name = "prefect_server"
     username = "prefect"
     # set to "" to generate a random password each time the database starts
@@ -23,14 +24,15 @@ endpoint = "${server.host}:${server.port}"
 
     [server.graphql]
     host = "0.0.0.0"
-    port = 4201
+    port = "4201"
+    host_port = "4201"
     debug = false
     path = "/graphql/"
 
     [server.hasura]
-
     host = "localhost"
-    port = 3000
+    port = "3000"
+    host_port = "3000"
     admin_secret = "" # a string. One will be automatically generated if not provided.
     claims_namespace = "hasura-claims"
     graphql_url = "http://${server.hasura.host}:${server.hasura.port}/v1alpha1/graphql"
@@ -40,6 +42,7 @@ endpoint = "${server.host}:${server.port}"
     [server.ui]
     host = "http://localhost"
     port = "8080"
+    host_port = "8080"
     endpoint = "${server.ui.host}:${server.ui.port}"
 
 [cloud]