refactor logic across the codebase to make use of the Docker client u…

…tils (#4423)

Dockerfile

@@ -33,7 +33,7 @@ RUN mkdir -p localstack/utils/kinesis/ && mkdir -p localstack/services/ && \
 ADD localstack/constants.py localstack/config.py localstack/
 ADD localstack/services/install.py localstack/services/
 ADD localstack/services/cloudformation/deployment_utils.py localstack/services/cloudformation/deployment_utils.py
-ADD localstack/utils/common.py localstack/utils/bootstrap.py localstack/utils/
+ADD localstack/utils/common.py localstack/utils/bootstrap.py localstack/utils/docker.py localstack/utils/run.py localstack/utils/
 ADD localstack/utils/aws/ localstack/utils/aws/
 ADD localstack/utils/kinesis/ localstack/utils/kinesis/
 ADD localstack/utils/analytics/ localstack/utils/analytics/

docker-compose.yml

@@ -11,12 +11,10 @@ services:
       - "127.0.0.1:443:443"
       - "127.0.0.1:4566:4566"
       - "127.0.0.1:4571:4571"
-      - "127.0.0.1:${PORT_WEB_UI-8080}:${PORT_WEB_UI-8080}"
     environment:
       - SERVICES=${SERVICES- }
       - DEBUG=${DEBUG- }
       - DATA_DIR=${DATA_DIR- }
-      - PORT_WEB_UI=${PORT_WEB_UI- }
       - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR- }
       - LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY- }
       - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- }

localstack/config.py

@@ -18,7 +18,6 @@
     DEFAULT_DEVELOP_PORT,
     DEFAULT_LAMBDA_CONTAINER_REGISTRY,
     DEFAULT_PORT_EDGE,
-    DEFAULT_PORT_WEB_UI,
     DEFAULT_SERVICE_PORTS,
     FALSE_STRINGS,
     LOCALHOST,
@@ -181,9 +180,6 @@ def is_env_not_false(env_var_name):
 # command used to run Docker containers (e.g., set to "sudo docker" to run as sudo)
 DOCKER_CMD = os.environ.get("DOCKER_CMD", "").strip() or "docker"
 
-# whether to start the web API
-START_WEB = os.environ.get("START_WEB", "").strip() in TRUE_STRINGS
-
 # whether to forward edge requests in-memory (instead of via proxy servers listening on backend ports)
 # TODO: this will likely become the default and may get removed in the future
 FORWARD_EDGE_INMEM = True
@@ -194,10 +190,6 @@ def is_env_not_false(env_var_name):
 # fallback port for non-SSL HTTP edge service (in case HTTPS edge service cannot be used)
 EDGE_PORT_HTTP = int(os.environ.get("EDGE_PORT_HTTP") or 0)
 
-# port of Web UI
-PORT_WEB_UI = int(os.environ.get("PORT_WEB_UI", "").strip() or DEFAULT_PORT_WEB_UI)
-PORT_WEB_UI_SSL = PORT_WEB_UI + 1
-
 # IP of the docker bridge used to enable access between containers
 DOCKER_BRIDGE_IP = os.environ.get("DOCKER_BRIDGE_IP", "").strip()
 
@@ -296,10 +288,8 @@ def is_linux():
     "DEBUG",
     "KINESIS_ERROR_PROBABILITY",
     "DYNAMODB_ERROR_PROBABILITY",
-    "PORT_WEB_UI",
     "DYNAMODB_READ_ERROR_PROBABILITY",
     "DYNAMODB_WRITE_ERROR_PROBABILITY",
-    "START_WEB",
     "DOCKER_BRIDGE_IP",
     "DEFAULT_REGION",
     "LAMBDA_JAVA_OPTS",

localstack/constants.py

@@ -19,7 +19,6 @@
 
 # backend service ports, for services that are behind a proxy (counting down from 4566)
 DEFAULT_PORT_EDGE = 4566
-DEFAULT_PORT_WEB_UI = 8080
 
 # host name for localhost
 LOCALHOST = "localhost"

localstack/services/awslambda/lambda_api.py

@@ -36,7 +36,6 @@
 from localstack.utils.aws.aws_models import CodeSigningConfig, LambdaFunction
 from localstack.utils.common import (
     TMP_FILES,
-    FuncThread,
     ensure_readable,
     first_char_to_lower,
     is_zip_file,
@@ -62,6 +61,7 @@
 )
 from localstack.utils.docker import DOCKER_CLIENT
 from localstack.utils.http_utils import parse_chunked_data
+from localstack.utils.run import FuncThread
 
 # logger
 LOG = logging.getLogger(__name__)

localstack/services/awslambda/lambda_executors.py

@@ -36,8 +36,6 @@
 from localstack.utils.common import (
     TMP_FILES,
     CaptureOutput,
-    FuncThread,
-    get_docker_container_names,
     get_free_tcp_port,
     in_docker,
     json_safe,
@@ -50,7 +48,8 @@
     to_bytes,
     to_str,
 )
-from localstack.utils.docker import DOCKER_CLIENT, ContainerException
+from localstack.utils.docker import DOCKER_CLIENT, ContainerException, PortMappings
+from localstack.utils.run import FuncThread
 
 # constants
 LAMBDA_EXECUTOR_JAR = INSTALL_PATH_LOCALSTACK_FAT_JAR
@@ -151,7 +150,7 @@ def rm_docker_container(container_name_or_id, check_existence=False, safe=False)
     # TODO: remove method / move to docker module
     if not container_name_or_id:
         return
-    if check_existence and container_name_or_id not in get_docker_container_names():
+    if check_existence and container_name_or_id not in DOCKER_CLIENT.get_running_container_names():
         # TODO: check names as well as container IDs!
         return
     try:
@@ -829,7 +828,7 @@ def execute_in_container(
 
         additional_flags = docker_flags or ""
         dns = config.LAMBDA_DOCKER_DNS
-        docker_java_ports = bootstrap.PortMappings()
+        docker_java_ports = PortMappings()
         if Util.debug_java_port:
             docker_java_ports.add(Util.debug_java_port)
         docker_image = Util.docker_image_for_lambda(func_details)

localstack/services/edge.py

@@ -13,7 +13,6 @@
 from localstack.constants import (
     HEADER_LOCALSTACK_EDGE_URL,
     HEADER_LOCALSTACK_REQUEST_URL,
-    HEADER_LOCALSTACK_TARGET,
     LOCALHOST,
     LOCALHOST_IP,
     LOCALSTACK_ROOT_FOLDER,
@@ -284,7 +283,6 @@ def get_api_from_headers(headers, method=None, path=None, data=None):
     if not auth_header and "." not in host:
         return result[0], result[1], path, host
 
-    ls_target = headers.get(HEADER_LOCALSTACK_TARGET, "")
     path = path or "/"
 
     # https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
@@ -324,8 +322,6 @@ def get_api_from_headers(headers, method=None, path=None, data=None):
     elif target.startswith("DynamoDBStreams") or host.startswith("streams.dynamodb."):
         # Note: DDB streams requests use ../dynamodb/.. auth header, hence we also need to update result_before
         result = result_before = "dynamodbstreams", config.PORT_DYNAMODBSTREAMS
-    elif ls_target == "web":
-        result = "web", config.PORT_WEB_UI
     elif result[0] == "EventBridge" or target.startswith("AWSEvents"):
         result = "events", config.PORT_EVENTS
     elif target.startswith("ResourceGroupsTaggingAPI_"):

localstack/services/events/scheduler.py

@@ -3,7 +3,8 @@
 
 from crontab import CronTab
 
-from localstack.utils.common import FuncThread, short_uid
+from localstack.utils.common import short_uid
+from localstack.utils.run import FuncThread
 
 LOG = logging.getLogger(__name__)
 

localstack/services/generic_proxy.py

@@ -79,10 +79,6 @@
 ALLOWED_CORS_ORIGINS = [
     "https://app.localstack.cloud",
     "http://app.localstack.cloud",
-    f"http://localhost:{config.PORT_WEB_UI}",
-    f"https://localhost:{config.PORT_WEB_UI_SSL}",
-    f"http://127.0.0.1:{config.PORT_WEB_UI}",
-    f"https://127.0.0.1:{config.PORT_WEB_UI_SSL}",
     f"https://localhost:{config.EDGE_PORT}",
     f"http://localhost:{config.EDGE_PORT}",
     f"https://localhost.localstack.cloud:{config.EDGE_PORT}",

localstack/services/install.py

@@ -8,6 +8,7 @@
 import sys
 import tempfile
 import time
+from pathlib import Path
 
 import requests
 
@@ -33,6 +34,7 @@
     THUNDRA_JAVA_AGENT_VERSION,
 )
 from localstack.utils import bootstrap
+from localstack.utils.docker import DOCKER_CLIENT
 
 if __name__ == "__main__":
     bootstrap.bootstrap_installation()
@@ -296,20 +298,24 @@ def install_stepfunctions_local():
         # TODO: works only when running on the host, outside of Docker -> add a fallback if running in Docker?
         log_install_msg("Step Functions")
         mkdir(INSTALL_DIR_STEPFUNCTIONS)
-        run("{dc} pull {img}".format(dc=config.DOCKER_CMD, img=IMAGE_NAME_SFN_LOCAL))
+        DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
         docker_name = "tmp-ls-sfn"
-        run(
-            ("{dc} run --name={dn} --entrypoint= -d --rm {img} sleep 15").format(
-                dc=config.DOCKER_CMD, dn=docker_name, img=IMAGE_NAME_SFN_LOCAL
-            )
+        DOCKER_CLIENT.run_container(
+            IMAGE_NAME_SFN_LOCAL,
+            remove=True,
+            entrypoint="",
+            name=docker_name,
+            detach=True,
+            command=["sleep", "15"],
         )
         time.sleep(5)
-        run(
-            "{dc} cp {dn}:/home/stepfunctionslocal/ {tgt}".format(
-                dc=config.DOCKER_CMD, dn=docker_name, tgt=INSTALL_DIR_INFRA
-            )
+        DOCKER_CLIENT.copy_from_container(
+            docker_name, local_path=INSTALL_DIR_INFRA, container_path="/home/stepfunctionslocal/"
         )
-        run("mv %s/stepfunctionslocal/*.jar %s" % (INSTALL_DIR_INFRA, INSTALL_DIR_STEPFUNCTIONS))
+
+        path = Path(f"{INSTALL_DIR_INFRA}/stepfunctionslocal/")
+        for file in path.glob("*.jar"):
+            file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
         rm_rf("%s/stepfunctionslocal" % INSTALL_DIR_INFRA)
     # apply patches
     patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, SFN_PATCH_CLASS)

localstack/utils/analytics/event_publisher.py

@@ -6,9 +6,10 @@
 
 from localstack import config
 from localstack.constants import API_ENDPOINT
-from localstack.utils.common import FuncThread, JsonObject, get_or_create_file
+from localstack.utils.common import JsonObject, get_or_create_file
 from localstack.utils.common import safe_requests as requests
 from localstack.utils.common import save_file, short_uid, timestamp
+from localstack.utils.run import FuncThread
 
 PROCESS_ID = short_uid()
 MACHINE_ID = None

localstack/utils/async_utils.py

@@ -5,7 +5,8 @@
 from contextvars import copy_context
 
 from localstack.utils import common
-from localstack.utils.common import TMP_THREADS, FuncThread, start_worker_thread
+from localstack.utils.common import TMP_THREADS, start_worker_thread
+from localstack.utils.run import FuncThread
 
 # reference to named event loop instances
 EVENT_LOOPS = {}