Add lambda config to ignore architecture (#7890)

localstack/config.py

@@ -645,6 +645,10 @@ def in_docker():
 # additional flags passed to Lambda Docker run/create commands
 LAMBDA_DOCKER_FLAGS = os.environ.get("LAMBDA_DOCKER_FLAGS", "").strip()
 
+# Enable this flag to run cross-platform compatible lambda functions natively (i.e., Docker selects architecture) and
+# ignore the AWS architectures (i.e., x86_64, arm64) configured for the lambda function.
+LAMBDA_IGNORE_ARCHITECTURE = is_env_true("LAMBDA_IGNORE_ARCHITECTURE")
+
 # prebuild images before execution? Increased cold start time on the tradeoff of increased time until lambda is ACTIVE
 LAMBDA_PREBUILD_IMAGES = is_env_true("LAMBDA_PREBUILD_IMAGES")
 

localstack/services/awslambda/invocation/docker_runtime_executor.py

@@ -50,7 +50,7 @@
 COPY code/ /var/task
 """
 
-PULLED_IMAGES: set[str] = set()
+PULLED_IMAGES: set[(str, DockerPlatform)] = set()
 
 HOT_RELOADING_ENV_VARIABLE = "LOCALSTACK_HOT_RELOADING_PATHS"
 
@@ -64,15 +64,17 @@
 )
 
 
-def docker_platform(lambda_architecture: Architecture) -> DockerPlatform:
+def docker_platform(lambda_architecture: Architecture) -> DockerPlatform | None:
     """
     Convert an AWS Lambda architecture into a Docker platform flag. Examples:
     * docker_platform("x86_64") == "linux/amd64"
     * docker_platform("arm64") == "linux/arm64"
 
     :param lambda_architecture: the instruction set that the function supports
-    :return: Docker platform in the format ``os[/arch[/variant]]``
+    :return: Docker platform in the format ``os[/arch[/variant]]`` or None if configured to ignore the architecture
     """
+    if config.LAMBDA_IGNORE_ARCHITECTURE:
+        return None
     return ARCHITECTURE_PLATFORM_MAPPING[lambda_architecture]
 
 
@@ -378,9 +380,11 @@ def prepare_version(cls, function_version: FunctionVersion) -> None:
         if function_version.config.code:
             function_version.config.code.prepare_for_execution()
             image_name = resolver.get_image_for_runtime(function_version.config.runtime)
-            if image_name not in PULLED_IMAGES:
-                CONTAINER_CLIENT.pull_image(image_name)
-                PULLED_IMAGES.add(image_name)
+            platform = docker_platform(function_version.config.architectures[0])
+            # Pull image for a given platform upon function creation such that invocations do not time out.
+            if (image_name, platform) not in PULLED_IMAGES:
+                CONTAINER_CLIENT.pull_image(image_name, platform)
+                PULLED_IMAGES.add((image_name, platform))
             if config.LAMBDA_PREBUILD_IMAGES:
                 target_path = function_version.config.code.get_unzipped_code_location()
                 prepare_image(target_path, function_version)

localstack/testing/aws/lambda_utils.py

@@ -1,8 +1,8 @@
 import json
 import os
-import platform
 from typing import Literal
 
+from localstack.services.awslambda.lambda_api import use_docker
 from localstack.utils.common import to_str
 from localstack.utils.sync import ShortCircuitWaitException, retry
 from localstack.utils.testutil import get_lambda_log_events
@@ -117,6 +117,14 @@ def get_events():
     return retry(get_events, retries=retries, sleep_before=2)
 
 
+def is_old_local_executor() -> bool:
+    """Returns True if running in local executor mode and False otherwise.
+    The new provider ignores the LAMBDA_EXECUTOR flag and `not use_docker()` covers the fallback case if
+    the Docker socket is not available.
+    """
+    return is_old_provider() and not use_docker()
+
+
 def is_old_provider():
     return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
         "PROVIDER_OVERRIDE_LAMBDA"
@@ -127,7 +135,3 @@ def is_new_provider():
     return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
         "PROVIDER_OVERRIDE_LAMBDA"
     ) in ["asf", "v2"]
-
-
-def is_arm_compatible():
-    return platform.machine() == "arm64"

localstack/utils/platform.py

@@ -28,14 +28,21 @@ def is_redhat() -> bool:
     return "rhel" in load_file("/etc/os-release", "")
 
 
+class Arch(str):
+    """LocalStack standardised machine architecture names"""
+
+    amd64 = "amd64"
+    arm64 = "arm64"
+
+
 def standardized_arch(arch: str):
     """
     Returns LocalStack standardised machine architecture name.
     """
     if arch == "x86_64":
-        return "amd64"
+        return Arch.amd64
     if arch == "aarch64":
-        return "arm64"
+        return Arch.arm64
     return arch
 
 
@@ -47,6 +54,11 @@ def get_arch() -> str:
     return standardized_arch(arch)
 
 
+def is_arm_compatible() -> bool:
+    """Returns true if the current machine is compatible with ARM instructions and false otherwise."""
+    return get_arch() == Arch.arm64
+
+
 def get_os() -> str:
     if is_mac_os():
         return "osx"

tests/integration/awslambda/test_lambda.py

@@ -10,22 +10,24 @@
 import pytest
 from botocore.response import StreamingBody
 
+from localstack import config
 from localstack.aws.api.lambda_ import Architecture, Runtime
 from localstack.services.awslambda.lambda_api import use_docker
 from localstack.testing.aws.lambda_utils import (
     concurrency_update_done,
     get_invoke_init_type,
-    is_arm_compatible,
+    is_old_local_executor,
     is_old_provider,
     update_done,
 )
 from localstack.testing.aws.util import create_client_with_keys
 from localstack.testing.pytest.snapshot import is_aws
 from localstack.testing.snapshots.transformer import KeyValueBasedTransformer
 from localstack.testing.snapshots.transformer_utility import PATTERN_UUID
-from localstack.utils import files, testutil
+from localstack.utils import files, platform, testutil
 from localstack.utils.files import load_file
 from localstack.utils.http import safe_requests
+from localstack.utils.platform import is_arm_compatible, standardized_arch
 from localstack.utils.strings import short_uid, to_bytes, to_str
 from localstack.utils.sync import retry, wait_until
 from localstack.utils.testutil import create_lambda_archive
@@ -377,6 +379,99 @@ def test_runtime_introspection_arm(self, lambda_client, create_lambda_function,
         invoke_result = lambda_client.invoke(FunctionName=func_name)
         snapshot.match("invoke_runtime_arm_introspection", invoke_result)
 
+    @pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
+    @pytest.mark.skipif(
+        is_old_local_executor(),
+        reason="Monkey-patching of Docker flags is not applicable because no new container is spawned",
+    )
+    @pytest.mark.only_localstack
+    def test_ignore_architecture(
+        self, lambda_client, create_lambda_function, snapshot, monkeypatch
+    ):
+        """Test configuration to ignore lambda architecture by creating a lambda with non-native architecture."""
+        monkeypatch.setattr(config, "LAMBDA_IGNORE_ARCHITECTURE", True)
+
+        # Assumes that LocalStack runs on native Docker host architecture
+        # This assumption could be violated when using remote Lambda executors
+        native_arch = platform.get_arch()
+        non_native_architecture = (
+            Architecture.x86_64 if native_arch == "arm64" else Architecture.arm64
+        )
+        func_name = f"test_lambda_arch_{short_uid()}"
+        create_lambda_function(
+            func_name=func_name,
+            handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
+            runtime=Runtime.python3_9,
+            Architectures=[non_native_architecture],
+        )
+
+        invoke_result = lambda_client.invoke(FunctionName=func_name)
+        payload = json.loads(to_str(invoke_result["Payload"].read()))
+        lambda_arch = standardized_arch(payload.get("platform_machine"))
+        assert lambda_arch == native_arch
+
+    @pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
+    @pytest.mark.skipif(
+        not is_arm_compatible() and not is_aws(),
+        reason="ARM architecture not supported on this host",
+    )
+    @pytest.mark.aws_validated
+    def test_mixed_architecture(self, lambda_client, create_lambda_function):
+        """Test emulation and interaction of lambda functions with different architectures.
+        Limitation: only works on ARM hosts that support x86 emulation.
+        """
+        func_name = f"test_lambda_x86_{short_uid()}"
+        create_lambda_function(
+            func_name=func_name,
+            handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
+            runtime=Runtime.python3_9,
+            Architectures=[Architecture.x86_64],
+        )
+
+        invoke_result = lambda_client.invoke(FunctionName=func_name)
+        assert "FunctionError" not in invoke_result
+        payload = json.loads(invoke_result["Payload"].read())
+        assert payload.get("platform_machine") == "x86_64"
+
+        func_name_arm = f"test_lambda_arm_{short_uid()}"
+        create_lambda_function(
+            func_name=func_name_arm,
+            handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
+            runtime=Runtime.python3_9,
+            Architectures=[Architecture.arm64],
+        )
+
+        invoke_result_arm = lambda_client.invoke(FunctionName=func_name_arm)
+        assert "FunctionError" not in invoke_result_arm
+        payload_arm = json.loads(invoke_result_arm["Payload"].read())
+        assert payload_arm.get("platform_machine") == "aarch64"
+
+        v1_result = lambda_client.publish_version(FunctionName=func_name)
+        v1 = v1_result["Version"]
+
+        # assert version is available(!)
+        lambda_client.get_waiter(waiter_name="function_active_v2").wait(
+            FunctionName=func_name, Qualifier=v1
+        )
+
+        arm_v1_result = lambda_client.publish_version(FunctionName=func_name_arm)
+        arm_v1 = arm_v1_result["Version"]
+
+        # assert version is available(!)
+        lambda_client.get_waiter(waiter_name="function_active_v2").wait(
+            FunctionName=func_name_arm, Qualifier=arm_v1
+        )
+
+        invoke_result_2 = lambda_client.invoke(FunctionName=func_name, Qualifier=v1)
+        assert "FunctionError" not in invoke_result_2
+        payload_2 = json.loads(invoke_result_2["Payload"].read())
+        assert payload_2.get("platform_machine") == "x86_64"
+
+        invoke_result_arm_2 = lambda_client.invoke(FunctionName=func_name_arm, Qualifier=arm_v1)
+        assert "FunctionError" not in invoke_result_arm_2
+        payload_arm_2 = json.loads(invoke_result_arm_2["Payload"].read())
+        assert payload_arm_2.get("platform_machine") == "aarch64"
+
     @pytest.mark.skip_snapshot_verify(
         condition=is_old_provider, paths=["$..Payload", "$..LogResult"]
     )