Merge branch 'master' into ec2-redundant-code

.github/workflows/asf-updates.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           source .venv/bin/activate
           # explicitly perform an unsafe fix to remove unused imports in the generated ASF APIs
-          ruff check --select F401 --unsafe-fixes --fix . --config "lint.ignore-init-module-imports = false"
+          ruff check --select F401 --unsafe-fixes --fix . --config "lint.preview = true"
           make format-modified
 
       - name: Check for changes

localstack/aws/api/ec2/__init__.py

@@ -1,7 +1,14 @@
 from datetime import datetime
 from typing import List, Optional, TypedDict
 
-from localstack.aws.api import RequestContext, ServiceRequest, handler
+from localstack.aws.api import (
+    RequestContext,
+    ServiceRequest,
+    handler,
+)
+from localstack.aws.api import (
+    ServiceException as ServiceException,
+)
 
 AddressMaxResults = int
 AllocationId = str
@@ -2552,6 +2559,11 @@ class PermissionGroup(str):
     all = "all"
 
 
+class PhcSupport(str):
+    unsupported = "unsupported"
+    supported = "supported"
+
+
 class PlacementGroupState(str):
     pending = "pending"
     available = "available"
@@ -11534,6 +11546,7 @@ class InstanceTypeInfo(TypedDict, total=False):
     NitroTpmInfo: Optional[NitroTpmInfo]
     MediaAcceleratorInfo: Optional[MediaAcceleratorInfo]
     NeuronInfo: Optional[NeuronInfo]
+    PhcSupport: Optional[PhcSupport]
 
 
 InstanceTypeInfoList = List[InstanceTypeInfo]

localstack/aws/api/sqs/__init__.py

@@ -14,6 +14,7 @@
 
 
 class MessageSystemAttributeName(str):
+    All = "All"
     SenderId = "SenderId"
     SentTimestamp = "SentTimestamp"
     ApproximateReceiveCount = "ApproximateReceiveCount"
@@ -457,6 +458,7 @@ class MessageSystemAttributeValue(TypedDict, total=False):
     MessageSystemAttributeNameForSends, MessageSystemAttributeValue
 ]
 MessageList = List[Message]
+MessageSystemAttributeList = List[MessageSystemAttributeName]
 
 
 class PurgeQueueRequest(ServiceRequest):
@@ -466,6 +468,7 @@ class PurgeQueueRequest(ServiceRequest):
 class ReceiveMessageRequest(ServiceRequest):
     QueueUrl: String
     AttributeNames: Optional[AttributeNameList]
+    MessageSystemAttributeNames: Optional[MessageSystemAttributeList]
     MessageAttributeNames: Optional[MessageAttributeNameList]
     MaxNumberOfMessages: Optional[NullableInteger]
     VisibilityTimeout: Optional[NullableInteger]
@@ -705,6 +708,7 @@ def receive_message(
         context: RequestContext,
         queue_url: String,
         attribute_names: AttributeNameList = None,
+        message_system_attribute_names: MessageSystemAttributeList = None,
         message_attribute_names: MessageAttributeNameList = None,
         max_number_of_messages: NullableInteger = None,
         visibility_timeout: NullableInteger = None,

localstack/services/events/models.py

@@ -1,12 +1,14 @@
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Optional, TypeAlias, TypedDict
 
 from localstack.aws.api.core import ServiceException
 from localstack.aws.api.events import (
     Arn,
     CreatedBy,
     EventBusName,
     EventPattern,
+    EventResourceList,
+    EventSourceName,
     ManagedBy,
     RoleArn,
     RuleDescription,
@@ -102,3 +104,18 @@ class InvalidEventPatternException(Exception):
     def __init__(self, reason=None, message=None) -> None:
         self.reason = reason
         self.message = message or f"Event pattern is not valid. Reason: {reason}"
+
+
+class FormattedEvent(TypedDict):
+    version: str
+    id: str
+    detail_type: Optional[str]  # key "detail-type" is automatically interpreted as detail_type
+    source: Optional[EventSourceName]
+    account: str
+    time: str
+    region: str
+    resources: Optional[EventResourceList]
+    detail: dict[str, str | dict]
+
+
+TransformedEvent: TypeAlias = FormattedEvent | dict | str

localstack/services/events/provider.py

@@ -59,6 +59,7 @@
     EventBus,
     EventBusDict,
     EventsStore,
+    FormattedEvent,
     Rule,
     RuleDict,
     TargetDict,
@@ -125,7 +126,7 @@ def validate_event(event: PutEventsRequestEntry) -> None | PutEventsResultEntry:
         }
 
 
-def format_event(event: PutEventsRequestEntry, region: str, account_id: str) -> dict:
+def format_event(event: PutEventsRequestEntry, region: str, account_id: str) -> FormattedEvent:
     # See https://docs.aws.amazon.com/AmazonS3/latest/userguide/ev-events.html
     formatted_event = {
         "version": "0",
@@ -685,7 +686,7 @@ def _process_entries(
                     for target in rule.targets.values():
                         target_sender = self._target_sender_store[target["Arn"]]
                         try:
-                            target_sender.send_event(event)
+                            target_sender.process_event(event)
                             processed_entries.append({"EventId": event["id"]})
                         except Exception as error:
                             processed_entries.append(

localstack/services/events/target.py

@@ -7,23 +7,32 @@
 
 from localstack.aws.api.events import (
     Arn,
-    PutEventsRequestEntry,
     Target,
+    TargetInputPath,
 )
 from localstack.aws.connect import connect_to
+from localstack.services.events.models import FormattedEvent, TransformedEvent
 from localstack.utils import collections
 from localstack.utils.aws.arns import (
     extract_service_from_arn,
     firehose_name,
     sqs_queue_url_for_arn,
 )
 from localstack.utils.aws.client_types import ServicePrincipal
+from localstack.utils.json import extract_jsonpath
 from localstack.utils.strings import to_bytes
 from localstack.utils.time import now_utc
 
 LOG = logging.getLogger(__name__)
 
 
+def transform_event_with_target_input_path(
+    input_path: TargetInputPath, event: FormattedEvent
+) -> TransformedEvent:
+    formatted_event = extract_jsonpath(event, input_path)
+    return formatted_event
+
+
 class TargetSender(ABC):
     def __init__(
         self,
@@ -54,9 +63,15 @@ def client(self):
         return self._client
 
     @abstractmethod
-    def send_event(self, event: PutEventsRequestEntry):
+    def send_event(self, event: FormattedEvent | TransformedEvent):
         pass
 
+    def process_event(self, event: FormattedEvent):
+        """Processes the event and send it to the target."""
+        if input_path := self.target.get("InputPath"):
+            event = transform_event_with_target_input_path(input_path, event)
+        self.send_event(event)
+
     def _validate_input(self, target: Target):
         """Provide a default implementation that does nothing if no specific validation is needed."""
         # TODO add For Lambda and Amazon SNS resources, EventBridge relies on resource-based policies.

localstack/services/sqs/provider.py

@@ -45,6 +45,7 @@
     MessageAttributeNameList,
     MessageBodyAttributeMap,
     MessageBodySystemAttributeMap,
+    MessageSystemAttributeList,
     MessageSystemAttributeName,
     NullableInteger,
     PurgeQueueInProgress,
@@ -1183,13 +1184,15 @@ def receive_message(
         context: RequestContext,
         queue_url: String,
         attribute_names: AttributeNameList = None,
+        message_system_attribute_names: MessageSystemAttributeList = None,
         message_attribute_names: MessageAttributeNameList = None,
         max_number_of_messages: NullableInteger = None,
         visibility_timeout: NullableInteger = None,
         wait_time_seconds: NullableInteger = None,
         receive_request_attempt_id: String = None,
         **kwargs,
     ) -> ReceiveMessageResult:
+        # TODO add support for message_system_attribute_names
         queue = self._resolve_queue(context, queue_url=queue_url)
 
         if wait_time_seconds is None:

pyproject.toml

@@ -48,9 +48,9 @@ Issues = "https://github.com/localstack/localstack/issues"
 # minimal required to actually run localstack on the host for services natively implemented in python
 base-runtime = [
     # pinned / updated by ASF update action
-    "boto3==1.34.98",
+    "boto3==1.34.103",
     # pinned / updated by ASF update action
-    "botocore==1.34.98",
+    "botocore==1.34.103",
     "awscrt>=0.13.14",
     "cbor2>=5.2.0",
     "dnspython>=1.16.0",
@@ -76,7 +76,7 @@ base-runtime = [
 runtime = [
     "localstack-core[base-runtime]",
     # pinned / updated by ASF update action
-    "awscli==1.32.98",
+    "awscli==1.32.103",
     "airspeed-ext>=0.6.3",
     "amazon_kclpy>=2.0.6,!=2.1.0,!=2.1.4",
     # antlr4-python3-runtime: exact pin because antlr4 runtime is tightly coupled to the generated parser code
@@ -133,7 +133,7 @@ typehint = [
     # typehint is an optional extension of the dev dependencies
     "localstack-core[dev]",
     # pinned / updated by ASF update action
-    "boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pipes,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,wafv2,xray]==1.34.98",
+    "boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pipes,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,wafv2,xray]==1.34.103",
 ]
 
 [tool.setuptools]

requirements-base-runtime.txt

@@ -14,9 +14,9 @@ blinker==1.8.2
     # via
     #   flask
     #   quart
-boto3==1.34.98
+boto3==1.34.103
     # via localstack-core (pyproject.toml)
-botocore==1.34.98
+botocore==1.34.103
     # via
     #   boto3
     #   localstack-core (pyproject.toml)

requirements-dev.txt

@@ -45,7 +45,7 @@ aws-sam-translator==1.88.0
     #   localstack-core
 aws-xray-sdk==2.13.0
     # via moto-ext
-awscli==1.32.98
+awscli==1.32.103
     # via localstack-core
 awscrt==0.20.9
     # via localstack-core
@@ -55,12 +55,12 @@ blinker==1.8.2
     #   quart
 boto==2.49.0
     # via amazon-kclpy
-boto3==1.34.98
+boto3==1.34.103
     # via
     #   aws-sam-translator
     #   localstack-core
     #   moto-ext
-botocore==1.34.98
+botocore==1.34.103
     # via
     #   aws-xray-sdk
     #   awscli
@@ -214,7 +214,7 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-joserfc==0.9.0
+joserfc==0.10.0
     # via moto-ext
 jpype1==1.5.0
     # via localstack-core
@@ -329,7 +329,7 @@ ply==3.11
     #   jsonpath-ng
     #   jsonpath-rw
     #   pandoc
-pre-commit==3.7.0
+pre-commit==3.7.1
     # via localstack-core (pyproject.toml)
 priority==1.3.0
     # via
@@ -509,7 +509,7 @@ urllib3==2.2.1
     #   opensearch-py
     #   requests
     #   responses
-virtualenv==20.26.1
+virtualenv==20.26.2
     # via pre-commit
 websocket-client==1.8.0
     # via

requirements-runtime.txt

@@ -33,7 +33,7 @@ aws-sam-translator==1.88.0
     #   localstack-core (pyproject.toml)
 aws-xray-sdk==2.13.0
     # via moto-ext
-awscli==1.32.98
+awscli==1.32.103
     # via localstack-core (pyproject.toml)
 awscrt==0.20.9
     # via localstack-core
@@ -43,12 +43,12 @@ blinker==1.8.2
     #   quart
 boto==2.49.0
     # via amazon-kclpy
-boto3==1.34.98
+boto3==1.34.103
     # via
     #   aws-sam-translator
     #   localstack-core
     #   moto-ext
-botocore==1.34.98
+botocore==1.34.103
     # via
     #   aws-xray-sdk
     #   awscli
@@ -161,7 +161,7 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-joserfc==0.9.0
+joserfc==0.10.0
     # via moto-ext
 jpype1==1.5.0
     # via localstack-core (pyproject.toml)

requirements-test.txt

@@ -45,7 +45,7 @@ aws-sam-translator==1.88.0
     #   localstack-core
 aws-xray-sdk==2.13.0
     # via moto-ext
-awscli==1.32.98
+awscli==1.32.103
     # via localstack-core
 awscrt==0.20.9
     # via localstack-core
@@ -55,12 +55,12 @@ blinker==1.8.2
     #   quart
 boto==2.49.0
     # via amazon-kclpy
-boto3==1.34.98
+boto3==1.34.103
     # via
     #   aws-sam-translator
     #   localstack-core
     #   moto-ext
-botocore==1.34.98
+botocore==1.34.103
     # via
     #   aws-xray-sdk
     #   awscli
@@ -198,7 +198,7 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-joserfc==0.9.0
+joserfc==0.10.0
     # via moto-ext
 jpype1==1.5.0
     # via localstack-core