diff --git a/.github/workflows/tests_real_aws.yml b/.github/workflows/tests_real_aws.yml index 835bcd28e39..3504641943d 100644 --- a/.github/workflows/tests_real_aws.yml +++ b/.github/workflows/tests_real_aws.yml @@ -42,4 +42,4 @@ jobs: env: MOTO_TEST_ALLOW_AWS_REQUEST: ${{ true }} run: | - pytest -sv tests/test_dynamodb/ tests/test_ec2/ tests/test_lakeformation/ tests/test_logs/ tests/test_ses/ tests/test_s3* tests/test_sns/ -m aws_verified + pytest -sv tests/test_dynamodb/ tests/test_ec2/ tests/test_iam/ tests/test_lakeformation/ tests/test_logs/ tests/test_sqs/ tests/test_ses/ tests/test_s3* tests/test_sns/ -m aws_verified diff --git a/.github/workflows/tests_terraform_examples.yml b/.github/workflows/tests_terraform_examples.yml index 751f98c064e..8dfa291b91a 100644 --- a/.github/workflows/tests_terraform_examples.yml +++ b/.github/workflows/tests_terraform_examples.yml @@ -39,3 +39,31 @@ jobs: sleep 30 terraform plan -detailed-exitcode terraform apply -destroy --auto-approve + + test_create_only: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: ["ec2"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python 3.8 + uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Start MotoServer + run: | + pip install build + python -m build + docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh & + python scripts/ci_wait_for_server.py + - name: Run tests + run: | + mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials + cd other_langs/terraform/${{ matrix.service }} + terraform init + terraform apply --auto-approve diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bffcde9579..03e188debbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,54 @@ Moto Changelog ============== +5.0.3 +----- +Docker Digest for 5.0.3: + + General: + * New configuration options for: + - Passing URL's through the proxy + - Configuring DOcker-less services in ServerMode + See http://docs.getmoto.org/en/latest/docs/configuration/index.html + + New Services: + * Route53Domains: + * delete_domain() + * list_domains() + * list_operations() + * register_domain() + * update_domain_nameservers() + + New Methods: + * CostExplorer: + * get_cost_and_usage() + + * ECR: + * get_registry_scanning_configuration() + + Miscellaneous: + * ApiGateway: update_usage_plan() now supports adding apiStages + * Athena: get_query_execution() now returns exact OutputLocation file + * Autoscaling: describe_auto_scaling_groups() now supports the filters-argument + * CloudFront: create_distribution() now supports CustomHeaders + * CloudFront: update_distribution() now handles updates to DistributionConfig correctly + * CloudFormation - Now supports creation and deletion of AWS::EMR::Cluster + * CloudFormation - Now supports creation and deletion of AWS::EMR::SecurityConfiguration + * CloudFormation - Now supports creation and deletion of AWS::EFS::AccessPoint + * CloudFormation - Now supports creation and deletion of AWS::EFS::FileSystem + * CloudFormation - Now supports creation and deletion of AWS::EMR::InstanceGroupConfig + * CloudFormation - Now supports deletion of AWS::Logs::LogGroup + * CloudFormation: delete_stack() now handles resource dependencies better + * CloudWatch: put_metric_data() now supports large (compressed) requests + * CognitoIDP: admin_initiate_auth() and respond_to_auth_challenge() now support SMS_MFA + * DynamoDB: transact_write_items() now raises ValidationException when putting and deleting the same item + * EC2: authorize_security_group_egress/_ingress now support the TagSpecifications-argument + * EC2: describe_security_group_rules() now supports Tag-filters + * S3: EventBridge notifications are now supported for ObjectCreated:POST/COPY/MULTIPART_UPLOAD and ObjectDeleted + * SNS: subscribe() now adds support the `$or`, `equals-ignore-case` and `suffix` features in a FilterPolicy + * SQS: send_message() should respect DelaySeconds of 0 + + 5.0.2 ----- Docker Digest for 5.0.2: _sha256:89cc6c764d714bf76e592a61f0c06fd142f672085e1dd3a53eb734aaeb4e14e2_ diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7e83f403440..a36e45472ba 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -735,7 +735,7 @@ ## ce
-21% implemented +20% implemented - [ ] create_anomaly_monitor - [ ] create_anomaly_subscription @@ -747,6 +747,7 @@ - [ ] get_anomalies - [ ] get_anomaly_monitors - [ ] get_anomaly_subscriptions +- [ ] get_approximate_usage_records - [X] get_cost_and_usage - [ ] get_cost_and_usage_with_resources - [ ] get_cost_categories @@ -2659,7 +2660,7 @@ ## ecr
-65% implemented +67% implemented - [ ] batch_check_layer_availability - [X] batch_delete_image @@ -3339,12 +3340,11 @@ ## firehose
-85% implemented +100% implemented - [X] create_delivery_stream - [X] delete_delivery_stream - [X] describe_delivery_stream -- [ ] get_kinesis_stream - [X] list_delivery_streams - [X] list_tags_for_delivery_stream - [X] put_record @@ -3354,7 +3354,6 @@ - [X] tag_delivery_stream - [X] untag_delivery_stream - [X] update_destination -- [ ] verify_resources_exist_for_tagris
## forecast @@ -4922,6 +4921,7 @@ - [ ] purchase_offering - [ ] reboot_input_device - [ ] reject_input_device_transfer +- [ ] restart_channel_pipelines - [X] start_channel - [ ] start_input_device - [ ] start_input_device_maintenance_window @@ -6393,6 +6393,46 @@ - [ ] update_traffic_policy_instance
+## route53domains +
+14% implemented + +- [ ] accept_domain_transfer_from_another_aws_account +- [ ] associate_delegation_signer_to_domain +- [ ] cancel_domain_transfer_to_another_aws_account +- [ ] check_domain_availability +- [ ] check_domain_transferability +- [X] delete_domain +- [ ] delete_tags_for_domain +- [ ] disable_domain_auto_renew +- [ ] disable_domain_transfer_lock +- [ ] disassociate_delegation_signer_from_domain +- [ ] enable_domain_auto_renew +- [ ] enable_domain_transfer_lock +- [ ] get_contact_reachability_status +- [ ] get_domain_detail +- [ ] get_domain_suggestions +- [ ] get_operation_detail +- [X] list_domains +- [X] list_operations +- [ ] list_prices +- [ ] list_tags_for_domain +- [ ] push_domain +- [X] register_domain +- [ ] reject_domain_transfer_from_another_aws_account +- [ ] renew_domain +- [ ] resend_contact_reachability_email +- [ ] resend_operation_authorization +- [ ] retrieve_domain_auth_code +- [ ] transfer_domain +- [ ] transfer_domain_to_another_aws_account +- [ ] update_domain_contact +- [ ] update_domain_contact_privacy +- [X] update_domain_nameservers +- [ ] update_tags_for_domain +- [ ] view_billing +
+ ## route53resolver
27% implemented @@ -6469,7 +6509,7 @@ ## s3
-66% implemented +67% implemented - [X] abort_multipart_upload - [X] complete_multipart_upload @@ -6567,6 +6607,8 @@ - [X] put_public_access_block - [ ] restore_object - [X] select_object_content +- [X] upload_file +- [X] upload_fileobj - [X] upload_part - [X] upload_part_copy - [ ] write_get_object_response @@ -7945,6 +7987,7 @@ - bedrock-runtime - billingconductor - braket +- chatbot - chime - chime-sdk-identity - chime-sdk-media-pipelines @@ -8122,7 +8165,6 @@ - route53-recovery-cluster - route53-recovery-control-config - route53-recovery-readiness -- route53domains - rum - s3outposts - sagemaker-a2i-runtime diff --git a/docs/docs/services/ce.rst b/docs/docs/services/ce.rst index bd609ebb311..ace02126480 100644 --- a/docs/docs/services/ce.rst +++ b/docs/docs/services/ce.rst @@ -38,6 +38,7 @@ ce - [ ] get_anomalies - [ ] get_anomaly_monitors - [ ] get_anomaly_subscriptions +- [ ] get_approximate_usage_records - [X] get_cost_and_usage There is no validation yet on any of the input parameters. diff --git a/docs/docs/services/cf.rst b/docs/docs/services/cf.rst index 86d7797bc64..730d31db198 100644 --- a/docs/docs/services/cf.rst +++ b/docs/docs/services/cf.rst @@ -113,7 +113,11 @@ Please let us know if you'd like support for a resource not yet listed here. +---------------------------------------+--------+--------+--------+----------------------------------------+ |AWS::ECS::TaskDefinition | x | x | | - [ ] TaskDefinitionArn | +---------------------------------------+--------+--------+--------+----------------------------------------+ - |AWS::EFS::FileSystem | x | x | x | - [ ] FileSystemId | + |AWS::EFS::AccessPoint | x | | | - [ ] AccessPointId | + +---------------------------------------+--------+--------+--------+ - [ ] Arn | + | | | | | | + +---------------------------------------+--------+--------+--------+----------------------------------------+ + |AWS::EFS::FileSystem | x | | x | - [ ] FileSystemId | +---------------------------------------+--------+--------+--------+ - [ ] Arn | | | | | | | +---------------------------------------+--------+--------+--------+----------------------------------------+ @@ -121,7 +125,9 @@ Please let us know if you'd like support for a resource not yet listed here. +---------------------------------------+--------+--------+--------+ - [ ] Id | | | | | | | +---------------------------------------+--------+--------+--------+----------------------------------------+ - |AWS::EMR::Cluster | x | | | - [ ] MasterPublicDNS | + |AWS::EMR::Cluster | x | | x | - [ ] MasterPublicDNS | + +---------------------------------------+--------+--------+--------+----------------------------------------+ + |AWS::EMR::SecurityConfiguration | x | | x | | +---------------------------------------+--------+--------+--------+----------------------------------------+ |AWS::Events::Archive | x | x | | - [x] Arn | +---------------------------------------+--------+--------+--------+----------------------------------------+ @@ -148,7 +154,7 @@ Please let us know if you'd like support for a resource not yet listed here. |AWS::IAM::Policy | | | | - [ ] Id | +---------------------------------------+--------+--------+--------+----------------------------------------+ |AWS::IAM::Role | x | | x | - [x] Arn | - +---------------------------------------+--------+--------+--------+ - [ ] RoleId | + +---------------------------------------+--------+--------+--------+ - [x] RoleId | | | | | | | +---------------------------------------+--------+--------+--------+----------------------------------------+ |AWS::IAM::User | x | x | x | - [x] Arn | diff --git a/docs/docs/services/firehose.rst b/docs/docs/services/firehose.rst index ec59a4e9c57..66215978057 100644 --- a/docs/docs/services/firehose.rst +++ b/docs/docs/services/firehose.rst @@ -33,7 +33,6 @@ firehose are not currently processed/implemented. -- [ ] get_kinesis_stream - [X] list_delivery_streams Return list of delivery streams in alphabetic order of names. @@ -55,5 +54,4 @@ firehose Removes tags from specified delivery stream. - [X] update_destination -- [ ] verify_resources_exist_for_tagris diff --git a/docs/docs/services/medialive.rst b/docs/docs/services/medialive.rst index 5509f048650..87381a0a55c 100644 --- a/docs/docs/services/medialive.rst +++ b/docs/docs/services/medialive.rst @@ -77,6 +77,7 @@ medialive - [ ] purchase_offering - [ ] reboot_input_device - [ ] reject_input_device_transfer +- [ ] restart_channel_pipelines - [X] start_channel - [ ] start_input_device - [ ] start_input_device_maintenance_window diff --git a/docs/docs/services/route53domains.rst b/docs/docs/services/route53domains.rst new file mode 100644 index 00000000000..45e317220ac --- /dev/null +++ b/docs/docs/services/route53domains.rst @@ -0,0 +1,55 @@ +.. _implementedservice_route53domains: + +.. |start-h3| raw:: html + +

+ +.. |end-h3| raw:: html + +

+ +============== +route53domains +============== + +.. autoclass:: moto.route53domains.models.Route53DomainsBackend + +|start-h3| Implemented features for this service |end-h3| + +- [ ] accept_domain_transfer_from_another_aws_account +- [ ] associate_delegation_signer_to_domain +- [ ] cancel_domain_transfer_to_another_aws_account +- [ ] check_domain_availability +- [ ] check_domain_transferability +- [X] delete_domain +- [ ] delete_tags_for_domain +- [ ] disable_domain_auto_renew +- [ ] disable_domain_transfer_lock +- [ ] disassociate_delegation_signer_from_domain +- [ ] enable_domain_auto_renew +- [ ] enable_domain_transfer_lock +- [ ] get_contact_reachability_status +- [ ] get_domain_detail +- [ ] get_domain_suggestions +- [ ] get_operation_detail +- [X] list_domains +- [X] list_operations +- [ ] list_prices +- [ ] list_tags_for_domain +- [ ] push_domain +- [X] register_domain + Register a domain + +- [ ] reject_domain_transfer_from_another_aws_account +- [ ] renew_domain +- [ ] resend_contact_reachability_email +- [ ] resend_operation_authorization +- [ ] retrieve_domain_auth_code +- [ ] transfer_domain +- [ ] transfer_domain_to_another_aws_account +- [ ] update_domain_contact +- [ ] update_domain_contact_privacy +- [X] update_domain_nameservers +- [ ] update_tags_for_domain +- [ ] view_billing + diff --git a/docs/docs/services/s3.rst b/docs/docs/services/s3.rst index cbc4ded7c26..bcabe4bbc9d 100644 --- a/docs/docs/services/s3.rst +++ b/docs/docs/services/s3.rst @@ -109,8 +109,11 @@ s3 - EventBridge For the following events: + - 's3:ObjectCreated:CompleteMultipartUpload' - 's3:ObjectCreated:Copy' + - 's3:ObjectCreated:Post' - 's3:ObjectCreated:Put' + - 's3:ObjectDeleted' - [X] put_bucket_ownership_controls diff --git a/moto/s3/models.py b/moto/s3/models.py index af52a9b792f..407b60e2a71 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -2336,8 +2336,11 @@ def put_bucket_notification_configuration( - EventBridge For the following events: + - 's3:ObjectCreated:CompleteMultipartUpload' - 's3:ObjectCreated:Copy' + - 's3:ObjectCreated:Post' - 's3:ObjectCreated:Put' + - 's3:ObjectDeleted' """ bucket = self.get_bucket(bucket_name) bucket.set_notification_configuration(notification_config) @@ -2851,6 +2854,16 @@ def select_object_content( for x in query_result ] + def upload_file(self) -> None: + # Listed for the implementation coverage + # Implementation part of responses.py + pass + + def upload_fileobj(self) -> None: + # Listed for the implementation coverage + # Implementation part of responses.py + pass + class S3BackendDict(BackendDict[S3Backend]): """ diff --git a/moto/sqs/models.py b/moto/sqs/models.py index d3f9705a626..66860fe49fe 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -850,15 +850,15 @@ def send_message( ) -> Message: queue = self.get_queue(queue_name) + self._validate_message( + queue, message_body, int(delay_seconds or 0), deduplication_id, group_id + ) + if delay_seconds is not None: delay_seconds = int(delay_seconds) else: delay_seconds = queue.delay_seconds # type: ignore - self._validate_message( - queue, message_body, delay_seconds, deduplication_id, group_id - ) - message_id = str(random.uuid4()) message = Message(message_id, message_body, system_attributes) diff --git a/scripts/implementation_coverage.py b/scripts/implementation_coverage.py index 05adab0b88c..63af558b662 100755 --- a/scripts/implementation_coverage.py +++ b/scripts/implementation_coverage.py @@ -47,6 +47,9 @@ def calculate_extended_implementation_coverage(): operation_names = [ xform_name(op) for op in real_client.meta.service_model.operation_names ] + # Not part of the spec - but very much part of S3 + if service_name == "s3": + operation_names.extend(["upload_file", "upload_fileobj"]) for op in operation_names: if moto_client and op in dir(moto_client): @@ -76,6 +79,9 @@ def calculate_implementation_coverage(): operation_names = [ xform_name(op) for op in real_client.meta.service_model.operation_names ] + # Not part of the spec - but very much part of S3 + if service_name == "s3": + operation_names.extend(["upload_file", "upload_fileobj"]) for op in operation_names: if moto_client and op in dir(moto_client): implemented.append(op) diff --git a/tests/test_iam/test_iam_cloudformation.py b/tests/test_iam/test_iam_cloudformation.py index 678939ae1e4..7c7e3707d60 100644 --- a/tests/test_iam/test_iam_cloudformation.py +++ b/tests/test_iam/test_iam_cloudformation.py @@ -1645,6 +1645,7 @@ def test_iam_roles(): """ +@pytest.mark.aws_verified @iam_aws_verified def test_delete_instance_profile_with_existing_role(): region = "us-east-1" diff --git a/tests/test_sqs/__init__.py b/tests/test_sqs/__init__.py index 08a1c1568c9..5d04e1d1720 100644 --- a/tests/test_sqs/__init__.py +++ b/tests/test_sqs/__init__.py @@ -1 +1,28 @@ -# This file is intentionally left blank. +import os +from functools import wraps + +from moto import mock_aws + + +def sqs_aws_verified(func): + """ + Function that is verified to work against AWS. + Can be run against AWS at any time by setting: + MOTO_TEST_ALLOW_AWS_REQUEST=true + + If this environment variable is not set, the function runs in a `mock_aws` context. + """ + + @wraps(func) + def pagination_wrapper(*args, **kwargs): + allow_aws_request = ( + os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" + ) + + if allow_aws_request: + return func(*args, **kwargs) + else: + with mock_aws(): + return func(*args, **kwargs) + + return pagination_wrapper diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 5ae192a6381..05d531f34f8 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -23,6 +23,8 @@ ) from moto.utilities.distutils_version import LooseVersion +from . import sqs_aws_verified + TEST_POLICY = """ { "Version":"2012-10-17", @@ -3307,18 +3309,16 @@ def test_fifo_dedupe_error_no_message_dedupe_id_batch(): ) -@mock_aws +@sqs_aws_verified +@pytest.mark.aws_verified @pytest.mark.parametrize( "queue_config", [{"FifoQueue": "true"}, {"FifoQueue": "true", "DelaySeconds": "10"}] ) def test_send_message_delay_seconds_validation(queue_config): # Setup client = boto3.client("sqs", region_name=REGION) - q = "test.fifo" - client.create_queue( - QueueName=q, - Attributes=queue_config, - ) + q = f"moto_{str(uuid4())[0:6]}.fifo" + client.create_queue(QueueName=q, Attributes=queue_config) # Execute with pytest.raises(ClientError) as err: @@ -3330,6 +3330,14 @@ def test_send_message_delay_seconds_validation(queue_config): MessageDeduplicationId=str(uuid4()), ) + # Verify + ex = err.value + assert ex.response["Error"]["Code"] == "InvalidParameterValue" + assert ex.response["Error"]["Message"] == ( + "Value 5 for parameter DelaySeconds is invalid. Reason: " + "The request include parameter that is not valid for this queue type." + ) + # this should succeed regardless of DelaySeconds configuration on the q # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#API_SendMessage_RequestSyntax client.send_message( @@ -3340,12 +3348,11 @@ def test_send_message_delay_seconds_validation(queue_config): MessageDeduplicationId=str(uuid4()), ) - # Verify - ex = err.value - assert ex.response["Error"]["Code"] == "InvalidParameterValue" - assert ex.response["Error"]["Message"] == ( - "Value 5 for parameter DelaySeconds is invalid. Reason: " - "The request include parameter that is not valid for this queue type." + client.send_message( + QueueUrl=q, + MessageBody="test", + MessageGroupId="test", + MessageDeduplicationId=str(uuid4()), ) # clean up for servertests