From 45cfdf9a9037247b7d2535c59d0715af9adb47ea Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Mon, 26 Oct 2015 09:53:58 -0700 Subject: [PATCH 1/8] Move multipart integration tests to Behat suite --- behat.yml | 3 + features/multipart/glacier.feature | 14 + features/multipart/s3.feature | 14 + tests/ConcurrencyTest.php | 101 +++++ tests/Integ/ClientSmokeTest.php | 598 --------------------------- tests/Integ/ConcurrencyTest.php | 74 ---- tests/Integ/GlacierMultipartTest.php | 73 ---- tests/Integ/MultipartContext.php | 173 ++++++++ tests/Integ/S3MultipartTest.php | 99 ----- 9 files changed, 305 insertions(+), 844 deletions(-) create mode 100644 features/multipart/glacier.feature create mode 100644 features/multipart/s3.feature create mode 100644 tests/ConcurrencyTest.php delete mode 100644 tests/Integ/ClientSmokeTest.php delete mode 100644 tests/Integ/ConcurrencyTest.php delete mode 100644 tests/Integ/GlacierMultipartTest.php create mode 100644 tests/Integ/MultipartContext.php delete mode 100644 tests/Integ/S3MultipartTest.php diff --git a/behat.yml b/behat.yml index dbd7f0b4f1..0b3e00542b 100644 --- a/behat.yml +++ b/behat.yml @@ -6,3 +6,6 @@ default: performance: paths: [ %paths.base%/features/performance ] contexts: [ Aws\Test\PerformanceContext ] + multipart: + paths: [ %paths.base%/features/multipart ] + contexts: [ Aws\Test\Integ\MultipartContext ] diff --git a/features/multipart/glacier.feature b/features/multipart/glacier.feature new file mode 100644 index 0000000000..bf892ff231 --- /dev/null +++ b/features/multipart/glacier.feature @@ -0,0 +1,14 @@ +@glacier +Feature: Glacier Multipart Uploads + + Scenario Outline: Uploading a stream + Given I have a read stream + When I upload the stream to Glacier with a concurrency factor of "" + Then the result should contain a(n) "location" + + Examples: + | seekable | concurrency | + | seekable | 1 | + | non-seekable | 1 | + | seekable | 3 | + | non-seekable | 3 | diff --git a/features/multipart/s3.feature b/features/multipart/s3.feature new file mode 100644 index 0000000000..9f59a6cfc2 --- /dev/null +++ b/features/multipart/s3.feature @@ -0,0 +1,14 @@ +@s3 +Feature: S3 Multipart Uploads + + Scenario Outline: Uploading a stream + Given I have a read stream + When I upload the stream to S3 with a concurrency factor of "" + Then the result should contain a(n) "ObjectURL" + + Examples: + | seekable | concurrency | + | seekable | 1 | + | non-seekable | 1 | + | seekable | 3 | + | non-seekable | 3 | diff --git a/tests/ConcurrencyTest.php b/tests/ConcurrencyTest.php new file mode 100644 index 0000000000..2d7faafe44 --- /dev/null +++ b/tests/ConcurrencyTest.php @@ -0,0 +1,101 @@ +getTestClient('s3'); + $this->addMockResults($client, [new Result(['Owner' => ['ID' => 'id']])]); + $result = $client->listBuckets(); + $this->assertInstanceOf(ResultInterface::class, $result); + $this->assertInternalType('string', $result['Owner']['ID']); + } + + public function testSendsPromisedSynchronousRequest() + { + /** @var S3Client $client */ + $client = $this->getTestClient('s3'); + $this->addMockResults($client, [new Result(['Owner' => ['ID' => 'id']])]); + $promise = $client->listBucketsAsync(); + $this->assertInstanceOf(PromiseInterface::class, $promise); + $result = $promise->wait(); + $this->assertInternalType('string', $result['Owner']['ID']); + } + + public function testSendsRequestsAsynchronously() + { + /** @var S3Client $client */ + $client = $this->getTestClient('s3'); + $this->addMockResults($client, [ + new Result(['Owner' => ['ID' => 'id']]), + new Result(['Owner' => ['ID' => 'id']]), + new Result(['Owner' => ['ID' => 'id']]), + ]); + + $promise = Promise\all([ + $client->listBucketsAsync(), + $client->listBucketsAsync(), + $client->listBucketsAsync() + ]); + $results = $promise->wait(); + + $this->assertCount(3, $results); + $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); + } + + public function testSendsRequestsConcurrentlyWithPool() + { + /** @var S3Client $client */ + $client = $this->getTestClient('s3'); + $this->addMockResults($client, [ + new Result(['Owner' => ['ID' => 'id']]), + new Result(['Owner' => ['ID' => 'id']]), + new Result(['Owner' => ['ID' => 'id']]), + ]); + + $commands = [ + $client->getCommand('ListBuckets'), + $client->getCommand('ListBuckets'), + $client->getCommand('ListBuckets') + ]; + $results = CommandPool::batch($client, $commands); + + $this->assertCount(3, $results); + $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); + } + + public function testSendsRequestsAsynchronouslyFromMultipleServices() + { + /** @var S3Client $s3 */ + $s3 = $this->getTestClient('s3'); + $this->addMockResults($s3, [new Result]); + /** @var DynamoDbClient $db */ + $db = $this->getTestClient('dynamodb'); + $this->addMockResults($db, [new Result]); + /** @var SqsClient $sqs */ + $sqs = $this->getTestClient('sqs'); + $this->addMockResults($sqs, [new Result]); + + $promise = Promise\all([ + $s3->listBucketsAsync(), + $db->listTablesAsync(), + $sqs->listQueuesAsync() + ]); + $results = $promise->wait(); + + $this->assertCount(3, $results); + } +} diff --git a/tests/Integ/ClientSmokeTest.php b/tests/Integ/ClientSmokeTest.php deleted file mode 100644 index 0ce007dff1..0000000000 --- a/tests/Integ/ClientSmokeTest.php +++ /dev/null @@ -1,598 +0,0 @@ -wait(); - - $data = json_decode($response->getBody()->getContents(), true); - $ua = $data['headers']['User-Agent']; - $this->assertStringStartsWith('aws-sdk-php/' . Sdk::VERSION, $ua); - } - - /** - * @dataProvider provideServiceTestCases - */ - public function testBasicOperationWorks($service, $class, $options, - $endpoint, $operation, $params, $succeed, $value - ) { - // Create the client and make sure it is the right class. - $client = $this->getSdk()->createClient($service, $options); - $this->assertInstanceOf($class, $client); - - // Setup event to get the request's host value. - $host = null; - $client->getHandlerList()->appendSign( - Middleware::tap(function ($command, RequestInterface $request) use (&$host) { - $host = $request->getUri()->getHost(); - } - )); - - // Execute the request and check if it behaved as intended. - try { - // Execute the operation. - $result = $client->execute($client->getCommand($operation, $params)); - if (!$succeed) { - $this->fail("The {$operation} operation of the {$service} " - . "service was supposed to fail."); - } - - // Examine the result. - if ($value !== null) { - // Ensure the presence of the specified key. - $this->assertArrayHasKey($value, $result); - } - } catch (AwsException $e) { - if ($succeed) { - $this->fail("The {$operation} operation of the {$service} " - . "service was supposed to succeed. (" . $e->getMessage() . ")"); - } - - // The exception class should have the same namespace as the client. - $this->assertStringStartsWith( - substr($class, 0, strrpos($class, '\\')), - get_class($e) - ); - - // Look at the error code first, then the root exception class, to - // see if it matches the value. - $error = $e; - while ($error->getPrevious()) $error = $error->getPrevious(); - $this->assertEquals( - $value, - $e->getAwsErrorCode() ?: get_class($error), - $e->getMessage() - ); - } catch (\Exception $e) { - // If something other than an AwsException was thrown, then - // something really went wrong. - $this->fail('An unexpected exception occurred: ' . get_class($e) - . ' - ' . $e->getMessage()); - } - - // Ensure the request's host is correct no matter the outcome. - $this->assertEquals($endpoint, $host); - } - - public function provideServiceTestCases() - { - return [ - /*[ - service (client to create `Sdk::createClient()`) - class (expected class name of instantiated client) - options (client options; besides region, version, & credentials) - endpoint (expected host of the request) - operation (service operation to execute) - params (parameters to use for the operation) - succeeds (bool - whether or not the request should succeed) - value (a key that should be present in the result - OR... the error code, in the case of failure) - ],*/ - [ - 'autoscaling', - 'Aws\\AutoScaling\\AutoScalingClient', - [], - 'autoscaling.us-east-1.amazonaws.com', - 'DescribeAccountLimits', - [], - true, - 'MaxNumberOfAutoScalingGroups' - ], - [ - 'cloudformation', - 'Aws\\CloudFormation\\CloudFormationClient', - [], - 'cloudformation.us-east-1.amazonaws.com', - 'DescribeStacks', - [], - true, - 'Stacks' - ], - [ - 'cloudfront', - 'Aws\\CloudFront\\CloudFrontClient', - [], - 'cloudfront.amazonaws.com', - 'ListDistributions', - [], - true, - 'DistributionList' - ], - [ - 'cloudhsm', - 'Aws\\CloudHsm\\CloudHsmClient', - [], - 'cloudhsm.us-east-1.amazonaws.com', - 'listAvailableZones', - [], - true, - 'AZList' - ], - [ - 'cloudsearch', - 'Aws\\CloudSearch\\CloudSearchClient', - [], - 'cloudsearch.us-east-1.amazonaws.com', - 'DescribeDomains', - [], - true, - 'DomainStatusList' - ], - [ - 'cloudsearchdomain', - 'Aws\\CloudSearchDomain\\CloudSearchDomainClient', - ['endpoint' => 'https://search-foo.cloudsearch.us-east-1.amazonaws.com'], - 'search-foo.cloudsearch.us-east-1.amazonaws.com', - 'Search', - ['query' => 'foo'], - false, - 'GuzzleHttp\Exception\ConnectException' - ], - [ - 'cloudtrail', - 'Aws\\CloudTrail\\CloudTrailClient', - [], - 'cloudtrail.us-east-1.amazonaws.com', - 'DeleteTrail', - ['Name' => 'foo'], - false, - 'TrailNotFoundException' - ], - [ - 'cloudwatch', - 'Aws\\CloudWatch\\CloudWatchClient', - [], - 'monitoring.us-east-1.amazonaws.com', - 'DescribeAlarms', - [], - true, - 'MetricAlarms' - ], - [ - 'cloudwatchlogs', - 'Aws\\CloudWatchLogs\\CloudWatchLogsClient', - [], - 'logs.us-east-1.amazonaws.com', - 'DescribeLogGroups', - [], - true, - 'logGroups' - ], - [ - 'cognitoidentity', - 'Aws\\CognitoIdentity\\CognitoIdentityClient', - [], - 'cognito-identity.us-east-1.amazonaws.com', - 'ListIdentityPools', - ['MaxResults' => 1], - true, - 'IdentityPools' - ], - [ - 'cognitosync', - 'Aws\\CognitoSync\\CognitoSyncClient', - [], - 'cognito-sync.us-east-1.amazonaws.com', - 'ListIdentityPoolUsage', - [], - true, - 'IdentityPoolUsages' - ], - [ - 'configservice', - 'Aws\\ConfigService\\ConfigServiceClient', - [], - 'config.us-east-1.amazonaws.com', - 'DeliverConfigSnapshot', - ['deliveryChannelName' => 'foo'], - false, - 'NoAvailableConfigurationRecorderException' - ], - [ - 'datapipeline', - 'Aws\\DataPipeline\\DataPipelineClient', - [], - 'datapipeline.us-east-1.amazonaws.com', - 'DescribePipelines', - ['pipelineIds' => ['foo']], - false, - 'PipelineNotFoundException' - ], - [ - 'directconnect', - 'Aws\\DirectConnect\\DirectConnectClient', - [], - 'directconnect.us-east-1.amazonaws.com', - 'DescribeConnections', - [], - true, - 'connections' - ], - [ - 'dynamodb', - 'Aws\\DynamoDb\\DynamoDbClient', - [], - 'dynamodb.us-east-1.amazonaws.com', - 'ListTables', - [], - true, - 'TableNames' - ], - [ - 'ec2', - 'Aws\\Ec2\\Ec2Client', - [], - 'ec2.us-east-1.amazonaws.com', - 'DescribeInstances', - [], - true, - 'Reservations' - ], - [ // ECS Success - 'ecs', - 'Aws\\Ecs\\EcsClient', - [], - 'ecs.us-east-1.amazonaws.com', - 'DescribeClusters', - ['clusters' => ['foo']], - true, - 'clusters' - ], - [ // ECS Failure - 'ecs', - 'Aws\\Ecs\\EcsClient', - [], - 'ecs.us-east-1.amazonaws.com', - 'DeleteCluster', - ['cluster' => 'foo'], - false, - 'ClusterNotFoundException' - ], - [ - 'elasticache', - 'Aws\\ElastiCache\\ElastiCacheClient', - [], - 'elasticache.us-east-1.amazonaws.com', - 'DescribeCacheClusters', - [], - true, - 'CacheClusters' - ], - [ - 'elasticbeanstalk', - 'Aws\\ElasticBeanstalk\\ElasticBeanstalkClient', - [], - 'elasticbeanstalk.us-east-1.amazonaws.com', - 'DescribeApplications', - [], - true, - 'Applications' - ], - [ - 'elastictranscoder', - 'Aws\\ElasticTranscoder\\ElasticTranscoderClient', - [], - 'elastictranscoder.us-east-1.amazonaws.com', - 'CancelJob', - ['Id' => '1111111111111-aaaaaa'], - false, - 'ResourceNotFoundException' - ], - [ - 'elasticloadbalancing', - 'Aws\\ElasticLoadBalancing\\ElasticLoadBalancingClient', - [], - 'elasticloadbalancing.us-east-1.amazonaws.com', - 'DescribeLoadBalancers', - [], - true, - 'LoadBalancerDescriptions' - ], - [ - 'emr', - 'Aws\\Emr\\EmrClient', - [], - 'elasticmapreduce.us-east-1.amazonaws.com', - 'ListClusters', - [], - true, - 'Clusters' - ], - [ - 'glacier', - 'Aws\\Glacier\\GlacierClient', - [], - 'glacier.us-east-1.amazonaws.com', - 'DescribeJob', - ['vaultName' => 'foo', 'jobId' => 'bar'], - false, - 'ResourceNotFoundException' - ], - [ - 'iam', - 'Aws\\Iam\\IamClient', - [], - 'iam.amazonaws.com', - 'ListGroups', - [], - true, - 'Groups' - ], - [ - 'kinesis', - 'Aws\\Kinesis\\KinesisClient', - [], - 'kinesis.us-east-1.amazonaws.com', - 'ListStreams', - [], - true, - 'StreamNames' - ], - [ // KMS Success - 'kms', - 'Aws\\Kms\\KmsClient', - [], - 'kms.us-east-1.amazonaws.com', - 'ListAliases', - [], - true, - 'Aliases' - ], - [ // KMS Failure - 'kms', - 'Aws\\Kms\\KmsClient', - [], - 'kms.us-east-1.amazonaws.com', - 'DeleteAlias', - ['AliasName' => 'foo'], - false, - 'ValidationException' - ], - [ // Lambda Success - 'lambda', - 'Aws\\Lambda\\LambdaClient', - [], - 'lambda.us-east-1.amazonaws.com', - 'ListFunctions', - [], - true, - 'Functions' - ], - [ // Lambda Failure - 'lambda', - 'Aws\\Lambda\\LambdaClient', - [], - 'lambda.us-east-1.amazonaws.com', - 'DeleteFunction', - ['FunctionName' => 'foo'], - false, - 'ResourceNotFoundException' - ], - [ // MachineLearning Success - 'machinelearning', - 'Aws\\MachineLearning\\MachineLearningClient', - [], - 'machinelearning.us-east-1.amazonaws.com', - 'DescribeDataSources', - [], - true, - 'Results' - ], - [ // MachineLearning Failure - 'machinelearning', - 'Aws\\MachineLearning\\MachineLearningClient', - [], - 'machinelearning.us-east-1.amazonaws.com', - 'DeleteDataSource', - ['DataSourceId' => 'foo'], - false, - 'ResourceNotFoundException' - ], - [ - 'opsworks', - 'Aws\\OpsWorks\\OpsWorksClient', - [], - 'opsworks.us-east-1.amazonaws.com', - 'DescribeStacks', - [], - true, - 'Stacks' - ], - [ - 'rds', - 'Aws\\Rds\\RdsClient', - [], - 'rds.us-east-1.amazonaws.com', - 'DescribeDBInstances', - [], - true, - 'DBInstances' - ], - [ - 'redshift', - 'Aws\\Redshift\\RedshiftClient', - [], - 'redshift.us-east-1.amazonaws.com', - 'DescribeClusters', - [], - true, - 'Clusters' - ], - [ - 'route53', - 'Aws\\Route53\\Route53Client', - [], - 'route53.amazonaws.com', - 'ListHostedZones', - [], - true, - 'HostedZones' - ], - [ - 'route53domains', - 'Aws\\Route53Domains\\Route53DomainsClient', - [], - 'route53domains.us-east-1.amazonaws.com', - 'ListDomains', - [], - true, - 'Domains' - ], - [ - 's3', - 'Aws\\S3\\S3Client', - [], - 's3.amazonaws.com', - 'ListObjects', - ['Bucket' => 't0tally-1nval1d-8uck3t-nam3'], - false, - 'NoSuchBucket' - ], - [ - 'ses', - 'Aws\\Ses\\SesClient', - [], - 'email.us-east-1.amazonaws.com', - 'ListIdentities', - [], - true, - 'Identities' - ], - [ - 'sns', - 'Aws\\Sns\\SnsClient', - [], - 'sns.us-east-1.amazonaws.com', - 'ListTopics', - [], - true, - 'Topics' - ], - [ - 'sqs', - 'Aws\\Sqs\\SqsClient', - [], - 'sqs.us-east-1.amazonaws.com', - 'ListQueues', - [], - true, - 'QueueUrls' - ], - [ // SSM Success - 'ssm', - 'Aws\\Ssm\\SsmClient', - [], - 'ssm.us-east-1.amazonaws.com', - 'ListDocuments', - [], - true, - 'DocumentIdentifiers' - ], - [ // SSM Failure - 'ssm', - 'Aws\\Ssm\\SsmClient', - [], - 'ssm.us-east-1.amazonaws.com', - 'DeleteDocument', - ['Name' => 'foo'], - false, - 'InvalidDocument' - ], - [ - 'storagegateway', - 'Aws\\StorageGateway\\StorageGatewayClient', - [], - 'storagegateway.us-east-1.amazonaws.com', - 'DeleteVolume', - ['VolumeARN' => 'foo'], - false, - 'ValidationException' - ], - [ - 'sts', - 'Aws\\Sts\\StsClient', - [], - 'sts.amazonaws.com', - 'GetSessionToken', - [], - true, - 'Credentials' - ], - [ - 'support', - 'Aws\\Support\\SupportClient', - [], - 'support.us-east-1.amazonaws.com', - 'DescribeCases', - [], - false, - 'SubscriptionRequiredException' - ], - [ - 'swf', - 'Aws\\Swf\\SwfClient', - [], - 'swf.us-east-1.amazonaws.com', - 'DescribeDomain', - ['name' => 'foo'], - false, - 'UnknownResourceFault' - ], - [ // WorkSpaces Success - 'workspaces', - 'Aws\\Workspaces\\WorkspacesClient', - [], - 'workspaces.us-east-1.amazonaws.com', - 'DescribeWorkspaces', - [], - true, - 'Workspaces' - ], - [ // WorkSpaces Failure - 'workspaces', - 'Aws\\WorkSpaces\\WorkSpacesClient', - [], - 'workspaces.us-east-1.amazonaws.com', - 'TerminateWorkspaces', - ['TerminateWorkspaceRequests' => [['WorkspaceId'=> 'foo']]], - false, - 'ValidationException' - ], - ]; - } -} diff --git a/tests/Integ/ConcurrencyTest.php b/tests/Integ/ConcurrencyTest.php deleted file mode 100644 index b68f6dec58..0000000000 --- a/tests/Integ/ConcurrencyTest.php +++ /dev/null @@ -1,74 +0,0 @@ -getSdk()->createClient('s3'); - $result = $client->listBuckets(); - $this->assertInstanceOf('Aws\Result', $result); - $this->assertInternalType('string', $result['Owner']['ID']); - } - - public function testSendsPromisedSynchronousRequest() - { - $client = $this->getSdk()->createClient('s3'); - $promise = $client->listBucketsAsync(); - $this->assertInstanceOf(PromiseInterface::class, $promise); - $result = $promise->wait(); - $this->assertInternalType('string', $result['Owner']['ID']); - } - - public function testSendsRequestsAsynchronously() - { - $s3 = $this->getSdk()->createS3(); - - $promise = Promise\all([ - $s3->listBucketsAsync(), - $s3->listBucketsAsync(), - $s3->listBucketsAsync() - ]); - $results = $promise->wait(); - - $this->assertCount(3, $results); - $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); - } - - public function testSendsRequestsConcurrentlyWithPool() - { - $s3 = $this->getSdk()->createS3(); - - $commands = [ - $s3->getCommand('ListBuckets'), - $s3->getCommand('ListBuckets'), - $s3->getCommand('ListBuckets') - ]; - $results = CommandPool::batch($s3, $commands); - - $this->assertCount(3, $results); - $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); - } - - public function testSendsRequestsAsynchronouslyFromMultipleServices() - { - $s3 = $this->getSdk()->createS3(); - $db = $this->getSdk()->createDynamoDb(); - $sqs = $this->getSdk()->createSqs(); - - $promise = Promise\all([ - $s3->listBucketsAsync(), - $db->listTablesAsync(), - $sqs->listQueuesAsync() - ]); - $results = $promise->wait(); - - $this->assertCount(3, $results); - } -} diff --git a/tests/Integ/GlacierMultipartTest.php b/tests/Integ/GlacierMultipartTest.php deleted file mode 100644 index c4f489a98f..0000000000 --- a/tests/Integ/GlacierMultipartTest.php +++ /dev/null @@ -1,73 +0,0 @@ -createGlacier(); - $client->createVault(['vaultName' => self::VAULT]); - $client->waitUntil('VaultExists', ['vaultName' => self::VAULT]); - } - - public function useCasesProvider() - { - return [ - ['SeekableSerialUpload', true, 1], - ['NonSeekableSerialUpload', false, 3], - ['SeekableConcurrentUpload', true, 1], - ['NonSeekableConcurrentUpload', false, 3], - ]; - } - - /** - * @param string $description - * @param bool $seekable - * @param int $concurrency - * @dataProvider useCasesProvider - */ - public function testMultipartUpload($description, $seekable, $concurrency) - { - $client = self::getSdk()->createGlacier(); - $tmpFile = sys_get_temp_dir() . '/aws-php-sdk-integ-glacier-mup'; - file_put_contents($tmpFile, str_repeat('x', 2 * self::MB + 1024)); - - $stream = Psr7\stream_for(fopen($tmpFile, 'r')); - if (!$seekable) { - $stream = new NoSeekStream($stream); - } - - $uploader = (new MultipartUploader($client, $stream, [ - 'vault_name' => self::VAULT, - 'archive_description' => $description, - 'concurrency' => $concurrency, - ])); - - try { - $result = $uploader->upload($concurrency); - $this->assertArrayHasKey('location', $result); - } catch (MultipartUploadException $e) { - $client->abortMultipartUpload($e->getState()->getId()); - $message = "=====\n"; - while ($e) { - $message .= $e->getMessage() . "\n"; - $e = $e->getPrevious(); - } - $message .= "=====\n"; - $this->fail($message); - } - - @unlink($tmpFile); - } -} diff --git a/tests/Integ/MultipartContext.php b/tests/Integ/MultipartContext.php new file mode 100644 index 0000000000..01cbd12d9f --- /dev/null +++ b/tests/Integ/MultipartContext.php @@ -0,0 +1,173 @@ +stream = Psr7\stream_for(Psr7\try_fopen(self::$tempFile, 'r')); + } + + /** + * @When /^I upload the stream to S3 with a concurrency factor of "(\d+)"$/ + */ + public function iUploadTheStreamToS3WithAConcurrencyFactorOf($concurrency) + { + $client = self::getSdk()->createS3(); + $uploader = new S3MultipartUploader($client, $this->stream, [ + 'bucket' => self::getResourceName(), + 'key' => get_class($this->stream) . $concurrency, + 'concurrency' => $concurrency, + ]); + + try { + $this->result = $uploader->upload(); + } catch (MultipartUploadException $e) { + $client->abortMultipartUpload($e->getState()->getId()); + $message = "=====\n"; + while ($e) { + $message .= $e->getMessage() . "\n"; + $e = $e->getPrevious(); + } + $message .= "=====\n"; + Assert::fail($message); + } + } + + /** + * @When /^I upload the stream to Glacier with a concurrency factor of "(\d+)"$/ + */ + public function iUploadTheStreamToGlacierWithAConcurrencyFactorOf($concurrency) + { + $client = self::getSdk()->createGlacier(); + $uploader = new GlacierMultipartUploader($client, $this->stream, [ + 'vault_name' => self::RESOURCE_POSTFIX, + 'archive_description' => get_class($this->stream) . $concurrency, + 'concurrency' => $concurrency, + ]); + + try { + $this->result = $uploader->upload(); + } catch (MultipartUploadException $e) { + $client->abortMultipartUpload($e->getState()->getId()); + $message = "=====\n"; + while ($e) { + $message .= $e->getMessage() . "\n"; + $e = $e->getPrevious(); + } + $message .= "=====\n"; + Assert::fail($message); + } + } + + /** + * @Then /^the result should contain a\(n\) "([^"]+)"$/ + */ + public function theResultShouldContainA($key) + { + Assert::assertArrayHasKey($key, $this->result); + } + + /** + * @Given I have a non-seekable read stream + */ + public function iHaveANonSeekableReadStream() + { + $this->iHaveASeekableReadStream(); + $this->stream = new NoSeekStream($this->stream); + } + + /** + * @BeforeSuite + */ + public static function createTempFile() + { + self::$tempFile = tempnam(sys_get_temp_dir(), self::getResourceName()); + file_put_contents(self::$tempFile, str_repeat('x', 10 * self::MB + 1024)); + } + + /** + * @AfterSuite + */ + public static function deleteTempFile() + { + unlink(self::$tempFile); + } + + /** + * @BeforeFeature @s3 + */ + public static function createTestBucket() + { + $client = self::getSdk()->createS3(); + if (!$client->doesBucketExist(self::getResourceName())) { + $client->createBucket(['Bucket' => self::getResourceName()]); + $client->waitUntil('BucketExists', ['Bucket' => self::getResourceName()]); + } + } + + /** + * @AfterFeature @s3 + */ + public static function deleteTestBucket() + { + $client = self::getSdk()->createS3(); + BatchDelete::fromListObjects($client, ['Bucket' => self::getResourceName()])->delete(); + $client->deleteBucket(['Bucket' => self::getResourceName()]); + $client->waitUntil('BucketNotExists', ['Bucket' => self::getResourceName()]); + } + + /** + * @BeforeFeature @glacier + */ + public static function createTestVault() + { + $client = self::getSdk()->createGlacier(); + $client->createVault(['vaultName' => self::RESOURCE_POSTFIX]); + $client->waitUntil('VaultExists', ['vaultName' => self::RESOURCE_POSTFIX]); + } + + private static function getResourceName() + { + static $bucketName; + + if (empty($bucketName)) { + $bucketName = self::getResourcePrefix() . self::RESOURCE_POSTFIX; + } + + return $bucketName; + } +} diff --git a/tests/Integ/S3MultipartTest.php b/tests/Integ/S3MultipartTest.php deleted file mode 100644 index 080f2d1ddb..0000000000 --- a/tests/Integ/S3MultipartTest.php +++ /dev/null @@ -1,99 +0,0 @@ -createS3(); - if (!$client->doesBucketExist(self::getBucketName())) { - $client->createBucket(['Bucket' => self::getBucketName()]); - $client->waitUntil('BucketExists', ['Bucket' => self::getBucketName()]); - } - } - - public static function tearDownAfterClass() - { - $client = self::getSdk()->createS3(); - BatchDelete::fromListObjects($client, ['Bucket' => self::getBucketName()])->delete(); - $client->deleteBucket(['Bucket' => self::getBucketName()]); - $client->waitUntil('BucketNotExists', ['Bucket' => self::getBucketName()]); - } - - public function useCasesProvider() - { - return [ - ['SeekableSerialUpload', true, 1], - ['NonSeekableSerialUpload', false, 1], - ['SeekableConcurrentUpload', true, 3], - ['NonSeekableConcurrentUpload', false, 3], - ]; - } - - /** - * @param string $key - * @param bool $seekable - * @param int $concurrency - * @dataProvider useCasesProvider - */ - public function testMultipartUpload($key, $seekable, $concurrency) - { - // Create a temp file - $client = self::getSdk()->createS3(); - $tmpFile = sys_get_temp_dir() . '/aws-php-sdk-integ-s3-mup'; - file_put_contents($tmpFile, str_repeat('x', 10 * self::MB + 1024)); - - // Create a stream - $stream = Psr7\stream_for(fopen($tmpFile, 'r')); - if (!$seekable) { - $stream = new NoSeekStream($stream); - } - - // Create the uploader - $uploader = new MultipartUploader($client, $stream, [ - 'bucket' => self::getBucketName(), - 'key' => $key, - 'concurrency' => $concurrency - ]); - - // Perform the upload - try { - $result = $uploader->upload(); - $this->assertArrayHasKey('ObjectURL', $result); - } catch (MultipartUploadException $e) { - $client->abortMultipartUpload($e->getState()->getId()); - $message = "=====\n"; - while ($e) { - $message .= $e->getMessage() . "\n"; - $e = $e->getPrevious(); - } - $message .= "=====\n"; - $this->fail($message); - } - - @unlink($tmpFile); - } - - - private static function getBucketName() - { - static $bucketName; - - if (empty($bucketName)) { - $bucketName = self::getResourcePrefix() . self::BUCKET; - } - - return $bucketName; - } -} From 0b12a2c08001bf3e847c671a2c54bc9020638bb8 Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Fri, 6 Nov 2015 16:37:59 -0800 Subject: [PATCH 2/8] Move batching tests to cucumber suite --- behat.yml | 3 + features/batching/batchSerialization.feature | 8 + features/batching/writeRequestBatch.feature | 9 + tests/Integ/BatchingContext.php | 222 ++++++++++++++++++ tests/Integ/DynamoDbWriteRequestBatchTest.php | 91 ------- tests/Integ/SqsBatchSerializationTest.php | 77 ------ 6 files changed, 242 insertions(+), 168 deletions(-) create mode 100644 features/batching/batchSerialization.feature create mode 100644 features/batching/writeRequestBatch.feature create mode 100644 tests/Integ/BatchingContext.php delete mode 100644 tests/Integ/DynamoDbWriteRequestBatchTest.php delete mode 100644 tests/Integ/SqsBatchSerializationTest.php diff --git a/behat.yml b/behat.yml index 0b3e00542b..352132c9a1 100644 --- a/behat.yml +++ b/behat.yml @@ -9,3 +9,6 @@ default: multipart: paths: [ %paths.base%/features/multipart ] contexts: [ Aws\Test\Integ\MultipartContext ] + batching: + paths: [ %paths.base%/features/batching ] + contexts: [ Aws\Test\Integ\BatchingContext ] diff --git a/features/batching/batchSerialization.feature b/features/batching/batchSerialization.feature new file mode 100644 index 0000000000..8dd46b7d69 --- /dev/null +++ b/features/batching/batchSerialization.feature @@ -0,0 +1,8 @@ +@sqs +Feature: SQS Batch Serialization + + Scenario: Deleting Message Batches + Given I have a "sqs" client + And I have put 10 messages in a queue + When I delete a batch of 10 messages + Then 10 messages should have been deleted from the queue diff --git a/features/batching/writeRequestBatch.feature b/features/batching/writeRequestBatch.feature new file mode 100644 index 0000000000..1408be8bc1 --- /dev/null +++ b/features/batching/writeRequestBatch.feature @@ -0,0 +1,9 @@ +@dynamodb +Feature: DynamoDB Write Request Batch + Scenario: Batching + Given I have a "dynamodb" client + When I create a WriteRequestBatch with a batch size of 3 and a pool size of 2 + And I put 20 items in the batch + And I flush the batch + Then 20 items should have been written + And the batch should have been flushed 7 times diff --git a/tests/Integ/BatchingContext.php b/tests/Integ/BatchingContext.php new file mode 100644 index 0000000000..56fbfe87b5 --- /dev/null +++ b/tests/Integ/BatchingContext.php @@ -0,0 +1,222 @@ +client = self::getSdk()->createClient($service); + } + + /** + * @BeforeFeature @dynamodb + * + * @param BeforeFeatureScope $scope + */ + public static function setUpDynamoTable(BeforeFeatureScope $scope) + { + self::$resource = self::getResourcePrefix() . '-wrb-test'; + $client = self::getSdk()->createDynamoDb(); + $client->createTable([ + 'TableName' => self::$resource, + 'AttributeDefinitions' => [ + ['AttributeName' => 'id', 'AttributeType' => 'N'] + ], + 'KeySchema' => [ + ['AttributeName' => 'id', 'KeyType' => 'HASH'] + ], + 'ProvisionedThroughput' => [ + 'ReadCapacityUnits' => 1, + 'WriteCapacityUnits' => 1 + ] + ]); + + $client->waitUntil('TableExists', ['TableName' => self::$resource]); + } + + /** + * @AfterFeature @dynamodb + * + * @param AfterFeatureScope $scope + */ + public static function tearDownDynamoTable(AfterFeatureScope $scope) + { + self::getSdk() + ->createDynamoDb() + ->deleteTable(['TableName' => self::$resource]); + + self::$resource = null; + } + + /** + * @BeforeFeature @sqs + * + * @param BeforeFeatureScope $scope + */ + public static function setUpQueue(BeforeFeatureScope $scope) + { + $sqs = self::getSdk()->createSqs(); + self::$resource = self::getResourcePrefix() . '-batch-test'; + + $sqs->createQueue(['QueueName' => self::$resource]); + $sqs->waitUntil('QueueExists', ['QueueName' => self::$resource]); + } + + /** + * @AfterFeature @sqs + * + * @param AfterFeatureScope $scope + */ + public static function tearDownQueue(AfterFeatureScope $scope) + { + $sqs = self::getSdk() + ->createSqs(); + + $sqs->deleteQueue([ + 'QueueUrl' => $sqs->getQueueUrl([ + 'QueueName' => self::$resource, + ])['QueueUrl'] + ]); + } + + /** + * @When /^I create a WriteRequestBatch with a batch size of (\d+) and a pool size of (\d+)$/ + */ + public function iCreateAWriteRequestBatch($batchSize, $poolSize) + { + $this->batch = new WriteRequestBatch($this->client, [ + 'table' => self::$resource, + 'batch_size' => $batchSize, + 'pool_size' => $poolSize, + 'before' => function () { + $this->flushCount++; + }, + 'error' => function (AwsException $e) { + trigger_error($e->getMessage(), E_USER_WARNING); + } + ]); + } + + /** + * @When /^I put (\d+) items in the batch$/ + */ + public function iPutItemsInTheBatch($itemCount) + { + for ($i = 0; $i < $itemCount; $i++) { + $this->batch->put(['id' => ['N' => (string) $i]]); + } + } + + /** + * @When I flush the batch + */ + public function iFlushTheBatch() + { + $this->batch->flush(); + } + + /** + * @Then /^(\d+) items should have been written$/ + */ + public function itemsShouldHaveBeenWritten($itemCount) + { + $actualItems = $this->client->getIterator('Scan', [ + 'TableName' => self::$resource, + ]); + + $this->assertSame((int) $itemCount, iterator_count($actualItems)); + } + + /** + * @Then /^the batch should have been flushed (\d+) times$/ + */ + public function theBatchShouldHaveBeenFlushedTimes($flushCount) + { + $this->assertSame((int) $flushCount, $this->flushCount); + } + + /** + * @Given /^I have put (\d+) messages in a queue$/ + */ + public function iHavePutMessagesInAQueue($messageCount) + { + $queueUrl = $this->client + ->getQueueUrl(['QueueName' => self::$resource])['QueueUrl']; + for ($i = 0; $i < $messageCount; $i++) { + $this->client->sendMessage([ + 'QueueUrl' => $queueUrl, + 'MessageBody' => json_encode(['testing' => 'testing']), + ]); + } + } + + /** + * @When /^I delete a batch of (\d+) messages$/ + */ + public function iDeleteABatchOfMessages($messageCount) + { + $queueUrl = $this->client + ->getQueueUrl(['QueueName' => self::$resource])['QueueUrl']; + $messages = []; + while (count($messages) < $messageCount) { + $result = $this->client->receiveMessage([ + 'QueueUrl' => $queueUrl, + 'MaxNumberOfMessages' => $messageCount, + ]); + + foreach ($result['Messages'] as $message) { + $messages[$message['MessageId']] = [ + 'Id' => $message['MessageId'], + 'ReceiptHandle' => $message['ReceiptHandle'], + ]; + } + } + + $this->response = $this->client + ->deleteMessageBatch([ + 'QueueUrl' => $queueUrl, + 'Entries' => array_values($messages), + ]); + } + + /** + * @Then /^(\d+) messages should have been deleted from the queue$/ + */ + public function messagesShouldHaveBeenDeletedFromTheQueue($messageCount) + { + $this->assertSame((int) $messageCount, count($this->response['Failed']) + + count($this->response['Successful'])); + } +} diff --git a/tests/Integ/DynamoDbWriteRequestBatchTest.php b/tests/Integ/DynamoDbWriteRequestBatchTest.php deleted file mode 100644 index 73e1c261ab..0000000000 --- a/tests/Integ/DynamoDbWriteRequestBatchTest.php +++ /dev/null @@ -1,91 +0,0 @@ -client, [ - 'table' => $this->table, - 'batch_size' => 3, - 'pool_size' => 2, - 'before' => function () use (&$autoFlushCount) { - self::log("The WriteRequestBatch was auto-flushed."); - $autoFlushCount++; - }, - 'error' => function (AwsException $e) { - self::log("There was an error: " . $e->getMessage()); - } - ]); - - $itemCount = 20; - for ($i = 0; $i < $itemCount; $i++) { - self::log("Putting an item to the WriteRequestBatch."); - $batch->put(['id' => ['N' => (string) $i]]); - } - self::log("Flushing the final items in the WriteRequestBatch."); - $batch->flush(); - - $actualItems = $this->client->getIterator('Scan', ['TableName' => $this->table]); - - // Assert that all the items were actually written. - $this->assertEquals($itemCount, iterator_count($actualItems)); - // Assert that there were the correct number of auto-flushes. - $this->assertEquals(7, $autoFlushCount); - } - - public function setUp() - { - $this->client = self::getSdk()->createDynamoDb(); - $this->table = self::getResourcePrefix() . '-wrb-test'; - - $this->cleanUpTable(); - - self::log("Creating table {$this->table}."); - try { - $this->client->createTable([ - 'TableName' => $this->table, - 'AttributeDefinitions' => [ - ['AttributeName' => 'id', 'AttributeType' => 'N'] - ], - 'KeySchema' => [ - ['AttributeName' => 'id', 'KeyType' => 'HASH'] - ], - 'ProvisionedThroughput' => [ - 'ReadCapacityUnits' => 1, - 'WriteCapacityUnits' => 1 - ] - ]); - self::log("Waiting until table {$this->table} has been created."); - $this->client->waitUntil('TableExists', ['TableName' => $this->table]); - } catch (DynamoDbException $e) { - $this->fail("Could not create table {$this->table}."); - } - } - - public function tearDown() - { - $this->cleanUpTable(); - } - - private function cleanUpTable() - { - self::log("Deleting table {$this->table}."); - try { - $this->client->deleteTable(['TableName' => $this->table]); - } catch (DynamoDbException $e) { - self::log("Table {$this->table} does not exist."); - } - } -} diff --git a/tests/Integ/SqsBatchSerializationTest.php b/tests/Integ/SqsBatchSerializationTest.php deleted file mode 100644 index c8641beaa0..0000000000 --- a/tests/Integ/SqsBatchSerializationTest.php +++ /dev/null @@ -1,77 +0,0 @@ -createSqs(); - self::cleanup($client, $queue); - self::log('Creating queue ' . $queue); - $client->createQueue(['QueueName' => $queue]); - } - - public static function tearDownAfterClass() - { - $queue = self::getIntegTestingQueueName(); - $client = self::getSdk()->createSqs(); - self::cleanup($client, $queue); - } - - public function testDeleteMessageBatchSendsCorrectly() - { - $client = self::getSdk()->createSqs(); - - $queue = $client->getQueueUrl(['QueueName' => self::getIntegTestingQueueName()]); - for ($i = 0; $i < 10; $i++) { - $client->sendMessage([ - 'QueueUrl' => $queue->get('QueueUrl'), - 'MessageBody' => json_encode(['testing' => 'testing']), - ]); - } - - $messages = $client->receiveMessage([ - 'QueueUrl' => $queue->get('QueueUrl'), - 'MaxNumberOfMessages' => 10, - ]); - - $client->deleteMessageBatch([ - 'QueueUrl' => $queue->get('QueueUrl'), - 'Entries' => array_map( - function (array $message) { - return [ - 'Id' => $message['MessageId'], - 'ReceiptHandle' => $message['ReceiptHandle'], - ]; - }, - $messages->get('Messages') - ) - ]); - } - - private static function cleanup(SqsClient $client, $queue) - { - try { - $queue = $client->getQueueUrl([ - 'QueueName' => $queue, - ]); - - $client->deleteQueue(['QueueUrl' => $queue->get('QueueUrl')]); - - return true; - } catch (\Exception $e) { - return false; - } - } - - private static function getIntegTestingQueueName() - { - return self::getResourcePrefix() . '-integ-testing-queue'; - } -} From 81d85a2556db38891426b24394635316569d0eb8 Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Fri, 6 Nov 2015 18:32:12 -0800 Subject: [PATCH 3/8] Move waiter test to cucumber --- behat.yml | 3 + features/blocking/waiters.feature | 24 +++++ tests/Integ/BatchingContext.php | 9 +- tests/Integ/BlockingContext.php | 158 ++++++++++++++++++++++++++++++ tests/Integ/WaiterTest.php | 126 ------------------------ 5 files changed, 192 insertions(+), 128 deletions(-) create mode 100644 features/blocking/waiters.feature create mode 100644 tests/Integ/BlockingContext.php delete mode 100644 tests/Integ/WaiterTest.php diff --git a/behat.yml b/behat.yml index 352132c9a1..42fdb08722 100644 --- a/behat.yml +++ b/behat.yml @@ -12,3 +12,6 @@ default: batching: paths: [ %paths.base%/features/batching ] contexts: [ Aws\Test\Integ\BatchingContext ] + blocking: + paths: [ %paths.base%/features/blocking ] + contexts: [ Aws\Test\Integ\BlockingContext ] diff --git a/features/blocking/waiters.feature b/features/blocking/waiters.feature new file mode 100644 index 0000000000..496429ac24 --- /dev/null +++ b/features/blocking/waiters.feature @@ -0,0 +1,24 @@ +@dynamodb +Feature: Waiters + + Scenario: Synchronous Waiters + Given I have a "dynamodb" client + And the table named "waiter-test" does not exist + When I create a table named "waiter-test" + And wait for the table named "waiter-test" to exist + Then the table named "waiter-test" exists + Then I can delete the table named "waiter-test" + And wait for the table named "waiter-test" to be deleted + And the table named "waiter-test" does not exist + + Scenario: Asynchronous Waiters + Given I have a "dynamodb" client + And the table named "waiter-test" does not exist + When I create a promise to create and await a table named "waiter-test" + And the table named "waiter-test" does not exist + Then I can wait on all promises + And the table named "waiter-test" exists + When I create a promise to delete and await the purging of the table named "waiter-test" + And the table named "waiter-test" exists + Then I can wait on all promises + And the table named "waiter-test" does not exist diff --git a/tests/Integ/BatchingContext.php b/tests/Integ/BatchingContext.php index 56fbfe87b5..1728c21280 100644 --- a/tests/Integ/BatchingContext.php +++ b/tests/Integ/BatchingContext.php @@ -47,7 +47,9 @@ public function iHaveAClient($service) */ public static function setUpDynamoTable(BeforeFeatureScope $scope) { - self::$resource = self::getResourcePrefix() . '-wrb-test'; + self::$resource = self::getResourcePrefix() + . str_replace(' ', '-', strtolower($scope->getName())); + $client = self::getSdk()->createDynamoDb(); $client->createTable([ 'TableName' => self::$resource, @@ -88,7 +90,8 @@ public static function tearDownDynamoTable(AfterFeatureScope $scope) public static function setUpQueue(BeforeFeatureScope $scope) { $sqs = self::getSdk()->createSqs(); - self::$resource = self::getResourcePrefix() . '-batch-test'; + self::$resource = self::getResourcePrefix() + . str_replace(' ', '-', strtolower($scope->getName())); $sqs->createQueue(['QueueName' => self::$resource]); $sqs->waitUntil('QueueExists', ['QueueName' => self::$resource]); @@ -109,6 +112,8 @@ public static function tearDownQueue(AfterFeatureScope $scope) 'QueueName' => self::$resource, ])['QueueUrl'] ]); + + self::$resource = null; } /** diff --git a/tests/Integ/BlockingContext.php b/tests/Integ/BlockingContext.php new file mode 100644 index 0000000000..991593169f --- /dev/null +++ b/tests/Integ/BlockingContext.php @@ -0,0 +1,158 @@ +client = self::getSdk()->createClient($service); + } + + /** + * @When I create a table named :table + */ + public function iCreateATableNamed($table) + { + $this->client->createTable([ + 'TableName' => self::getResourcePrefix() . "-$table", + 'AttributeDefinitions' => [ + ['AttributeName' => 'id', 'AttributeType' => 'N'] + ], + 'KeySchema' => [ + ['AttributeName' => 'id', 'KeyType' => 'HASH'] + ], + 'ProvisionedThroughput' => [ + 'ReadCapacityUnits' => 20, + 'WriteCapacityUnits' => 20 + ] + ]); + } + + /** + * @When wait for the table named :table to exist + */ + public function waitForTheTableNamedToExist($table) + { + $this->client->waitUntil('TableExists', [ + 'TableName' => self::getResourcePrefix() . "-$table", + ]); + } + + /** + * @When the table named :table exists + */ + public function theTableNamedWillExist($table) + { + self::getSdk(['http' => ['synchronous' => true]]) + ->createDynamoDb() + ->describeTable(['TableName' => self::getResourcePrefix() . "-$table"]); + } + + /** + * @Then I can delete the table named :table + */ + public function iCanDeleteTheTableNamed($table) + { + $this->client->deleteTable([ + 'TableName' => self::getResourcePrefix() . "-$table", + ]); + } + + /** + * @Then wait for the table named :table to be deleted + */ + public function waitForTheTableNamedToBeDeleted($table) + { + $this->client->waitUntil('TableNotExists', [ + 'TableName' => self::getResourcePrefix() . "-$table", + ]); + } + + /** + * @Then the table named :table does not exist + */ + public function theTableNamedWillNotExist($table) + { + try { + $this->theTableNamedWillExist($table); + $this->fail("$table exists but should not."); + } catch (DynamoDbException $e) { + $this->assertSame('ResourceNotFoundException', $e->getAwsErrorCode()); + } + } + + /** + * @When I create a promise to create and await a table named :table + */ + public function iCreateAPromiseToCreateAndAwaitATableNamed($table) + { + $this->promises []= $this->client->createTableAsync([ + 'TableName' => self::getResourcePrefix() . "-$table", + 'AttributeDefinitions' => [ + ['AttributeName' => 'id', 'AttributeType' => 'N'] + ], + 'KeySchema' => [ + ['AttributeName' => 'id', 'KeyType' => 'HASH'] + ], + 'ProvisionedThroughput' => [ + 'ReadCapacityUnits' => 20, + 'WriteCapacityUnits' => 20 + ] + ]) + ->then(function () use ($table) { + return $this->client + ->getWaiter('TableExists', [ + 'TableName' => self::getResourcePrefix() . "-$table", + ])->promise(); + }); + } + + /** + * @Then I can wait on all promises + */ + public function iCanWaitOnAllPromises() + { + Promise\all($this->promises) + ->wait(); + } + + /** + * @When I create a promise to delete and await the purging of the table named :table + */ + public function iCreateAPromiseToDeleteAndAwaitThePurgingOfTheTableNamed($table) + { + $this->promises []= $this->client + ->deleteTableAsync([ + 'TableName' => self::getResourcePrefix() . "-$table", + ])->then(function () use ($table) { + return $this->client + ->getWaiter('TableNotExists', [ + 'TableName' => self::getResourcePrefix() . "-$table", + ])->promise(); + }); + } +} diff --git a/tests/Integ/WaiterTest.php b/tests/Integ/WaiterTest.php deleted file mode 100644 index 11ad15a205..0000000000 --- a/tests/Integ/WaiterTest.php +++ /dev/null @@ -1,126 +0,0 @@ -createDynamoDb(); - $table = self::getResourcePrefix() . '-test-table'; - - self::log('Testing synchronous waiters.'); - - try { - self::log('Creating table.'); - $client->createTable([ - 'TableName' => $table, - 'AttributeDefinitions' => [ - ['AttributeName' => 'id', 'AttributeType' => 'N'] - ], - 'KeySchema' => [ - ['AttributeName' => 'id', 'KeyType' => 'HASH'] - ], - 'ProvisionedThroughput' => [ - 'ReadCapacityUnits' => 20, - 'WriteCapacityUnits' => 20 - ] - ]); - - self::log('Waiting for the table to be active.'); - $client->waitUntil( - 'TableExists', - ['TableName' => $table], - ['retry' => function ($attempt) { - self::log("TableExists waiter has made {$attempt} attempts."); - }] - ); - - self::log('Deleting table.'); - $client->deleteTable(['TableName' => $table]); - - self::log('Waiting for the table to be deleted.'); - $client->waitUntil( - 'TableNotExists', - ['TableName' => $table], - [ - 'initDelay' => 1, - 'retry' => function ($attempt) { - self::log("TableNotExists waiter has made {$attempt} attempts."); - } - ] - ); - - self::log('All done waiting.'); - } catch (\Exception $e) { - self::log($e->getMessage()); - $this->fail('Synchronous waiters failed.'); - } - } - - public function testWaiterWorkflows() - { - self::log('Testing complicated waiter workflows.'); - $sdk = self::getSdk(); - $promises = []; - - self::log('Creating a DynamoDB table.'); - $client = $sdk->createDynamoDb(); - $table = self::getResourcePrefix() . '-test-table'; - $promises[] = $client->createTableAsync([ - 'TableName' => $table, - 'AttributeDefinitions' => [ - ['AttributeName' => 'id', 'AttributeType' => 'N'] - ], - 'KeySchema' => [ - ['AttributeName' => 'id', 'KeyType' => 'HASH'] - ], - 'ProvisionedThroughput' => [ - 'ReadCapacityUnits' => 20, - 'WriteCapacityUnits' => 20 - ] - ])->then( - function () use ($client, $table) { - self::log('Waiting for the table to be active.'); - return $client->getWaiter('TableExists', ['TableName' => $table])->promise(); - } - )->then( - function () use ($client, $table) { - return $client->deleteTableAsync(['TableName' => $table]); - } - )->then( - function () use ($client, $table) { - self::log('Deleting table.'); - self::log('Waiting for the table to be deleted.'); - return $client->getWaiter('TableNotExists', ['TableName' => $table])->promise(); - } - )->then( - function () use ($client, $table) { - self::log('All done waiting.'); - }, - function (AwsException $error) { - self::log($error->getMessage()); - $this->fail('Asynchronous waiters failed.'); - } - ); - - self::log('Initiating an S3 ListBuckets operation.'); - $promises[] = $sdk->createS3() - ->listBucketsAsync() - ->then(function () { - self::log('Completed the ListBuckets operation.'); - }); - - try { - Promise\all($promises)->then(function () { - self::log('Done with everything!'); - })->wait(); - } catch (\Exception $e) { - $this->fail($e->getMessage()); - } - } -} From 13e20d566f1652015cc3699cc380b456178a686d Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Tue, 10 Nov 2015 12:45:07 -0800 Subject: [PATCH 4/8] Move stream tests to cucumber suite --- behat.yml | 3 + features/streams/streamWrapper.feature | 57 +++++++ tests/Integ/NativeStreamContext.php | 196 +++++++++++++++++++++++++ tests/Integ/S3StreamWrapperTest.php | 153 ------------------- 4 files changed, 256 insertions(+), 153 deletions(-) create mode 100644 features/streams/streamWrapper.feature create mode 100644 tests/Integ/NativeStreamContext.php delete mode 100644 tests/Integ/S3StreamWrapperTest.php diff --git a/behat.yml b/behat.yml index 42fdb08722..8774f1e152 100644 --- a/behat.yml +++ b/behat.yml @@ -15,3 +15,6 @@ default: blocking: paths: [ %paths.base%/features/blocking ] contexts: [ Aws\Test\Integ\BlockingContext ] + streams: + paths: [ %paths.base%/features/streams ] + contexts: [ Aws\Test\Integ\NativeStreamContext ] diff --git a/features/streams/streamWrapper.feature b/features/streams/streamWrapper.feature new file mode 100644 index 0000000000..1d9bd4a0d0 --- /dev/null +++ b/features/streams/streamWrapper.feature @@ -0,0 +1,57 @@ +#language: en +@s3 +Feature: S3 Stream Wrapper + + Background: + Given I have a "s3" client + And have registered an s3 stream wrapper + + Scenario: Making directories + Given I create a subdirectory "subdir" with mkdir + When I call is_dir on the subdir path + Then the call should return true + + Scenario Outline: Checking existence of directories + When I call is_dir on the path + Then the call should return + + Examples: + | subdirectory | boolean | + | / | true | + | /foo | false | + | /bar | false | + + Scenario: Uploading Files + Given I have a file at "key" with the content "testing!" + When I call file_exists on the key path + Then the call should return true + And the file at "key" should contain "testing!" + + Scenario: Deleting Files + Given I have a file at "key" with the content "testing!" + When I call unlink on the key path + And I call file_exists on the key path + Then the call should return false + + Scenario Outline: Opening streams + Given I have a file at "" with the content "" + And I have a read handle on the file at "" + Then reading 2 bytes should return + And reading 1000 bytes should return + And calling fstat should report a size of + + Examples: + | path | contents | size | first2 | next1000 | + | key1 | testing! | 8 | te | sting! | + | key2 | foo, bar, baz | 13 | fo | o, bar, baz | + + Scenario: No errors raised for missing files + Given I have cleared the last error + When I call file_exists on the jkfdsalhjkgdfhsurew path + Then the call should return false + And no errors should have been raised + + Scenario: Traversing empty directories + Given I have a file at "/empty/" with no content + When I have a file at "/empty/bar" with the content "hello" + Then scanning the directory at "/empty/" should return a list with one member named "bar" diff --git a/tests/Integ/NativeStreamContext.php b/tests/Integ/NativeStreamContext.php new file mode 100644 index 0000000000..8ed1ed0471 --- /dev/null +++ b/tests/Integ/NativeStreamContext.php @@ -0,0 +1,196 @@ +handle)) { + fclose($this->handle); + } + } + + /** + * @BeforeFeature @s3 + * + * @param BeforeFeatureScope $scope + */ + public static function setUpS3Bucket(BeforeFeatureScope $scope) + { + $client = self::getSdk() + ->createS3(); + + self::$bucket = self::getResourcePrefix() + . str_replace(' ', '-', strtolower($scope->getName())); + + $client->createBucket(['Bucket' => self::$bucket]); + $client->waitUntil('BucketExists', ['Bucket' => self::$bucket]); + } + + /** + * @AfterFeature @s3 + * + * @param AfterFeatureScope $scope + */ + public static function tearDownS3Bucket(AfterFeatureScope $scope) + { + $client = self::getSdk()->createS3(); + + $client->deleteMatchingObjects(self::$bucket, '', '//'); + $client->deleteBucket(['Bucket' => self::$bucket]); + + self::$bucket = null; + } + + /** + * @Given I have a :service client + */ + public function iHaveAClient($service) + { + $this->client = self::getSdk()->createClient($service); + } + + /** + * @Given have registered an s3 stream wrapper + */ + public function haveRegisteredAnS3StreamWrapper() + { + $this->client->registerStreamWrapper(); + } + + /** + * @Given I create a subdirectory :subdir with mkdir + */ + public function iCreateASubdirectory($subdir) + { + mkdir($this->getS3Path($subdir)); + sleep(1); + } + + /** + * @When /^I call (\w+) on the (\S+) path$/ + */ + public function iCallOnThePath($method, $path) + { + $this->callSucceeded = call_user_func($method, $this->getS3Path($path)); + } + + /** + * @Then /^the call should return (true|false)$/ + */ + public function theCallShouldReturn($booleanString) + { + $this->assertSame( + filter_var($booleanString, FILTER_VALIDATE_BOOLEAN), + $this->callSucceeded + ); + } + + /** + * @Given I have a file at :path with the content :contents + */ + public function iHaveAFileAtWithTheContent($path, $contents) + { + $this->assertGreaterThan( + 0, + file_put_contents($this->getS3Path($path), $contents) + ); + } + + /** + * @Given I have a file at :path with no content + */ + public function iHaveAFileAtWithNoContent($path) + { + $this->assertSame(0, file_put_contents($this->getS3Path($path), '')); + } + + /** + * @Then the file at :arg1 should contain :arg2 + */ + public function theFileAtShouldContain($key, $contents) + { + $this->assertSame($contents, file_get_contents($this->getS3Path($key))); + } + + /** + * @Given I have a read handle on the file at :arg1 + */ + public function iHaveAReadHandleOnTheFileAt($key) + { + $this->handle = fopen($this->getS3Path($key), 'r'); + } + + /** + * @Then /^reading (\d+) bytes should return (.+)$/ + */ + public function readingBytesShouldReturn($byteCount, $expected) + { + $this->assertSame($expected, fread($this->handle, $byteCount)); + } + + /** + * @Then /^calling fstat should report a size of (\d+)$/ + */ + public function callingFstatShouldReportASizeOf($size) + { + $this->assertSame((int) $size, fstat($this->handle)['size']); + } + + /** + * @Given I have cleared the last error + */ + public function iHaveClearedTheLastError() + { + while (error_get_last()) { + error_clear_last(); + } + } + + /** + * @Then no errors should have been raised + */ + public function errorsShouldHaveBeenRaised() + { + $this->assertNull(error_get_last()); + } + + /** + * @Then scanning the directory at :dir should return a list with one member named :file + */ + public function scanningTheDirectoryAtShouldReturnAListWithOneMemberNamed($dir, $file) + { + $this->assertSame([$file], scandir($this->getS3Path($dir))); + } + + private function getS3Path($path) + { + return 's3://' . self::$bucket . '/' . ltrim($path, '/'); + } +} diff --git a/tests/Integ/S3StreamWrapperTest.php b/tests/Integ/S3StreamWrapperTest.php deleted file mode 100644 index 7ef7ba573b..0000000000 --- a/tests/Integ/S3StreamWrapperTest.php +++ /dev/null @@ -1,153 +0,0 @@ -doesBucketExist($bucket)) { - self::log($bucket . ' exists... Deleting'); - $delete = BatchDelete::fromListObjects($client, ['Bucket' => $bucket]); - $delete->delete(); - try { - $client->deleteBucket(['Bucket' => $bucket]); - } catch (\Exception $e) {} - self::log($bucket . ' deleted'); - return true; - } - - return false; - } - - public static function setUpBeforeClass() - { - $bucket = self::getResourcePrefix() . 'php-stream'; - $client = self::getSdk()->createS3(); - $client->registerStreamWrapper(); - if (self::cleanup($client, $bucket)) { - $client->waitUntil('BucketNotExists', ['Bucket' => $bucket]); - } - self::log('Creating bucket ' . $bucket); - mkdir('s3://' . $bucket); - $client->waitUntil('BucketExists', ['Bucket' => $bucket]); - } - - public static function tearDownAfterClass() - { - $bucket = self::getResourcePrefix() . 'php-stream'; - $client = self::getSdk()->createS3(); - self::cleanup($client, $bucket); - } - - public function setUp() - { - $this->bucket = $this->getResourcePrefix() . 'php-stream'; - $client = self::getSdk()->createS3(); - $client->waitUntil('BucketExists', ['Bucket' => $this->bucket]); - } - - public function testMkdirs() - { - $path = 's3://' . $this->bucket . '/subdir'; - $this->assertTrue(mkdir($path)); - sleep(1); - $this->assertTrue(is_dir($path)); - unlink($path); - } - - /** - * @depends testMkdirs - */ - public function testChecksIfThingsExist() - { - $this->assertTrue(is_dir('s3://' . $this->bucket . '/')); - $this->assertFalse(is_dir('s3://wefwefwe' . $this->bucket)); - $this->assertFalse(is_file('s3://wefwefwe' . $this->bucket . '/wefweewegr')); - } - - /** - * @depends testChecksIfThingsExist - */ - public function testUploadsFile() - { - self::log('Uploading a simple file'); - $path = $this->getKey('simple'); - file_put_contents($path, 'testing!'); - $this->assertEquals('testing!', file_get_contents($path)); - - return $path; - } - - /** - * @depends testUploadsFile - */ - public function testDoesFileExist($path) - { - $this->assertTrue(file_exists($path)); - $this->assertTrue(is_file($path)); - - return $path; - } - - /** - * @depends testDoesFileExist - */ - public function testDeletesFiles($path) - { - unlink($path); - sleep(1); - $this->assertFalse(file_exists($path)); - } - - /** - * @depends testDeletesFiles - */ - public function testOpensStreams() - { - $path = $this->getKey('stream'); - file_put_contents($path, 'testing'); - $client = self::getSdk()->createS3(); - $client->waitUntil('ObjectExists', ['Bucket' => $this->bucket, 'Key' => 'stream']); - $this->assertEquals('testing', file_get_contents($path)); - $h = fopen($path, 'r'); - $this->assertEquals('te', fread($h, 2)); - $this->assertEquals('sting', fread($h, 1000)); - $stat = fstat($h); - $this->assertEquals(7, $stat['size']); - fclose($h); - } - - /** - * @depends testOpensStreams - */ - public function testDoesNotRaiseErrorForMissingFile() - { - self::log('Testing invalid file'); - $this->assertFalse(is_file('s3://ewfwefwfeweff/' . uniqid('foo'))); - $this->assertFalse(is_link('s3://ewfwefwfeweff/' . uniqid('foo'))); - try { - lstat('s3://ewfwefwfeweff/' . uniqid('foo')); - $this->fail('Did not trigger a warning'); - } catch (\PHPUnit_Framework_Error_Warning $e) {} - } - - public function testCanListWithEmptyDirs() - { - $file = 's3://' . $this->bucket . '/empty/'; - file_put_contents($file, ''); - file_put_contents($file . 'bar', 'hello'); - $this->assertEquals(['bar'], scandir($file)); - } - - private function getKey($name) - { - return 's3://' . $this->bucket . '/' . $name; - } -} From 7f71c2733b3c5f39642209e81d371a3f7e89254e Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Wed, 2 Dec 2015 10:20:31 -0800 Subject: [PATCH 5/8] Move encoding tests to unit suite --- tests/CognitoSync/CognitoSyncClientTest.php | 38 ++++++ ...Test.php => GuzzleV6StreamHandlerTest.php} | 2 +- tests/Integ/PaginatorTest.php | 119 ------------------ tests/Integ/SignatutePathEncodingTest.php | 50 -------- tests/S3/S3ClientTest.php | 21 ++++ 5 files changed, 60 insertions(+), 170 deletions(-) create mode 100644 tests/CognitoSync/CognitoSyncClientTest.php rename tests/Integ/{UseStreamTest.php => GuzzleV6StreamHandlerTest.php} (97%) delete mode 100644 tests/Integ/PaginatorTest.php delete mode 100644 tests/Integ/SignatutePathEncodingTest.php diff --git a/tests/CognitoSync/CognitoSyncClientTest.php b/tests/CognitoSync/CognitoSyncClientTest.php new file mode 100644 index 0000000000..2531faf0c1 --- /dev/null +++ b/tests/CognitoSync/CognitoSyncClientTest.php @@ -0,0 +1,38 @@ +getTestClient('CognitoSync', [ + 'http_handler' => function (RequestInterface $request) use ( + $identityId, + $identityPoolId + ) { + foreach ([$identityId, $identityPoolId] as $unencodedString) { + $this->assertContains( + urlencode($unencodedString), + (string) $request->getUri() + ); + } + + return new FulfilledPromise(new Response); + }, + ]); + + $client->describeIdentityUsage([ + 'IdentityId' => $identityId, + 'IdentityPoolId' => $identityPoolId, + ]); + } +} diff --git a/tests/Integ/UseStreamTest.php b/tests/Integ/GuzzleV6StreamHandlerTest.php similarity index 97% rename from tests/Integ/UseStreamTest.php rename to tests/Integ/GuzzleV6StreamHandlerTest.php index 890b3cd823..18236ffc79 100644 --- a/tests/Integ/UseStreamTest.php +++ b/tests/Integ/GuzzleV6StreamHandlerTest.php @@ -3,7 +3,7 @@ use GuzzleHttp\Handler\StreamHandler; -class UseStreamTest extends \PHPUnit_Framework_TestCase +class GuzzleV6StreamHandlerTest extends \PHPUnit_Framework_TestCase { use IntegUtils; diff --git a/tests/Integ/PaginatorTest.php b/tests/Integ/PaginatorTest.php deleted file mode 100644 index 9f915d80f6..0000000000 --- a/tests/Integ/PaginatorTest.php +++ /dev/null @@ -1,119 +0,0 @@ -getCommand('PutObject', [ - 'Bucket' => $bucket, - 'Key' => 'object' . $i, - 'Body' => 'foo' - ]); - } - } - - public function testNormalPaginators() - { - $client = self::getSdk()->createS3(); - $bucket = self::getResourcePrefix() . '-php-sdk-test-bucket'; - - self::log('Testing synchronous paginators.'); - - try { - self::log('Creating bucket.'); - $client->createBucket(['Bucket' => $bucket]); - - self::log('Waiting for the bucket to be available.'); - $client->waitUntil('BucketExists', ['Bucket' => $bucket]); - - self::log('Uploading objects.'); - CommandPool::batch($client, $this->getCommands($client, $bucket, 100)); - - self::log('List the objects.'); - $results = $client->getPaginator('ListObjects', [ - 'Bucket' => $bucket, - 'MaxKeys' => 20 - ]); - $objects = $results->search('Contents[].Key'); - $this->assertEquals(100, iterator_count($objects)); - - self::log('Deleting objects.'); - BatchDelete::fromListObjects($client, ['Bucket' => $bucket])->delete(); - - self::log('Deleting bucket.'); - $client->deleteBucket(['Bucket' => $bucket]); - - self::log('Waiting for the bucket to be deleted.'); - $client->waitUntil('BucketNotExists', ['Bucket' => $bucket], ['initDelay' => 1]); - - self::log('All done waiting.'); - } catch (\Exception $e) { - self::log($e->getMessage()); - $this->fail('Synchronous paginators failed.'); - } - } - - public function testAsyncPaginators() - { - $sdk = self::getSdk(); - $client = $sdk->createS3(); - $bucket = self::getResourcePrefix() . '-php-sdk-test-bucket-async'; - - self::log('Testing asynchronous paginators.'); - $promises = []; - $promises[] = Promise\coroutine(function () use ($client, $bucket) { - self::log('Creating bucket.'); - yield $client->createBucketAsync(['Bucket' => $bucket]); - - self::log('Waiting for the bucket to be available.'); - yield $client->getWaiter('BucketExists', ['Bucket' => $bucket])->promise(); - - self::log('Uploading objects.'); - yield (new CommandPool($client, $this->getCommands($client, $bucket, 100)))->promise(); - - self::log('List the objects.'); - $pages = 0; - yield $client->getPaginator('ListObjects', [ - 'Bucket' => $bucket, - 'MaxKeys' => 20 - ])->each(function (Result $result) use (&$pages) { - $pages++; - }); - $this->assertEquals(5, $pages); - - self::log('Deleting objects.'); - yield BatchDelete::fromListObjects($client, ['Bucket' => $bucket])->promise(); - - self::log('Deleting bucket.'); - yield $client->deleteBucketAsync(['Bucket' => $bucket]); - - self::log('Waiting for the bucket to be deleted.'); - yield $client->getWaiter('BucketNotExists', ['Bucket' => $bucket])->promise(); - }); - - self::log('Initiating a DynamoDB ListTables operation.'); - $promises[] = $sdk->createDynamoDb() - ->listTablesAsync() - ->then(function () { - self::log('Completed the DynamoDB ListTables operation.'); - }); - - try { - Promise\all($promises)->wait(); - self::log('Done with everything!'); - } catch (\Exception $e) { - self::log($e->getMessage()); - $this->fail('Asynchronous paginators failed.'); - } - } -} diff --git a/tests/Integ/SignatutePathEncodingTest.php b/tests/Integ/SignatutePathEncodingTest.php deleted file mode 100644 index b62c0c5aea..0000000000 --- a/tests/Integ/SignatutePathEncodingTest.php +++ /dev/null @@ -1,50 +0,0 @@ -getSdk()->createClient('CognitoSync'); - $error = null; - try { - $client->describeIdentityUsage([ - 'IdentityId' => 'aaa:bbb', - 'IdentityPoolId' => 'ccc:ddd', - ]); - } catch (CognitoSyncException $e) { - $error = $e->getAwsErrorCode(); - } - - $this->assertEquals('ResourceNotFoundException', $error); - } - - public function testS3RequestSucceedsWithColon() - { - $s3 = $this->getSdk()->createClient('S3'); - $bucket = self::getResourcePrefix() . '-php-sdk-colon-test-bucket'; - - $s3->createBucketAsync(['Bucket' => $bucket])->then(function () use ($s3, $bucket) { - return $s3->getWaiter('BucketExists', ['Bucket' => $bucket])->promise(); - })->wait(); - - $error = null; - try { - $s3->getObject([ - 'Bucket' => $bucket, - 'Key' => 'aaa:bbb', - ]); - } catch (S3Exception $e) { - $error = $e->getAwsErrorCode(); - } - - $s3->deleteBucket(['Bucket' => $bucket]); - - $this->assertEquals('NoSuchKey', $error); - } -} diff --git a/tests/S3/S3ClientTest.php b/tests/S3/S3ClientTest.php index 753c0dfeb6..9cab947089 100644 --- a/tests/S3/S3ClientTest.php +++ b/tests/S3/S3ClientTest.php @@ -13,6 +13,7 @@ use GuzzleHttp\Psr7; use GuzzleHttp\Psr7\FnStream; use GuzzleHttp\Psr7\Response; +use Psr\Http\Message\RequestInterface; use Psr\Http\Message\StreamInterface; /** @@ -413,6 +414,26 @@ public function testSaveAsParamAddsSink() $this->assertEquals('sink=baz', (string) $result['Body']); } + public function testRequestSucceedsWithColon() + { + $key = 'aaa:bbb'; + $s3 = $this->getTestClient('S3', [ + 'http_handler' => function (RequestInterface $request) use ($key) { + $this->assertContains( + urlencode($key), + (string) $request->getUri() + ); + + return Promise\promise_for(new Psr7\Response); + } + ]); + + $s3->getObject([ + 'Bucket' => 'bucket', + 'Key' => $key, + ]); + } + public function testRetriesConnectionErrors() { $retries = 11; From 4531305f7da5e740c454de0e9bf2d6784b1f9da5 Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Mon, 7 Dec 2015 12:58:49 -0800 Subject: [PATCH 6/8] Move concurrency tests to cucumber suite --- behat.yml | 3 + features/concurrency/requests.feature | 42 +++++++ tests/ConcurrencyTest.php | 101 --------------- tests/Integ/ConcurrencyContext.php | 173 ++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 101 deletions(-) create mode 100644 features/concurrency/requests.feature delete mode 100644 tests/ConcurrencyTest.php create mode 100644 tests/Integ/ConcurrencyContext.php diff --git a/behat.yml b/behat.yml index 8774f1e152..f76d24c0d9 100644 --- a/behat.yml +++ b/behat.yml @@ -15,6 +15,9 @@ default: blocking: paths: [ %paths.base%/features/blocking ] contexts: [ Aws\Test\Integ\BlockingContext ] + concurrency: + paths: [ %paths.base%/features/concurrency ] + contexts: [ Aws\Test\Integ\ConcurrencyContext ] streams: paths: [ %paths.base%/features/streams ] contexts: [ Aws\Test\Integ\NativeStreamContext ] diff --git a/features/concurrency/requests.feature b/features/concurrency/requests.feature new file mode 100644 index 0000000000..0eda99242a --- /dev/null +++ b/features/concurrency/requests.feature @@ -0,0 +1,42 @@ +@s3 +Feature: Concurrent Requests + + Scenario: Sending a normal, synchronous request + Given I have a "s3" client + When I call the "ListBuckets" API + Then the value at "Owner.ID" should be a "string" + + Scenario: Sending a promised, synchronous request + Given I have a "s3" client + When I call the "ListBuckets" API asynchronously + And I wait on the promise + Then the value at "Owner.ID" should be a "string" + + Scenario: Sending requests asynchronously + Given a promise composed of the following asynchronous operations: + | service | command | payload | + | s3 | ListBuckets | | + | s3 | ListBuckets | | + | s3 | ListBuckets | | + When I wait on the promise + Then there should be 3 results + And there should be 1 value at "[*].Owner.ID" + + Scenario: Sending commands concurrently + Given a pool composed of the following commands: + | service | command | payload | + | s3 | ListBuckets | | + | s3 | ListBuckets | | + | s3 | ListBuckets | | + When I send the commands as a batch to "s3" + Then there should be 3 results + And there should be 1 value at "[*].Owner.ID" + + Scenario: Sending requests asynchronously to multiple services + Given a promise composed of the following asynchronous operations: + | service | command | payload | + | s3 | ListBuckets | | + | dynamodb | ListTables | | + | sqs | ListQueues | | + When I wait on the promise + Then there should be 3 results diff --git a/tests/ConcurrencyTest.php b/tests/ConcurrencyTest.php deleted file mode 100644 index 2d7faafe44..0000000000 --- a/tests/ConcurrencyTest.php +++ /dev/null @@ -1,101 +0,0 @@ -getTestClient('s3'); - $this->addMockResults($client, [new Result(['Owner' => ['ID' => 'id']])]); - $result = $client->listBuckets(); - $this->assertInstanceOf(ResultInterface::class, $result); - $this->assertInternalType('string', $result['Owner']['ID']); - } - - public function testSendsPromisedSynchronousRequest() - { - /** @var S3Client $client */ - $client = $this->getTestClient('s3'); - $this->addMockResults($client, [new Result(['Owner' => ['ID' => 'id']])]); - $promise = $client->listBucketsAsync(); - $this->assertInstanceOf(PromiseInterface::class, $promise); - $result = $promise->wait(); - $this->assertInternalType('string', $result['Owner']['ID']); - } - - public function testSendsRequestsAsynchronously() - { - /** @var S3Client $client */ - $client = $this->getTestClient('s3'); - $this->addMockResults($client, [ - new Result(['Owner' => ['ID' => 'id']]), - new Result(['Owner' => ['ID' => 'id']]), - new Result(['Owner' => ['ID' => 'id']]), - ]); - - $promise = Promise\all([ - $client->listBucketsAsync(), - $client->listBucketsAsync(), - $client->listBucketsAsync() - ]); - $results = $promise->wait(); - - $this->assertCount(3, $results); - $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); - } - - public function testSendsRequestsConcurrentlyWithPool() - { - /** @var S3Client $client */ - $client = $this->getTestClient('s3'); - $this->addMockResults($client, [ - new Result(['Owner' => ['ID' => 'id']]), - new Result(['Owner' => ['ID' => 'id']]), - new Result(['Owner' => ['ID' => 'id']]), - ]); - - $commands = [ - $client->getCommand('ListBuckets'), - $client->getCommand('ListBuckets'), - $client->getCommand('ListBuckets') - ]; - $results = CommandPool::batch($client, $commands); - - $this->assertCount(3, $results); - $this->assertCount(1, array_unique(\JmesPath\search('[*].Owner.ID', $results))); - } - - public function testSendsRequestsAsynchronouslyFromMultipleServices() - { - /** @var S3Client $s3 */ - $s3 = $this->getTestClient('s3'); - $this->addMockResults($s3, [new Result]); - /** @var DynamoDbClient $db */ - $db = $this->getTestClient('dynamodb'); - $this->addMockResults($db, [new Result]); - /** @var SqsClient $sqs */ - $sqs = $this->getTestClient('sqs'); - $this->addMockResults($sqs, [new Result]); - - $promise = Promise\all([ - $s3->listBucketsAsync(), - $db->listTablesAsync(), - $sqs->listQueuesAsync() - ]); - $results = $promise->wait(); - - $this->assertCount(3, $results); - } -} diff --git a/tests/Integ/ConcurrencyContext.php b/tests/Integ/ConcurrencyContext.php new file mode 100644 index 0000000000..d4cf23a83f --- /dev/null +++ b/tests/Integ/ConcurrencyContext.php @@ -0,0 +1,173 @@ +createS3(); + + self::$bucket = self::getResourcePrefix() + . str_replace(' ', '-', strtolower($scope->getName())); + + $client->createBucket(['Bucket' => self::$bucket]); + $client->waitUntil('BucketExists', ['Bucket' => self::$bucket]); + } + + /** + * @AfterFeature @s3 + * + * @param AfterFeatureScope $scope + */ + public static function tearDownS3Bucket(AfterFeatureScope $scope) + { + $client = self::getSdk()->createS3(); + + $client->deleteMatchingObjects(self::$bucket, '', '//'); + $client->deleteBucket(['Bucket' => self::$bucket]); + + self::$bucket = null; + } + + /** + * @Given I have a :service client + */ + public function iHaveAClient($service) + { + $this->client = self::getSdk()->createClient($service); + } + + /** + * @When I call the :command API + */ + public function iCallTheApi($command) + { + $this->result = $this->client->{$command}(); + } + + /** + * @Then the value at :key should be a :type + */ + public function theValueAtShouldBeA($key, $type) + { + $this->assertInstanceOf(Result::class, $this->result); + $this->assertInternalType($type, $this->result->search($key)); + } + + /** + * @When I call the :command API asynchronously + */ + public function iCallTheApiAsynchronously($command) + { + $this->promise = call_user_func([$this->client, "{$command}Async"]); + } + + /** + * @When I wait on the promise + */ + public function thenWaitOnThePromise() + { + $this->result = $this->promise->wait(); + } + + /** + * @Given a promise composed of the following asynchronous operations: + */ + public function aPromiseComposedOfTheFollowingAsynchronousOperations(TableNode $table) + { + $this->promise = Promise\all(array_map(function (array $row) { + return call_user_func( + [ + self::getSdk()->createClient($row['service']), + "{$row['command']}Async", + ], + json_decode($row['payload'], true) ?: [] + ); + }, iterator_to_array($table))); + } + + /** + * @Given a pool composed of the following commands: + */ + public function aPoolComposedOfTheFollowingCommands(TableNode $table) + { + $this->commands = array_map(function (array $row) { + return self::getSdk() + ->createClient($row['service']) + ->getCommand( + $row['command'], + json_decode($row['payload'], true) ?: [] + ); + }, iterator_to_array($table)); + } + + /** + * @When I send the commands as a batch to :service + */ + public function iSendTheCommandsAsABatchTo($service) + { + $this->result = CommandPool::batch( + self::getSdk()->createClient($service), + $this->commands + ); + } + + /** + * @Then there should be :count results + */ + public function thereShouldBeResults($count) + { + $this->assertCount((int) $count, $this->result); + } + + /** + * @Then there should be :count value at :path + */ + public function thereShouldBeValueAt($count, $path) + { + $this->assertCount((int) $count, array_unique( + JmesPath\search($path, $this->result) + )); + } +} From 460188621fea871bb68560435501f85f39e97f53 Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Mon, 7 Dec 2015 14:10:51 -0800 Subject: [PATCH 7/8] Reconfigure integ make target --- Makefile | 2 +- features/batching/batchSerialization.feature | 2 +- features/batching/writeRequestBatch.feature | 2 +- features/blocking/waiters.feature | 2 +- features/concurrency/requests.feature | 2 +- features/multipart/glacier.feature | 2 +- features/multipart/s3.feature | 2 +- features/streams/streamWrapper.feature | 2 +- tests/Integ/BatchingContext.php | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 28786cd85e..3afa772cbc 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ coverage-show: open build/artifacts/coverage/index.html integ: - vendor/bin/phpunit --debug --testsuite=integ $(TEST) + vendor/bin/behat --format=progress --tags=integ smoke: vendor/bin/behat --format=progress diff --git a/features/batching/batchSerialization.feature b/features/batching/batchSerialization.feature index 8dd46b7d69..b66eedb388 100644 --- a/features/batching/batchSerialization.feature +++ b/features/batching/batchSerialization.feature @@ -1,4 +1,4 @@ -@sqs +@sqs @integ Feature: SQS Batch Serialization Scenario: Deleting Message Batches diff --git a/features/batching/writeRequestBatch.feature b/features/batching/writeRequestBatch.feature index 1408be8bc1..d2e160b59b 100644 --- a/features/batching/writeRequestBatch.feature +++ b/features/batching/writeRequestBatch.feature @@ -1,4 +1,4 @@ -@dynamodb +@dynamodb @integ Feature: DynamoDB Write Request Batch Scenario: Batching Given I have a "dynamodb" client diff --git a/features/blocking/waiters.feature b/features/blocking/waiters.feature index 496429ac24..bd42701d8b 100644 --- a/features/blocking/waiters.feature +++ b/features/blocking/waiters.feature @@ -1,4 +1,4 @@ -@dynamodb +@dynamodb @integ Feature: Waiters Scenario: Synchronous Waiters diff --git a/features/concurrency/requests.feature b/features/concurrency/requests.feature index 0eda99242a..cd38530f77 100644 --- a/features/concurrency/requests.feature +++ b/features/concurrency/requests.feature @@ -1,4 +1,4 @@ -@s3 +@s3 @integ Feature: Concurrent Requests Scenario: Sending a normal, synchronous request diff --git a/features/multipart/glacier.feature b/features/multipart/glacier.feature index bf892ff231..11b8079d51 100644 --- a/features/multipart/glacier.feature +++ b/features/multipart/glacier.feature @@ -1,4 +1,4 @@ -@glacier +@glacier @integ Feature: Glacier Multipart Uploads Scenario Outline: Uploading a stream diff --git a/features/multipart/s3.feature b/features/multipart/s3.feature index 9f59a6cfc2..330bb3d582 100644 --- a/features/multipart/s3.feature +++ b/features/multipart/s3.feature @@ -1,4 +1,4 @@ -@s3 +@s3 @integ Feature: S3 Multipart Uploads Scenario Outline: Uploading a stream diff --git a/features/streams/streamWrapper.feature b/features/streams/streamWrapper.feature index 1d9bd4a0d0..5b4953ea0c 100644 --- a/features/streams/streamWrapper.feature +++ b/features/streams/streamWrapper.feature @@ -1,5 +1,5 @@ #language: en -@s3 +@s3 @integ Feature: S3 Stream Wrapper Background: diff --git a/tests/Integ/BatchingContext.php b/tests/Integ/BatchingContext.php index 1728c21280..3d81f7a32a 100644 --- a/tests/Integ/BatchingContext.php +++ b/tests/Integ/BatchingContext.php @@ -91,7 +91,7 @@ public static function setUpQueue(BeforeFeatureScope $scope) { $sqs = self::getSdk()->createSqs(); self::$resource = self::getResourcePrefix() - . str_replace(' ', '-', strtolower($scope->getName())); + . preg_replace('/\W/', '-', strtolower($scope->getName())); $sqs->createQueue(['QueueName' => self::$resource]); $sqs->waitUntil('QueueExists', ['QueueName' => self::$resource]); From cb52921fb3fc99a6971b11347af52d56e0b2acf4 Mon Sep 17 00:00:00 2001 From: Jonathan Eskew Date: Mon, 7 Dec 2015 14:42:43 -0800 Subject: [PATCH 8/8] Disable xdebug.overload_var_dump in Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e8cb96c6c4..4c859e77a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ matrix: install: - export AWS_ACCESS_KEY_ID=foo - export AWS_SECRET_ACCESS_KEY=bar + - echo "xdebug.overload_var_dump = 0" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - travis_retry composer update $COMPOSER_OPTS --no-interaction --prefer-source script: