diff --git a/aws-runtime/crt-util/common/src/aws/sdk/kotlin/runtime/crt/ReadChannelBodyStream.kt b/aws-runtime/crt-util/common/src/aws/sdk/kotlin/runtime/crt/ReadChannelBodyStream.kt index 080841a0f75..fb006aff295 100644 --- a/aws-runtime/crt-util/common/src/aws/sdk/kotlin/runtime/crt/ReadChannelBodyStream.kt +++ b/aws-runtime/crt-util/common/src/aws/sdk/kotlin/runtime/crt/ReadChannelBodyStream.kt @@ -28,7 +28,7 @@ internal expect fun transferRequestBody(outgoing: SdkBuffer, dest: MutableBuffer public class ReadChannelBodyStream( // the request body channel private val bodyChan: SdkByteReadChannel, - callContext: CoroutineContext + private val callContext: CoroutineContext ) : HttpRequestBodyStream, CoroutineScope { private val producerJob = Job(callContext.job) @@ -60,6 +60,8 @@ public class ReadChannelBodyStream( if (bufferChan.isClosedForReceive) { return true } + // ensure the request context hasn't been cancelled + callContext.ensureActive() outgoing = bufferChan.tryReceive().getOrNull() ?: return false } diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt new file mode 100644 index 00000000000..84d72ee8aac --- /dev/null +++ b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt @@ -0,0 +1,72 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package aws.sdk.kotlin.runtime.testing + +import java.io.IOException +import java.io.InputStream +import kotlin.random.Random + +/** + * Test utility InputStream implementation that generates random ASCII data when + * read, up to the size specified when constructed. + */ +public class RandomInputStream constructor( + /** The requested amount of data contained in this random stream. */ + private val lengthInBytes: Long, + + /** Flag controlling whether binary or character data is used. */ + private val binaryData: Boolean = false +) : InputStream() { + + /** The number of bytes of data remaining in this random stream. */ + protected var remainingBytes: Long = lengthInBytes + + public val bytesRead: Long + get() = lengthInBytes - remainingBytes + + @Throws(IOException::class) + override fun read(b: ByteArray, off: Int, len: Int): Int { + // Signal that we're out of data if we've hit our limit + if (remainingBytes <= 0) { + return -1 + } + var bytesToRead = len + if (bytesToRead > remainingBytes) { + bytesToRead = remainingBytes.toInt() + } + remainingBytes -= bytesToRead.toLong() + if (binaryData) { + val endExclusive = off + bytesToRead + Random.nextBytes(b, off, endExclusive) + } else { + for (i in 0 until bytesToRead) { + b[off + i] = Random.nextInt(MIN_CHAR_CODE, MAX_CHAR_CODE + 1).toByte() + } + } + return bytesToRead + } + + @Throws(IOException::class) + override fun read(): Int { + // Signal that we're out of data if we've hit our limit + if (remainingBytes <= 0) { + return -1 + } + remainingBytes-- + return if (binaryData) { + Random.nextInt() + } else { + Random.nextInt(MIN_CHAR_CODE, MAX_CHAR_CODE + 1) + } + } + + public companion object { + /** The minimum ASCII code contained in the data in this stream. */ + private const val MIN_CHAR_CODE = 32 + + /** The maximum ASCII code contained in the data in this stream. */ + private const val MAX_CHAR_CODE = 125 + } +} diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt new file mode 100644 index 00000000000..f554c7245cc --- /dev/null +++ b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt @@ -0,0 +1,75 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package aws.sdk.kotlin.runtime.testing + +import java.io.BufferedOutputStream +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.util.* + +/** + * Extension of File that creates a temporary file with a specified name in + * Java's temporary directory, as declared in the JRE's system properties. The + * file is immediately filled with a specified amount of random ASCII data. + * + * @see RandomInputStream + */ +public class RandomTempFile : File { + /** Flag controlling whether binary or character data is used. */ + private val binaryData: Boolean + + /** + * Creates, and fills, a temp file with a randomly generated name and specified size of random ASCII data. + * + * @param sizeInBytes The amount of random ASCII data, in bytes, for the new temp + * file. + * @throws IOException If any problems were encountered creating the new temp file. + */ + public constructor(sizeInBytes: Long) : this(UUID.randomUUID().toString(), sizeInBytes, false) + + /** + * Creates, and fills, a temp file with the specified name and specified + * size of random data. + * + * @param filename The name for the new temporary file, within the Java temp + * directory as declared in the JRE's system properties. + * @param sizeInBytes The amount of random ASCII data, in bytes, for the new temp + * file. + * @param binaryData Whether to fill the file with binary or character data. + * + * @throws IOException + * If any problems were encountered creating the new temp file. + */ + public constructor(filename: String, sizeInBytes: Long, binaryData: Boolean = false) : super( + TEMP_DIR + separator + System.currentTimeMillis().toString() + "-" + filename + ) { + this.binaryData = binaryData + createFile(sizeInBytes) + } + + @Throws(IOException::class) + public fun createFile(sizeInBytes: Long) { + deleteOnExit() + FileOutputStream(this).use { outputStream -> + BufferedOutputStream(outputStream).use { bufferedOutputStream -> + RandomInputStream(sizeInBytes, binaryData).use { inputStream -> + inputStream.copyTo(bufferedOutputStream) + } + } + } + } + + override fun delete(): Boolean { + if (!super.delete()) { + throw RuntimeException("Could not delete: $absolutePath") + } + return true + } + + public companion object { + private val TEMP_DIR: String = System.getProperty("java.io.tmpdir") + } +} diff --git a/services/build.gradle.kts b/services/build.gradle.kts index 3e743220c59..ebe9b0e6875 100644 --- a/services/build.gradle.kts +++ b/services/build.gradle.kts @@ -81,6 +81,33 @@ subprojects { } apply(from = rootProject.file("gradle/publish.gradle")) + + if (project.file("e2eTest").exists()) { + + kotlin.target.compilations { + val main by getting + val e2eTest by creating { + defaultSourceSet { + kotlin.srcDir("e2eTest") + dependencies { + implementation(main.compileDependencyFiles + main.runtimeDependencyFiles + main.output.classesDirs) + + implementation(kotlin("test")) + implementation(kotlin("test-junit5")) + implementation(project(":aws-runtime:testing")) + } + } + + tasks.register("e2eTest") { + description = "Run e2e service tests" + group = "verification" + classpath = compileDependencyFiles + runtimeDependencyFiles + testClassesDirs = output.classesDirs + useJUnitPlatform() + } + } + } + } } diff --git a/services/s3/e2eTest/S3IntegrationTest.kt b/services/s3/e2eTest/S3IntegrationTest.kt new file mode 100644 index 00000000000..e6c88a4e236 --- /dev/null +++ b/services/s3/e2eTest/S3IntegrationTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.runtime.testing.RandomTempFile +import aws.sdk.kotlin.runtime.testing.runSuspendTest +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.* +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.decodeToString +import aws.smithy.kotlin.runtime.content.fromFile +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +/** + * Tests for bucket operations + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class S3BucketOpsIntegrationTest { + companion object { + const val DEFAULT_REGION = "us-east-2" + } + + val client = S3Client { + region = DEFAULT_REGION + } + + lateinit var testBucket: String + + @BeforeAll + private fun createResources(): Unit = runBlocking { + testBucket = S3TestUtils.getTestBucket(client) + } + + @AfterAll + private fun cleanup() = runBlocking { + S3TestUtils.deleteBucketAndAllContents(client, testBucket) + } + + @Test + fun testPutObjectFromMemory() = runSuspendTest { + val contents = """ + A lep is a ball. + A tay is a hammer. + A korf is a tiger. + A flix is a comb. + A wogsin is a gift. + """.trimIndent() + + val keyName = "put-obj-from-memory.txt" + + client.putObject { + bucket = testBucket + key = keyName + body = ByteStream.fromString(contents) + } + + val req = GetObjectRequest { + bucket = testBucket + key = keyName + } + val roundTrippedContents = client.getObject(req) { it.body?.decodeToString() } + + assertEquals(contents, roundTrippedContents) + } + + @OptIn(ExperimentalTime::class) + @Test + fun testPutObjectFromFile() = runSuspendTest { + val tempFile = RandomTempFile(1024) + val keyName = "put-obj-from-file.txt" + + // This test fails sporadically (by never completing) + // see https://github.com/awslabs/aws-sdk-kotlin/issues/282 + withTimeout(Duration.seconds(5)) { + client.putObject { + bucket = testBucket + key = keyName + body = ByteStream.fromFile(tempFile) + } + } + + val req = GetObjectRequest { + bucket = testBucket + key = keyName + } + val roundTrippedContents = client.getObject(req) { it.body?.decodeToString() } + + val contents = tempFile.readText() + assertEquals(contents, roundTrippedContents) + } +} diff --git a/services/s3/e2eTest/S3TestUtils.kt b/services/s3/e2eTest/S3TestUtils.kt new file mode 100644 index 00000000000..d9ce2af700d --- /dev/null +++ b/services/s3/e2eTest/S3TestUtils.kt @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.* +import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeout +import java.util.* +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.seconds + +object S3TestUtils { + + const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + + suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) + + @OptIn(ExperimentalTime::class) + suspend fun getBucketWithPrefix(client: S3Client, prefix: String): String = withTimeout(Duration.seconds(60)) { + var testBucket = client.listBuckets {} + .buckets + ?.mapNotNull { it.name } + ?.firstOrNull { it.startsWith(prefix) } + + if (testBucket == null) { + testBucket = prefix + UUID.randomUUID() + client.createBucket { + bucket = testBucket + createBucketConfiguration { + locationConstraint = BucketLocationConstraint.fromValue(client.config.region!!) + } + } + + do { + val bucketExists = try { + client.headBucket { bucket = testBucket } + true + } catch (ex: NotFound) { + delay(300) + false + } + } while (!bucketExists) + } + + client.putBucketLifecycleConfiguration { + bucket = testBucket + lifecycleConfiguration { + rules = listOf( + LifecycleRule { + expiration { days = 1 } + filter = LifecycleRuleFilter.Prefix("") + status = ExpirationStatus.Enabled + id = "delete-old" + } + ) + } + } + + testBucket + } + + suspend fun deleteBucketAndAllContents(client: S3Client, bucketName: String) { + try { + println("Deleting S3 bucket: $bucketName") + + var resp = client.listObjectsV2 { bucket = bucketName } + + do { + val objects = resp.contents + val truncated = resp.isTruncated + + objects?.forEach { + client.deleteObject { + bucket = bucketName + key = it.key + } + } + + if (truncated) { + resp = client.listObjectsV2 { + bucket = bucketName + continuationToken = resp.continuationToken + } + } + } while (truncated) + + client.deleteBucket { bucket = bucketName } + } catch (ex: Exception) { + println("Failed to delete bucket: $bucketName") + throw ex + } + } +}