Add s3 file ingestion sample to examples source tree for M1 release.

examples/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 
 allprojects {
     group = "aws.sdk.kotlin.example"
-    version = "1.0-SNAPSHOT"
+    version = "2.0-SNAPSHOT"
 
     repositories {
         maven {

examples/gradle.properties

@@ -1,4 +1,4 @@
 
 # AWS SDK
-awsSdkKotlinVersion=0.1.0
+awsSdkKotlinVersion=0.2.0
 

examples/s3-media-ingestion/build.gradle.kts

@@ -0,0 +1,11 @@
+plugins {
+    kotlin("jvm")
+}
+
+val awsSdkKotlinVersion: String by project
+
+dependencies {
+    implementation(kotlin("stdlib"))
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.3")
+    implementation("aws.sdk.kotlin:s3:$awsSdkKotlinVersion")
+}

examples/s3-media-ingestion/src/main/kotlin/Main.kt

@@ -0,0 +1,169 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.*
+import kotlinx.coroutines.flow.*
+import kotlinx.coroutines.runBlocking
+import software.aws.clientrt.content.ByteStream
+import software.aws.clientrt.content.fromFile
+import software.aws.clientrt.content.writeToFile
+import java.io.File
+import java.nio.file.Files
+
+/**
+ * This program reads media files from a specified directory and uploads media files to S3.
+ * After uploading it will then download uploaded files back into a local directory.
+ *
+ * Any file with the extension `.avi` will be processed.  To test create a text file and
+ * name it such that it matches the [filenameMetadataRegex] regex, ex:
+ * `title_2000.avi`.
+ *
+ * When running the sample adjust the following path constants as needed for your local environment.
+ */
+const val bucketName = "s3-media-ingestion-example"
+const val ingestionDirPath = "/tmp/media-in"
+const val completedDirPath = "/tmp/media-processed"
+const val failedDirPath = "/tmp/media-failed"
+const val downloadDirPath = "/tmp/media-down"
+
+// media metadata is extracted from filename: <title>_<year>.avi
+val filenameMetadataRegex = "([\\w\\s]+)_([\\d]+).avi".toRegex()
+
+fun main(): Unit = runBlocking {
+    val client = S3Client { region = "us-east-2" }
+
+    try {
+        // Setup
+        client.ensureBucketExists(bucketName)
+        listOf(completedDirPath, failedDirPath, downloadDirPath).forEach { validateDirectory(it) }
+        val ingestionDir = validateDirectory(ingestionDirPath)
+
+        // Upload files
+        val uploadResults = ingestionDir
+            .walk()
+            .asFlow()
+            .mapNotNull(::mediaMetadataExtractor)
+            .map { mediaMetadata -> client.uploadToS3(mediaMetadata) }
+            .toList()
+
+        moveFiles(uploadResults)
+
+        // Print results of operation
+        val (successes, failures) = uploadResults.partition { it is Success }
+        when (failures.isEmpty()) {
+            true -> println("Media uploaded successfully: $successes")
+            false -> println("Successfully uploaded: $successes \nFailed to upload: $failures")
+        }
+
+        // Download files to verify
+        client.listObjects(ListObjectsRequest { bucket = bucketName }).contents?.forEach { obj ->
+            client.getObject(GetObjectRequest { key = obj.key; bucket = bucketName }) { response ->
+                val outputFile = File(downloadDirPath, obj.key!!)
+                response.body?.writeToFile(outputFile).also { size ->
+                    println("Downloaded $outputFile ($size bytes) from S3")
+                }
+            }
+        }
+    } finally {
+        client.close()
+    }
+}
+
+/** Check for valid S3 configuration based on account */
+suspend fun S3Client.ensureBucketExists(bucketName: String) {
+    if (!bucketExists(bucketName)) {
+        createBucket(
+            CreateBucketRequest {
+                bucket = bucketName
+                createBucketConfiguration {
+                    locationConstraint = BucketLocationConstraint.UsEast2
+                }
+            }
+        )
+    }
+}
+
+/** Upload to S3 if file not already uploaded */
+suspend fun S3Client.uploadToS3(mediaMetadata: MediaMetadata): UploadResult {
+    if (keyExists(bucketName, mediaMetadata.s3KeyName))
+        return FileExistsError("${mediaMetadata.s3KeyName} already uploaded.", mediaMetadata)
+
+    return try {
+        putObject(
+            PutObjectRequest {
+                bucket = bucketName
+                key = mediaMetadata.s3KeyName
+                body = ByteStream.fromFile(mediaMetadata.file)
+                metadata = mediaMetadata.toMap()
+            }
+        )
+        Success("$bucketName/${mediaMetadata.s3KeyName}", mediaMetadata)
+    } catch (e: Exception) { // Checking Service Exception coming in future release
+        UploadError(e, mediaMetadata)
+    }
+}
+
+/** Determine if a object exists in a bucket */
+suspend fun S3Client.keyExists(s3bucket: String, s3key: String) =
+    try {
+        headObject(
+            HeadObjectRequest {
+                bucket = s3bucket
+                key = s3key
+            }
+        )
+        true
+    } catch (e: Exception) { // Checking Service Exception coming in future release
+        false
+    }
+
+/** Determine if a object exists in a bucket */
+suspend fun S3Client.bucketExists(s3bucket: String) =
+    try {
+        headBucket(HeadBucketRequest { bucket = s3bucket })
+        true
+    } catch (e: Exception) { // Checking Service Exception coming in future release
+        false
+    }
+
+/** Move files to directories based on upload results */
+fun moveFiles(uploadResults: List<UploadResult>) =
+    uploadResults
+        .map { uploadResult -> uploadResult.mediaMetadata.file.toPath() to (uploadResult is Success) }
+        .forEach { (file, uploadSuccess) ->
+            val targetFilePath = if (uploadSuccess) completedDirPath else failedDirPath
+            val targetPath = File(targetFilePath)
+            Files.move(file, File(targetPath, file.fileName.toString()).toPath())
+        }
+
+// Classes for S3 upload results
+sealed class UploadResult { abstract val mediaMetadata: MediaMetadata }
+data class Success(val location: String, override val mediaMetadata: MediaMetadata) : UploadResult()
+data class UploadError(val error: Throwable, override val mediaMetadata: MediaMetadata) : UploadResult()
+data class FileExistsError(val reason: String, override val mediaMetadata: MediaMetadata) : UploadResult()
+
+// Classes, properties, and functions for media metadata
+data class MediaMetadata(val title: String, val year: Int, val file: File)
+val MediaMetadata.s3KeyName get() = "$title-$year"
+fun MediaMetadata.toMap() = mapOf("title" to title, "year" to year.toString())
+fun mediaMetadataExtractor(file: File): MediaMetadata? {
+    if (!file.isFile || file.length() == 0L) return null
+
+    val matchResult = filenameMetadataRegex.find(file.name) ?: return null
+
+    val (title, year) = matchResult.destructured
+    return MediaMetadata(title, year.toInt(), file)
+}
+
+/** Validate file path and optionally create directory */
+fun validateDirectory(dirPath: String): File {
+    val dir = File(dirPath)
+
+    require(dir.isDirectory || !dir.exists()) { "Unable to use $dir" }
+
+    if (!dir.exists()) require(dir.mkdirs()) { "Unable to create $dir" }
+
+    return dir
+}

examples/settings.gradle.kts

@@ -1,3 +1,4 @@
 rootProject.name = "aws-sdk-kotlin-examples"
 
 include(":dynamodb-movies")
+include(":s3-media-ingestion")