First pass for Amazon S3 support

CONFIG.md

@@ -147,6 +147,19 @@ database:
 ##### ProjectID
 ```project_id``` *Optional* The Google Cloud project ID of the project owning the above credentials and GCS bucket.
 
+#### AWS
+```aws_s3:``` *Optional* The AWS section configures AWS S3 storage.
+
+##### Region
+```region``` *Required* The AWS region
+
+##### Bucket
+```bucket``` *Required* The AWS S3 bucket (will be created automatically)
+
+##### Credentials Profile
+```credentials_profile```  *Optional* If a profile other than default is chosen, use that one.
+
+By default, the S3 blobstore will rely on environment variables, shared credentials, or IAM roles. See https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials for more information.
 
 Example Storage section: (disk)
 ```
@@ -169,6 +182,18 @@ storage:
     credentials_file: "enterprise/config/my-cool-project-7a9d15f66e69.json"
 ```
 
+Example Storage section: (aws_s3)
+
+```
+storage:
+  aws_s3:
+     # required
+     region: "us-west-2"
+     bucket: "buddybuild-bucket"
+     # optional
+     credentials_profile: "other-profile"
+``
+`
 ## Integrations
 ```integrations:``` *Optional* A section configuring optional external services BuildBuddy can integrate with, like Slack.
 

deps.bzl

@@ -355,6 +355,20 @@ def install_buildbuddy_dependencies():
 	version = "v0.50.0",
     )
 
+    go_repository(
+        name = "com_github_aws_aws_sdk_go",
+        importpath = "github.com/aws/aws-sdk-go",
+        version = "v1.30.14",
+        sum = "h1:vZfX2b/fknc9wKcytbLWykM7in5k6dbQ8iHTJDUP1Ng="
+    )
+
+    go_repository(
+        name="com_github_jmespath_go_jmespath",
+        importpath = "github.com/jmespath/go-jmespath",
+        version = "v0.3.0",
+        sum = "h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc="
+    )
+
     go_repository(
 	name = "com_google_cloud_go_bigquery",
 	importpath = "cloud.google.com/go/bigquery",

server/backends/BUILD

@@ -11,6 +11,13 @@ go_library(
         "//server:config",
         "//server:interfaces",
         "//server/util:disk",
+        "@com_github_aws_aws_sdk_go//:go_default_library",
+        "@com_github_aws_aws_sdk_go//aws:go_default_library",
+        "@com_github_aws_aws_sdk_go//aws/awserr:go_default_library",
+        "@com_github_aws_aws_sdk_go//aws/credentials:go_default_library",
+        "@com_github_aws_aws_sdk_go//aws/session:go_default_library",
+        "@com_github_aws_aws_sdk_go//service/s3:go_default_library",
+        "@com_github_aws_aws_sdk_go//service/s3/s3manager:go_default_library",
         "@com_google_cloud_go_storage//:go_default_library",
         "@org_golang_google_api//option:go_default_library",
     ],

server/backends/blobstore.go

@@ -12,6 +12,12 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/buildbuddy-io/buildbuddy/server/config"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
 	"github.com/buildbuddy-io/buildbuddy/server/util/disk"
@@ -30,6 +36,10 @@ func GetConfiguredBlobstore(c *config.Configurator) (interfaces.Blobstore, error
 		}
 		return NewGCSBlobStore(gcsConfig.Bucket, gcsConfig.ProjectID, opts...)
 	}
+
+	if awsConfig := c.GetStorageAWSS3Config(); awsConfig != nil && awsConfig.Bucket != "" {
+		return NewAwsS3BlobStore(awsConfig)
+	}
 	return nil, fmt.Errorf("No storage backend configured -- please specify at least one in the config")
 }
 
@@ -207,3 +217,124 @@ func (g *GCSBlobStore) BlobExists(ctx context.Context, blobName string) (bool, e
 		return false, err
 	}
 }
+
+// AWS stuff
+type AwsS3BlobStore struct {
+	s3         *s3.S3
+	bucket     *string
+	downloader *s3manager.Downloader
+	uploader   *s3manager.Uploader
+}
+
+func NewAwsS3BlobStore(awsConfig *config.AwsS3Config) (*AwsS3BlobStore, error) {
+	ctx := context.Background()
+
+	var creds *credentials.Credentials
+
+	// See https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials
+	if awsConfig.CredentialsProfile != "" {
+		creds = credentials.NewSharedCredentials("", awsConfig.CredentialsProfile)
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(awsConfig.Region),
+		Credentials: creds,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Create S3 service client
+	svc := s3.New(sess)
+
+	awsBlobStore := &AwsS3BlobStore{
+		s3:         svc,
+		bucket:     aws.String(awsConfig.Bucket),
+		downloader: s3manager.NewDownloader(sess),
+		uploader:   s3manager.NewUploader(sess),
+	}
+
+	if err := awsBlobStore.createBucketIfNotExists(ctx, awsConfig.Bucket); err != nil {
+		return nil, err
+	}
+
+	return awsBlobStore, nil
+}
+
+func (a *AwsS3BlobStore) createBucketIfNotExists(ctx context.Context, bucketName string) error {
+	// HeadBucket call will return 404 or 403
+	if _, err := a.s3.HeadBucketWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+		aerr := err.(awserr.Error)
+		// AWS returns codes as strings
+		// https://github.com/aws/aws-sdk-go/blob/master/service/s3/s3manager/bucket_region_test.go#L70
+		if aerr.Code() != "NotFound" {
+			return err
+		}
+
+		log.Printf("Creating storage bucket: %s", bucketName)
+		if _, err := a.s3.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+			return err
+		}
+		return a.s3.WaitUntilBucketExistsWithContext(ctx, &s3.HeadBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+	}
+	return nil
+}
+
+func (a *AwsS3BlobStore) ReadBlob(ctx context.Context, blobName string) ([]byte, error) {
+	buff := &aws.WriteAtBuffer{}
+
+	_, err := a.downloader.DownloadWithContext(ctx, buff,
+		&s3.GetObjectInput{
+			Bucket: a.bucket,
+			Key:    aws.String(blobName),
+		})
+
+	return decompress(buff.Bytes(), err)
+}
+
+func (a *AwsS3BlobStore) WriteBlob(ctx context.Context, blobName string, data []byte) (int, error) {
+	compressedData, err := compress(data)
+	if err != nil {
+		return 0, err
+	}
+	uploadParams := &s3manager.UploadInput{
+		Bucket: a.bucket,
+		Key:    aws.String(blobName),
+		Body:   bytes.NewReader(compressedData)}
+	if _, err := a.uploader.UploadWithContext(ctx, uploadParams); err != nil {
+		return -1, err
+	}
+	return len(compressedData), nil
+}
+
+func (a *AwsS3BlobStore) DeleteBlob(ctx context.Context, blobName string) error {
+	deleteParams := &s3.DeleteObjectInput{
+		Bucket: a.bucket,
+		Key:    aws.String(blobName)}
+
+	if _, err := a.s3.DeleteObjectWithContext(ctx, deleteParams); err != nil {
+		return err
+	}
+
+	return a.s3.WaitUntilObjectNotExistsWithContext(ctx, &s3.HeadObjectInput{
+		Bucket: a.bucket,
+		Key:    aws.String(blobName),
+	})
+}
+
+func (a *AwsS3BlobStore) BlobExists(ctx context.Context, blobName string) (bool, error) {
+
+	params := &s3.HeadObjectInput{
+		Bucket: a.bucket,
+		Key:    aws.String(blobName),
+	}
+
+	if _, err := a.s3.HeadObjectWithContext(ctx, params); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

server/config.go

@@ -40,10 +40,11 @@ type databaseConfig struct {
 }
 
 type storageConfig struct {
-	Disk               DiskConfig `yaml:"disk"`
-	GCS                GCSConfig  `yaml:"gcs"`
-	TTLSeconds         int        `yaml:"ttl_seconds"`
-	ChunkFileSizeBytes int        `yaml:"chunk_file_size_bytes"`
+	Disk               DiskConfig  `yaml:"disk"`
+	GCS                GCSConfig   `yaml:"gcs"`
+	AwsS3              AwsS3Config `yaml:"aws_s3"`
+	TTLSeconds         int         `yaml:"ttl_seconds"`
+	ChunkFileSizeBytes int         `yaml:"chunk_file_size_bytes"`
 }
 
 type DiskConfig struct {
@@ -56,6 +57,12 @@ type GCSConfig struct {
 	ProjectID       string `yaml:"project_id"`
 }
 
+type AwsS3Config struct {
+	Region             string `yaml:"region"`
+	Bucket             string `yaml:"bucket"`
+	CredentialsProfile string `yaml:"credentials_profile"`
+}
+
 type integrationsConfig struct {
 	Slack SlackConfig `yaml:"slack"`
 }
@@ -183,6 +190,11 @@ func (c *Configurator) GetStorageGCSConfig() *GCSConfig {
 	return &c.gc.Storage.GCS
 }
 
+func (c *Configurator) GetStorageAWSS3Config() *AwsS3Config {
+	c.rereadIfStale()
+	return &c.gc.Storage.AwsS3
+}
+
 func (c *Configurator) GetDBDataSource() string {
 	c.rereadIfStale()
 	return c.gc.Database.DataSource