cmd/sunlight,internal/ctlog: add S3 key prefix option (#8)

FiloSottile · Feb 14, 2024 · 4a6fdd7 · 4a6fdd7
1 parent 02638a2
commit 4a6fdd7
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/cmd/sunlight/main.go b/cmd/sunlight/main.go
@@ -141,6 +141,12 @@ type LogConfig struct {
 	// S3Endpoint is the base URL the AWS SDK will use to connect to S3. Optional.
 	S3Endpoint string
 
+	// S3KeyPrefix is a prefix on all keys written to S3. Optional.
+	//
+	// S3 doesn't have directories, but using a prefix ending in a "/" is
+	// going to be treated like a directory in many tools using S3.
+	S3KeyPrefix string
+
 	// NotAfterStart is the start of the validity range for certificates
 	// accepted by this log instance, as and RFC 3339 date.
 	NotAfterStart string
@@ -229,7 +235,7 @@ func main() {
 			slog.String("log", lc.ShortName),
 		}))
 
-		b, err := ctlog.NewS3Backend(ctx, lc.S3Region, lc.S3Bucket, lc.S3Endpoint, logger)
+		b, err := ctlog.NewS3Backend(ctx, lc.S3Region, lc.S3Bucket, lc.S3Endpoint, lc.S3KeyPrefix, logger)
 		if err != nil {
 			logger.Error("failed to create backend", "err", err)
 			os.Exit(1)

diff --git a/internal/ctlog/s3.go b/internal/ctlog/s3.go
@@ -22,6 +22,7 @@ import (
 type S3Backend struct {
 	client        *s3.Client
 	bucket        string
+	keyPrefix     string
 	metrics       []prometheus.Collector
 	uploadSize    prometheus.Summary
 	compressRatio prometheus.Summary
@@ -30,7 +31,7 @@ type S3Backend struct {
 	log           *slog.Logger
 }
 
-func NewS3Backend(ctx context.Context, region, bucket, endpoint string, l *slog.Logger) (*S3Backend, error) {
+func NewS3Backend(ctx context.Context, region, bucket, endpoint, keyPrefix string, l *slog.Logger) (*S3Backend, error) {
 	counter := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "s3_requests_total",
@@ -97,7 +98,8 @@ func NewS3Backend(ctx context.Context, region, bucket, endpoint string, l *slog.
 				o.BaseEndpoint = aws.String(endpoint)
 			}
 		}),
-		bucket: bucket,
+		bucket:    bucket,
+		keyPrefix: keyPrefix,
 		metrics: []prometheus.Collector{counter, duration,
 			uploadSize, compressRatio, hedgeRequests, hedgeWins},
 		uploadSize:    uploadSize,
@@ -141,7 +143,7 @@ func (s *S3Backend) Upload(ctx context.Context, key string, data []byte, opts *U
 			s.hedgeRequests.Inc()
 			_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
 				Bucket:          aws.String(s.bucket),
-				Key:             aws.String(key),
+				Key:             aws.String(s.keyPrefix + key),
 				Body:            bytes.NewReader(data),
 				ContentLength:   aws.Int64(int64(len(data))),
 				ContentEncoding: contentEncoding,
@@ -154,7 +156,7 @@ func (s *S3Backend) Upload(ctx context.Context, key string, data []byte, opts *U
 	}()
 	_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
 		Bucket:          aws.String(s.bucket),
-		Key:             aws.String(key),
+		Key:             aws.String(s.keyPrefix + key),
 		Body:            bytes.NewReader(data),
 		ContentLength:   aws.Int64(int64(len(data))),
 		ContentEncoding: contentEncoding,
@@ -179,7 +181,7 @@ func (s *S3Backend) Upload(ctx context.Context, key string, data []byte, opts *U
 func (s *S3Backend) Fetch(ctx context.Context, key string) ([]byte, error) {
 	out, err := s.client.GetObject(ctx, &s3.GetObjectInput{
 		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
+		Key:    aws.String(s.keyPrefix + key),
 	})
 	if err != nil {
 		s.log.DebugContext(ctx, "S3 GET", "key", key, "err", err)