feat(storage): support custom file options for upload (#1903)

Author: sweatybridge

cmd/storage.go

@@ -4,6 +4,7 @@ import (
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/supabase/cli/internal/storage"
+	"github.com/supabase/cli/internal/storage/client"
 	"github.com/supabase/cli/internal/storage/cp"
 	"github.com/supabase/cli/internal/storage/ls"
 	"github.com/supabase/cli/internal/storage/mv"
@@ -33,6 +34,7 @@ var (
 		},
 	}
 
+	options client.FileOptions
 	maxJobs uint
 
 	cpCmd = &cobra.Command{
@@ -44,7 +46,11 @@ cp -r ss:///bucket/docs .
 		Short: "Copy objects from src to dst path",
 		Args:  cobra.ExactArgs(2),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return cp.Run(cmd.Context(), args[0], args[1], recursive, maxJobs, afero.NewOsFs())
+			opts := func(fo *client.FileOptions) {
+				fo.CacheControl = options.CacheControl
+				fo.ContentType = options.ContentType
+			}
+			return cp.Run(cmd.Context(), args[0], args[1], recursive, maxJobs, afero.NewOsFs(), opts)
 		},
 	}
 
@@ -76,6 +82,9 @@ func init() {
 	storageCmd.AddCommand(lsCmd)
 	cpFlags := cpCmd.Flags()
 	cpFlags.BoolVarP(&recursive, "recursive", "r", false, "Recursively copy a directory.")
+	cpFlags.StringVar(&options.CacheControl, "cache-control", "max-age=3600", "Custom Cache-Control header for HTTP upload.")
+	cpFlags.StringVar(&options.ContentType, "content-type", "", "Custom Content-Type header for HTTP upload.")
+	cpFlags.Lookup("content-type").DefValue = "auto-detect"
 	cpFlags.UintVarP(&maxJobs, "jobs", "j", 1, "Maximum number of parallel jobs.")
 	storageCmd.AddCommand(cpCmd)
 	rmCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively remove a directory.")

internal/storage/client/objects.go

@@ -63,22 +63,38 @@ func ListStorageObjects(ctx context.Context, projectRef, bucket, prefix string,
 	return *data, nil
 }
 
-func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs) error {
+type FileOptions struct {
+	CacheControl string
+	ContentType  string
+}
+
+func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath string, fsys afero.Fs, opts ...func(*FileOptions)) error {
 	f, err := fsys.Open(localPath)
 	if err != nil {
 		return errors.Errorf("failed to open file: %w", err)
 	}
 	defer f.Close()
-	// Decode mimetype
-	header := io.LimitReader(f, 512)
-	buf, err := io.ReadAll(header)
-	if err != nil {
-		return errors.Errorf("failed to read file: %w", err)
+	// Customise file options
+	fo := &FileOptions{}
+	for _, apply := range opts {
+		apply(fo)
 	}
-	mimetype := http.DetectContentType(buf)
-	_, err = f.Seek(0, io.SeekStart)
-	if err != nil {
-		return errors.Errorf("failed to seek file: %w", err)
+	// Use default value of storage-js: https://github.com/supabase/storage-js/blob/main/src/packages/StorageFileApi.ts#L22
+	if len(fo.CacheControl) == 0 {
+		fo.CacheControl = "max-age=3600"
+	}
+	// Decode mimetype
+	if len(fo.ContentType) == 0 {
+		header := io.LimitReader(f, 512)
+		buf, err := io.ReadAll(header)
+		if err != nil {
+			return errors.Errorf("failed to read file: %w", err)
+		}
+		fo.ContentType = http.DetectContentType(buf)
+		_, err = f.Seek(0, io.SeekStart)
+		if err != nil {
+			return errors.Errorf("failed to seek file: %w", err)
+		}
 	}
 	// Prepare request
 	apiKey, err := tenant.GetApiKeys(ctx, projectRef)
@@ -92,9 +108,8 @@ func UploadStorageObject(ctx context.Context, projectRef, remotePath, localPath
 		return errors.Errorf("failed to initialise http request: %w", err)
 	}
 	req.Header.Add("Authorization", "Bearer "+apiKey.ServiceRole)
-	req.Header.Add("Content-Type", mimetype)
-	// Use default value of storage-js: https://github.com/supabase/storage-js/blob/main/src/packages/StorageFileApi.ts#L22
-	req.Header.Add("Cache-Control", "max-age=3600")
+	req.Header.Add("Content-Type", fo.ContentType)
+	req.Header.Add("Cache-Control", fo.CacheControl)
 	// Sends request
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {

internal/storage/cp/cp.go

@@ -21,7 +21,7 @@ import (
 
 var errUnsupportedOperation = errors.New("Unsupported operation")
 
-func Run(ctx context.Context, src, dst string, recursive bool, maxJobs uint, fsys afero.Fs) error {
+func Run(ctx context.Context, src, dst string, recursive bool, maxJobs uint, fsys afero.Fs, opts ...func(*client.FileOptions)) error {
 	srcParsed, err := url.Parse(src)
 	if err != nil {
 		return errors.Errorf("failed to parse src url: %w", err)
@@ -41,9 +41,9 @@ func Run(ctx context.Context, src, dst string, recursive bool, maxJobs uint, fsy
 		return client.DownloadStorageObject(ctx, projectRef, srcParsed.Path, dst, fsys)
 	} else if srcParsed.Scheme == "" && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME {
 		if recursive {
-			return UploadStorageObjectAll(ctx, projectRef, dstParsed.Path, src, maxJobs, fsys)
+			return UploadStorageObjectAll(ctx, projectRef, dstParsed.Path, src, maxJobs, fsys, opts...)
 		}
-		return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys)
+		return client.UploadStorageObject(ctx, projectRef, dstParsed.Path, src, fsys, opts...)
 	} else if strings.ToLower(srcParsed.Scheme) == storage.STORAGE_SCHEME && strings.ToLower(dstParsed.Scheme) == storage.STORAGE_SCHEME {
 		return errors.New("Copying between buckets is not supported")
 	}
@@ -81,7 +81,7 @@ func DownloadStorageObjectAll(ctx context.Context, projectRef, remotePath, local
 	return errors.Join(err, jq.Collect())
 }
 
-func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, maxJobs uint, fsys afero.Fs) error {
+func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPath string, maxJobs uint, fsys afero.Fs, opts ...func(*client.FileOptions)) error {
 	noSlash := strings.TrimSuffix(remotePath, "/")
 	// Check if directory exists on remote
 	dirExists := false
@@ -125,14 +125,14 @@ func UploadStorageObjectAll(ctx context.Context, projectRef, remotePath, localPa
 		}
 		fmt.Fprintln(os.Stderr, "Uploading:", filePath, "=>", dstPath)
 		job := func() error {
-			err := client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys)
+			err := client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys, opts...)
 			if err != nil && strings.Contains(err.Error(), `"error":"Bucket not found"`) {
 				// Retry after creating bucket
 				if bucket, prefix := storage.SplitBucketPrefix(dstPath); len(prefix) > 0 {
 					if _, err := client.CreateStorageBucket(ctx, projectRef, bucket); err != nil {
 						return err
 					}
-					err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys)
+					err = client.UploadStorageObject(ctx, projectRef, dstPath, filePath, fsys, opts...)
 				}
 			}
 			return err