From d0e65ab11db7a9ea250ef393f3cacaac784eeff2 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Mon, 1 Dec 2025 17:55:57 +0800 Subject: [PATCH 1/4] feat: add vector and analytics buckets to storage config --- pkg/config/config.go | 8 +++++++ pkg/config/storage.go | 37 ++++++++++++++++++++++++++++++++ pkg/config/templates/config.toml | 17 +++++++++++++-- pkg/config/testdata/config.toml | 15 ++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 230cc3e16..ca2e1bff1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -295,6 +295,14 @@ func (s *storage) Clone() storage { img := *s.ImageTransformation copy.ImageTransformation = &img } + if s.IcebergCatalog != nil { + iceberg := *s.IcebergCatalog + copy.IcebergCatalog = &iceberg + } + if s.VectorBuckets != nil { + vector := *s.VectorBuckets + copy.VectorBuckets = &vector + } if s.S3Protocol != nil { s3 := *s.S3Protocol copy.S3Protocol = &s3 diff --git a/pkg/config/storage.go b/pkg/config/storage.go index f26646344..138d86c39 100644 --- a/pkg/config/storage.go +++ b/pkg/config/storage.go @@ -14,6 +14,8 @@ type ( ImgProxyImage string `toml:"-"` FileSizeLimit sizeInBytes `toml:"file_size_limit"` ImageTransformation *imageTransformation `toml:"image_transformation"` + IcebergCatalog *icebergCatalog `toml:"iceberg_catalog"` + VectorBuckets *vectorBuckets `toml:"vector_buckets"` S3Protocol *s3Protocol `toml:"s3_protocol"` S3Credentials storageS3Credentials `toml:"-"` Buckets BucketConfig `toml:"buckets"` @@ -23,6 +25,19 @@ type ( Enabled bool `toml:"enabled"` } + icebergCatalog struct { + Enabled bool `toml:"enabled"` + MaxNamespaces uint `toml:"max_namespaces"` + MaxTables uint `toml:"max_tables"` + MaxCatalogs uint `toml:"max_catalogs"` + } + + vectorBuckets struct { + Enabled bool `toml:"enabled"` + MaxBuckets uint `toml:"max_buckets"` + MaxIndexes uint `toml:"max_indexes"` + } + s3Protocol struct { Enabled bool `toml:"enabled"` } @@ -70,6 +85,17 @@ func (s *storage) ToUpdateStorageConfigBody() v1API.UpdateStorageConfigBody { if s.ImageTransformation != nil { body.Features.ImageTransformation.Enabled = s.ImageTransformation.Enabled } + if s.IcebergCatalog != nil { + body.Features.IcebergCatalog.Enabled = s.IcebergCatalog.Enabled + body.Features.IcebergCatalog.MaxNamespaces = cast.UintToInt(s.IcebergCatalog.MaxNamespaces) + body.Features.IcebergCatalog.MaxTables = cast.UintToInt(s.IcebergCatalog.MaxTables) + body.Features.IcebergCatalog.MaxCatalogs = cast.UintToInt(s.IcebergCatalog.MaxCatalogs) + } + if s.VectorBuckets != nil { + body.Features.VectorBuckets.Enabled = s.VectorBuckets.Enabled + body.Features.VectorBuckets.MaxBuckets = cast.UintToInt(s.VectorBuckets.MaxBuckets) + body.Features.VectorBuckets.MaxIndexes = cast.UintToInt(s.VectorBuckets.MaxIndexes) + } if s.S3Protocol != nil { body.Features.S3Protocol.Enabled = s.S3Protocol.Enabled } @@ -83,6 +109,17 @@ func (s *storage) FromRemoteStorageConfig(remoteConfig v1API.StorageConfigRespon if s.ImageTransformation != nil { s.ImageTransformation.Enabled = remoteConfig.Features.ImageTransformation.Enabled } + if s.IcebergCatalog != nil { + s.IcebergCatalog.Enabled = remoteConfig.Features.IcebergCatalog.Enabled + s.IcebergCatalog.MaxNamespaces = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxNamespaces) + s.IcebergCatalog.MaxTables = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxTables) + s.IcebergCatalog.MaxCatalogs = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxCatalogs) + } + if s.VectorBuckets != nil { + s.VectorBuckets.Enabled = remoteConfig.Features.VectorBuckets.Enabled + s.VectorBuckets.MaxBuckets = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxBuckets) + s.VectorBuckets.MaxIndexes = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxIndexes) + } if s.S3Protocol != nil { s.S3Protocol.Enabled = remoteConfig.Features.S3Protocol.Enabled } diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index 6a070de52..61b45ed15 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -105,12 +105,25 @@ enabled = true # The maximum file size allowed (e.g. "5MB", "500KB"). file_size_limit = "50MiB" +# [storage.s3_protocol] +# enabled = true + # Image transformation API is available to Supabase Pro plan. # [storage.image_transformation] # enabled = true -# [storage.s3_protocol] -# enabled = false +# Iceberge Catalog is available to Supabase Pro plan. +# [storage.iceberg_catalog] +# enabled = true +# max_namespaces = 5 +# max_tables = 10 +# max_catalogs = 2 + +# Vector Buckets is available to Supabase Pro plan. +# [storage.vector_buckets] +# enabled = true +# max_buckets = 10 +# max_indexes = 5 # Uncomment to configure local storage buckets # [storage.buckets.images] diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index 7804e2df3..8b300fa76 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -105,12 +105,25 @@ enabled = true # The maximum file size allowed (e.g. "5MB", "500KB"). file_size_limit = "50MiB" +[storage.s3_protocol] +enabled = true + # Image transformation API is available to Supabase Pro plan. [storage.image_transformation] enabled = true -[storage.s3_protocol] +# Iceberge Catalog is available to Supabase Pro plan. +[storage.iceberg_catalog] +enabled = true +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +# Vector Buckets is available to Supabase Pro plan. +[storage.vector_buckets] enabled = true +max_buckets = 10 +max_indexes = 5 # Uncomment to configure local storage buckets [storage.buckets.images] From 08422ad18da77104c5129d0d57e2ec5aa496778e Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 2 Dec 2025 14:48:32 +0800 Subject: [PATCH 2/4] chore: allow declaring analytics and vector buckets --- pkg/config/config.go | 10 +--- pkg/config/storage.go | 82 +++++++++++++++++++++----------- pkg/config/templates/config.toml | 41 +++++++++------- pkg/config/testdata/config.toml | 27 +++++++---- 4 files changed, 96 insertions(+), 64 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index ca2e1bff1..cc2fa282e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -291,18 +291,12 @@ func (a *auth) Clone() auth { func (s *storage) Clone() storage { copy := *s copy.Buckets = maps.Clone(s.Buckets) + copy.AnalyticsBuckets.Buckets = maps.Clone(s.AnalyticsBuckets.Buckets) + copy.VectorBuckets.Buckets = maps.Clone(s.VectorBuckets.Buckets) if s.ImageTransformation != nil { img := *s.ImageTransformation copy.ImageTransformation = &img } - if s.IcebergCatalog != nil { - iceberg := *s.IcebergCatalog - copy.IcebergCatalog = &iceberg - } - if s.VectorBuckets != nil { - vector := *s.VectorBuckets - copy.VectorBuckets = &vector - } if s.S3Protocol != nil { s3 := *s.S3Protocol copy.S3Protocol = &s3 diff --git a/pkg/config/storage.go b/pkg/config/storage.go index 875c7d640..7fdffb3a8 100644 --- a/pkg/config/storage.go +++ b/pkg/config/storage.go @@ -14,28 +14,30 @@ type ( ImgProxyImage string `toml:"-"` FileSizeLimit sizeInBytes `toml:"file_size_limit"` ImageTransformation *imageTransformation `toml:"image_transformation"` - IcebergCatalog *icebergCatalog `toml:"iceberg_catalog"` - VectorBuckets *vectorBuckets `toml:"vector_buckets"` S3Protocol *s3Protocol `toml:"s3_protocol"` S3Credentials storageS3Credentials `toml:"-"` Buckets BucketConfig `toml:"buckets"` + AnalyticsBuckets analyticsBuckets `toml:"analytics"` + VectorBuckets vectorBuckets `toml:"vector"` } imageTransformation struct { Enabled bool `toml:"enabled"` } - icebergCatalog struct { - Enabled bool `toml:"enabled"` - MaxNamespaces uint `toml:"max_namespaces"` - MaxTables uint `toml:"max_tables"` - MaxCatalogs uint `toml:"max_catalogs"` + analyticsBuckets struct { + Enabled bool `toml:"enabled"` + MaxNamespaces uint `toml:"max_namespaces"` + MaxTables uint `toml:"max_tables"` + MaxCatalogs uint `toml:"max_catalogs"` + Buckets map[string]struct{} `toml:"buckets"` } vectorBuckets struct { - Enabled bool `toml:"enabled"` - MaxBuckets uint `toml:"max_buckets"` - MaxIndexes uint `toml:"max_indexes"` + Enabled bool `toml:"enabled"` + MaxBuckets uint `toml:"max_buckets"` + MaxIndexes uint `toml:"max_indexes"` + Buckets map[string]struct{} `toml:"buckets"` } s3Protocol struct { @@ -83,21 +85,43 @@ func (s *storage) ToUpdateStorageConfigBody() v1API.UpdateStorageConfigBody { } // When local config is not set, we assume platform defaults should not change if s.ImageTransformation != nil { - body.Features.ImageTransformation.Enabled = s.ImageTransformation.Enabled - } - if s.IcebergCatalog != nil { - body.Features.IcebergCatalog.Enabled = s.IcebergCatalog.Enabled - body.Features.IcebergCatalog.MaxNamespaces = cast.UintToInt(s.IcebergCatalog.MaxNamespaces) - body.Features.IcebergCatalog.MaxTables = cast.UintToInt(s.IcebergCatalog.MaxTables) - body.Features.IcebergCatalog.MaxCatalogs = cast.UintToInt(s.IcebergCatalog.MaxCatalogs) - } - if s.VectorBuckets != nil { - body.Features.VectorBuckets.Enabled = s.VectorBuckets.Enabled - body.Features.VectorBuckets.MaxBuckets = cast.UintToInt(s.VectorBuckets.MaxBuckets) - body.Features.VectorBuckets.MaxIndexes = cast.UintToInt(s.VectorBuckets.MaxIndexes) + body.Features.ImageTransformation = &struct { + Enabled bool `json:"enabled"` + }{ + Enabled: s.ImageTransformation.Enabled, + } + } + // Disabling analytics and vector buckets means leaving platform values unchanged + if s.AnalyticsBuckets.Enabled { + body.Features.IcebergCatalog = &struct { + Enabled bool `json:"enabled"` + MaxCatalogs int `json:"maxCatalogs"` + MaxNamespaces int `json:"maxNamespaces"` + MaxTables int `json:"maxTables"` + }{ + Enabled: true, + MaxNamespaces: cast.UintToInt(s.AnalyticsBuckets.MaxNamespaces), + MaxTables: cast.UintToInt(s.AnalyticsBuckets.MaxTables), + MaxCatalogs: cast.UintToInt(s.AnalyticsBuckets.MaxCatalogs), + } + } + if s.VectorBuckets.Enabled { + body.Features.VectorBuckets = &struct { + Enabled bool `json:"enabled"` + MaxBuckets int `json:"maxBuckets"` + MaxIndexes int `json:"maxIndexes"` + }{ + Enabled: true, + MaxBuckets: cast.UintToInt(s.VectorBuckets.MaxBuckets), + MaxIndexes: cast.UintToInt(s.VectorBuckets.MaxIndexes), + } } if s.S3Protocol != nil { - body.Features.S3Protocol.Enabled = s.S3Protocol.Enabled + body.Features.S3Protocol = &struct { + Enabled bool `json:"enabled"` + }{ + Enabled: s.S3Protocol.Enabled, + } } return body } @@ -109,13 +133,13 @@ func (s *storage) FromRemoteStorageConfig(remoteConfig v1API.StorageConfigRespon if s.ImageTransformation != nil { s.ImageTransformation.Enabled = remoteConfig.Features.ImageTransformation.Enabled } - if s.IcebergCatalog != nil { - s.IcebergCatalog.Enabled = remoteConfig.Features.IcebergCatalog.Enabled - s.IcebergCatalog.MaxNamespaces = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxNamespaces) - s.IcebergCatalog.MaxTables = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxTables) - s.IcebergCatalog.MaxCatalogs = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxCatalogs) + if s.AnalyticsBuckets.Enabled { + s.AnalyticsBuckets.Enabled = remoteConfig.Features.IcebergCatalog.Enabled + s.AnalyticsBuckets.MaxNamespaces = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxNamespaces) + s.AnalyticsBuckets.MaxTables = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxTables) + s.AnalyticsBuckets.MaxCatalogs = cast.IntToUint(remoteConfig.Features.IcebergCatalog.MaxCatalogs) } - if s.VectorBuckets != nil { + if s.VectorBuckets.Enabled { s.VectorBuckets.Enabled = remoteConfig.Features.VectorBuckets.Enabled s.VectorBuckets.MaxBuckets = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxBuckets) s.VectorBuckets.MaxIndexes = cast.IntToUint(remoteConfig.Features.VectorBuckets.MaxIndexes) diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index 61b45ed15..4acfbb6c0 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -105,6 +105,14 @@ enabled = true # The maximum file size allowed (e.g. "5MB", "500KB"). file_size_limit = "50MiB" +# Uncomment to configure local storage buckets +# [storage.buckets.images] +# public = false +# file_size_limit = "50MiB" +# allowed_mime_types = ["image/png", "image/jpeg"] +# objects_path = "./images" + +# Uncomment to allow connections via S3 compatible clients # [storage.s3_protocol] # enabled = true @@ -112,25 +120,24 @@ file_size_limit = "50MiB" # [storage.image_transformation] # enabled = true -# Iceberge Catalog is available to Supabase Pro plan. -# [storage.iceberg_catalog] -# enabled = true -# max_namespaces = 5 -# max_tables = 10 -# max_catalogs = 2 +# Store analytical data in S3 for running ETL jobs over Iceberg Catalog +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 -# Vector Buckets is available to Supabase Pro plan. -# [storage.vector_buckets] -# enabled = true -# max_buckets = 10 -# max_indexes = 5 +# Analytics Buckets is available to Supabase Pro plan. +# [storage.analytics.buckets.my-warehouse] -# Uncomment to configure local storage buckets -# [storage.buckets.images] -# public = false -# file_size_limit = "50MiB" -# allowed_mime_types = ["image/png", "image/jpeg"] -# objects_path = "./images" +# Store vector embeddings in S3 for large and durable datasets +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +# Vector Buckets is available to Supabase Pro plan. +# [storage.vector.buckets.documents-openai] [auth] enabled = true diff --git a/pkg/config/testdata/config.toml b/pkg/config/testdata/config.toml index 8b300fa76..0b4974321 100644 --- a/pkg/config/testdata/config.toml +++ b/pkg/config/testdata/config.toml @@ -105,6 +105,14 @@ enabled = true # The maximum file size allowed (e.g. "5MB", "500KB"). file_size_limit = "50MiB" +# Uncomment to configure local storage buckets +[storage.buckets.images] +public = false +file_size_limit = "50MiB" +allowed_mime_types = ["image/png", "image/jpeg"] +objects_path = "./images" + +# Uncomment to allow connections via S3 compatible clients [storage.s3_protocol] enabled = true @@ -112,25 +120,24 @@ enabled = true [storage.image_transformation] enabled = true -# Iceberge Catalog is available to Supabase Pro plan. -[storage.iceberg_catalog] +# Store analytical data in S3 for running ETL jobs over Iceberg Catalog +[storage.analytics] enabled = true max_namespaces = 5 max_tables = 10 max_catalogs = 2 -# Vector Buckets is available to Supabase Pro plan. -[storage.vector_buckets] +# Analytics Buckets is available to Supabase Pro plan. +[storage.analytics.buckets.my-warehouse] + +# Store vector embeddings in S3 for large and durable datasets +[storage.vector] enabled = true max_buckets = 10 max_indexes = 5 -# Uncomment to configure local storage buckets -[storage.buckets.images] -public = false -file_size_limit = "50MiB" -allowed_mime_types = ["image/png", "image/jpeg"] -objects_path = "./images" +# Vector Buckets is available to Supabase Pro plan. +# [storage.vector.buckets.documents-openai] [auth] enabled = true From e26f6a0c8ad1dfdd176043b95c2f9fb6af7deb46 Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 2 Dec 2025 17:43:21 +0800 Subject: [PATCH 3/4] feat: implement bucket upsert clients --- internal/seed/buckets/buckets.go | 21 +++++++++ pkg/config/config.go | 9 ++++ pkg/storage/analytics.go | 68 +++++++++++++++++++++++++++ pkg/storage/vector.go | 81 ++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 pkg/storage/analytics.go create mode 100644 pkg/storage/vector.go diff --git a/internal/seed/buckets/buckets.go b/internal/seed/buckets/buckets.go index 365c3a395..015ddab40 100644 --- a/internal/seed/buckets/buckets.go +++ b/internal/seed/buckets/buckets.go @@ -3,6 +3,7 @@ package buckets import ( "context" "fmt" + "os" "github.com/spf13/afero" "github.com/supabase/cli/internal/storage/client" @@ -29,5 +30,25 @@ func Run(ctx context.Context, projectRef string, interactive bool, fsys afero.Fs if err := api.UpsertBuckets(ctx, utils.Config.Storage.Buckets, filter); err != nil { return err } + prune := func(name string) bool { + label := fmt.Sprintf("Bucket %s not found in %s. Do you want to prune it?", utils.Bold(name), utils.Bold(utils.ConfigPath)) + shouldPrune, err := console.PromptYesNo(ctx, label, false) + if err != nil { + fmt.Fprintln(utils.GetDebugLogger(), err) + } + return shouldPrune + } + if utils.Config.Storage.AnalyticsBuckets.Enabled { + fmt.Fprintln(os.Stderr, "Updating analytics buckets...") + if err := api.UpsertAnalyticsBuckets(ctx, utils.Config.Storage.AnalyticsBuckets.Buckets, prune); err != nil { + return err + } + } + if utils.Config.Storage.VectorBuckets.Enabled { + fmt.Fprintln(os.Stderr, "Updating vector buckets...") + if err := api.UpsertVectorBuckets(ctx, utils.Config.Storage.VectorBuckets.Buckets, prune); err != nil { + return err + } + } return api.UpsertObjects(ctx, utils.Config.Storage.Buckets, utils.NewRootFS(fsys)) } diff --git a/pkg/config/config.go b/pkg/config/config.go index cc2fa282e..ee899f4ac 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -546,6 +546,15 @@ func (c *config) load(v *viper.Viper) error { }); err != nil { return errors.Errorf("failed to parse config: %w", err) } + // Manually parse config to map + c.Storage.AnalyticsBuckets.Buckets = map[string]struct{}{} + for key := range v.GetStringMap("storage.analytics.buckets") { + c.Storage.AnalyticsBuckets.Buckets[key] = struct{}{} + } + c.Storage.VectorBuckets.Buckets = map[string]struct{}{} + for key := range v.GetStringMap("storage.vector.buckets") { + c.Storage.VectorBuckets.Buckets[key] = struct{}{} + } // Convert keys to upper case: https://github.com/spf13/viper/issues/1014 secrets := make(SecretsConfig, len(c.EdgeRuntime.Secrets)) for k, v := range c.EdgeRuntime.Secrets { diff --git a/pkg/storage/analytics.go b/pkg/storage/analytics.go new file mode 100644 index 000000000..12f2abe01 --- /dev/null +++ b/pkg/storage/analytics.go @@ -0,0 +1,68 @@ +package storage + +import ( + "context" + "fmt" + "net/http" + "os" + + "github.com/supabase/cli/pkg/fetcher" +) + +type AnalyticsBucketResponse struct { + Id string `json:"id"` // "test" + Name string `json:"name"` // "test" + CreatedAt string `json:"created_at"` // "2023-10-13T17:48:58.491Z" + UpdatedAt string `json:"updated_at"` // "2023-10-13T17:48:58.491Z" +} + +type CreateAnalyticsBucketRequest struct { + BucketName string `json:"bucketName"` +} + +func (s *StorageAPI) UpsertAnalyticsBuckets(ctx context.Context, bucketConfig map[string]struct{}, filter ...func(string) bool) error { + resp, err := s.Send(ctx, http.MethodGet, "/storage/v1/iceberg/bucket", nil) + if err != nil { + return err + } + buckets, err := fetcher.ParseJSON[[]AnalyticsBucketResponse](resp.Body) + if err != nil { + return err + } + var toDelete []string + exists := make(map[string]struct{}, len(buckets)) + for _, b := range buckets { + exists[b.Name] = struct{}{} + if _, ok := bucketConfig[b.Name]; !ok { + toDelete = append(toDelete, b.Name) + } + } + for name := range bucketConfig { + if _, ok := exists[name]; ok { + fmt.Fprintln(os.Stderr, "Bucket already exists:", name) + continue + } + fmt.Fprintln(os.Stderr, "Creating analytics bucket:", name) + body := CreateAnalyticsBucketRequest{BucketName: name} + if resp, err := s.Send(ctx, http.MethodPost, "/storage/v1/iceberg/bucket", body); err != nil { + return err + } else if err := resp.Body.Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } +OUTER: + for _, name := range toDelete { + for _, keep := range filter { + if !keep(name) { + continue OUTER + } + } + fmt.Fprintln(os.Stderr, "Pruning analytics bucket:", name) + if resp, err := s.Send(ctx, http.MethodDelete, "/storage/v1/iceberg/bucket/"+name, nil); err != nil { + return err + } else if err := resp.Body.Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } + return nil +} diff --git a/pkg/storage/vector.go b/pkg/storage/vector.go new file mode 100644 index 000000000..7218f7b28 --- /dev/null +++ b/pkg/storage/vector.go @@ -0,0 +1,81 @@ +package storage + +import ( + "context" + "fmt" + "net/http" + "os" + + "github.com/supabase/cli/pkg/fetcher" +) + +type VectorBucket struct { + VectorBucketName string `json:"vectorBucketName"` + CreationTime uint64 `json:"creationTime"` +} + +type ListVectorBucketsResponse struct { + VectorBuckets []VectorBucket `json:"vectorBuckets"` +} + +type ListVectorBucketsRequest struct { + MaxResults uint64 `json:"maxResults,omitempty"` + NextToken string `json:"nextToken,omitempty"` + Prefix string `json:"prefix,omitempty"` +} + +type CreateVectorBucketRequest struct { + VectorBucketName string `json:"vectorBucketName"` +} + +type DeleteVectorBucketRequest struct { + VectorBucketName string `json:"vectorBucketName"` +} + +func (s *StorageAPI) UpsertVectorBuckets(ctx context.Context, bucketConfig map[string]struct{}, filter ...func(string) bool) error { + resp, err := s.Send(ctx, http.MethodPost, "/storage/v1/vector/ListVectorBuckets", ListVectorBucketsRequest{}) + if err != nil { + return err + } + result, err := fetcher.ParseJSON[ListVectorBucketsResponse](resp.Body) + if err != nil { + return err + } + var toDelete []string + exists := make(map[string]struct{}, len(result.VectorBuckets)) + for _, b := range result.VectorBuckets { + exists[b.VectorBucketName] = struct{}{} + if _, ok := bucketConfig[b.VectorBucketName]; !ok { + toDelete = append(toDelete, b.VectorBucketName) + } + } + for name := range bucketConfig { + if _, ok := exists[name]; ok { + fmt.Fprintln(os.Stderr, "Bucket already exists:", name) + continue + } + fmt.Fprintln(os.Stderr, "Creating vector bucket:", name) + body := CreateVectorBucketRequest{VectorBucketName: name} + if resp, err := s.Send(ctx, http.MethodPost, "/storage/v1/vector/CreateVectorBucket", body); err != nil { + return err + } else if err := resp.Body.Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } +OUTER: + for _, name := range toDelete { + for _, keep := range filter { + if !keep(name) { + continue OUTER + } + } + fmt.Fprintln(os.Stderr, "Pruning vector bucket:", name) + body := DeleteVectorBucketRequest{VectorBucketName: name} + if resp, err := s.Send(ctx, http.MethodPost, "/storage/v1/vector/DeleteVectorBucket", body); err != nil { + return err + } else if err := resp.Body.Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } + return nil +} From 133ce31ea4f1506671c74bd6005abe2c24260acc Mon Sep 17 00:00:00 2001 From: Qiao Han Date: Tue, 2 Dec 2025 23:48:58 +0800 Subject: [PATCH 4/4] chore: note on hosted platform --- internal/seed/buckets/buckets.go | 4 ++-- pkg/config/templates/config.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/seed/buckets/buckets.go b/internal/seed/buckets/buckets.go index 015ddab40..b922e63d8 100644 --- a/internal/seed/buckets/buckets.go +++ b/internal/seed/buckets/buckets.go @@ -38,13 +38,13 @@ func Run(ctx context.Context, projectRef string, interactive bool, fsys afero.Fs } return shouldPrune } - if utils.Config.Storage.AnalyticsBuckets.Enabled { + if utils.Config.Storage.AnalyticsBuckets.Enabled && len(projectRef) > 0 { fmt.Fprintln(os.Stderr, "Updating analytics buckets...") if err := api.UpsertAnalyticsBuckets(ctx, utils.Config.Storage.AnalyticsBuckets.Buckets, prune); err != nil { return err } } - if utils.Config.Storage.VectorBuckets.Enabled { + if utils.Config.Storage.VectorBuckets.Enabled && len(projectRef) > 0 { fmt.Fprintln(os.Stderr, "Updating vector buckets...") if err := api.UpsertVectorBuckets(ctx, utils.Config.Storage.VectorBuckets.Buckets, prune); err != nil { return err diff --git a/pkg/config/templates/config.toml b/pkg/config/templates/config.toml index 4acfbb6c0..3fb0d0b54 100644 --- a/pkg/config/templates/config.toml +++ b/pkg/config/templates/config.toml @@ -121,6 +121,7 @@ file_size_limit = "50MiB" # enabled = true # Store analytical data in S3 for running ETL jobs over Iceberg Catalog +# This feature is only available on the hosted platform. [storage.analytics] enabled = false max_namespaces = 5 @@ -131,6 +132,7 @@ max_catalogs = 2 # [storage.analytics.buckets.my-warehouse] # Store vector embeddings in S3 for large and durable datasets +# This feature is only available on the hosted platform. [storage.vector] enabled = false max_buckets = 10