feat(model): support search/filter with list endpoints (#559)

Because


- Model page UI needs to allow user to search and filter

This commit

- add search and filter fields in list model methods
- refactor ACL logic to support replica connection
- remove AuthUser

cmd/init/main.go

@@ -3,20 +3,13 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"log"
-	"strings"
 
-	"github.com/gofrs/uuid"
 	"go.opentelemetry.io/otel"
 	"gorm.io/gorm"
 
-	openfgaClient "github.com/openfga/go-sdk/client"
-
 	"github.com/instill-ai/model-backend/config"
-	"github.com/instill-ai/model-backend/pkg/acl"
 	"github.com/instill-ai/model-backend/pkg/datamodel"
-	"github.com/instill-ai/model-backend/pkg/repository"
 
 	database "github.com/instill-ai/model-backend/pkg/db"
 	databaseInit "github.com/instill-ai/model-backend/pkg/init"
@@ -45,7 +38,7 @@ func createModelDefinition(db *gorm.DB, modelDef *modelPB.ModelDefinition) error
 
 func main() {
 
-	if err := config.Init(); err != nil {
+	if err := config.Init(config.ParseConfigFlag()); err != nil {
 		log.Fatal(err.Error())
 	}
 
@@ -71,53 +64,6 @@ func main() {
 	db := database.GetConnection()
 	defer database.Close(db)
 
-	repo := repository.NewRepository(db)
-
-	fgaClient, err := openfgaClient.NewSdkClient(&openfgaClient.ClientConfiguration{
-		ApiScheme: "http",
-		ApiHost:   fmt.Sprintf("%s:%d", config.Config.OpenFGA.Host, config.Config.OpenFGA.Port),
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	var aclClient acl.ACLClient
-	if stores, err := fgaClient.ListStores(context.Background()).Execute(); err == nil {
-		fgaClient.SetStoreId(stores.Stores[0].Id)
-		if models, err := fgaClient.ReadAuthorizationModels(context.Background()).Execute(); err == nil {
-			aclClient = acl.NewACLClient(fgaClient, &models.AuthorizationModels[0].Id)
-		} else {
-			panic(err)
-		}
-
-	} else {
-		panic(err)
-	}
-
-	var models []*datamodel.Model
-	pageToken := ""
-	for {
-		models, _, pageToken, err = repo.ListModelsAdmin(ctx, 100, pageToken, true, false)
-		if err != nil {
-			panic(err)
-		}
-		for _, model := range models {
-			nsType := strings.Split(model.Owner, "/")[0]
-			nsType = nsType[0 : len(nsType)-1]
-			userUID, err := uuid.FromString(strings.Split(model.Owner, "/")[1])
-			if err != nil {
-				panic(err)
-			}
-			err = aclClient.SetOwner("model_", model.UID, nsType, userUID)
-			if err != nil {
-				panic(err)
-			}
-		}
-		if pageToken == "" {
-			break
-		}
-	}
-
 	datamodel.InitJSONSchema(ctx)
 
 	modelDefs := []*modelPB.ModelDefinition{}

cmd/main/main.go

@@ -30,7 +30,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
-	openfgaClient "github.com/openfga/go-sdk/client"
+	openfga "github.com/openfga/api/proto/openfga/v1"
 
 	"github.com/instill-ai/model-backend/config"
 	"github.com/instill-ai/model-backend/pkg/acl"
@@ -75,7 +75,7 @@ func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler) http.Handl
 
 func main() {
 
-	if err := config.Init(); err != nil {
+	if err := config.Init(config.ParseConfigFlag()); err != nil {
 		log.Fatal(err.Error())
 	}
 
@@ -210,28 +210,20 @@ func main() {
 	}
 	defer temporalClient.Close()
 
-	fgaClient, err := openfgaClient.NewSdkClient(&openfgaClient.ClientConfiguration{
-		ApiScheme: "http",
-		ApiHost:   fmt.Sprintf("%s:%d", config.Config.OpenFGA.Host, config.Config.OpenFGA.Port),
-	})
-
-	if err != nil {
-		panic(err)
+	fgaClient, fgaClientConn := acl.InitOpenFGAClient(ctx, config.Config.OpenFGA.Host, config.Config.OpenFGA.Port)
+	if fgaClientConn != nil {
+		defer fgaClientConn.Close()
 	}
+	var fgaReplicaClient openfga.OpenFGAServiceClient
+	var fgaReplicaClientConn *grpc.ClientConn
+	if config.Config.OpenFGA.Replica.Host != "" {
 
-	var aclClient acl.ACLClient
-	if stores, err := fgaClient.ListStores(context.Background()).Execute(); err == nil {
-		fgaClient.SetStoreId(stores.Stores[0].Id)
-		if models, err := fgaClient.ReadAuthorizationModels(context.Background()).Execute(); err == nil {
-			aclClient = acl.NewACLClient(fgaClient, &models.AuthorizationModels[0].Id)
+		fgaReplicaClient, fgaReplicaClientConn = acl.InitOpenFGAClient(ctx, config.Config.OpenFGA.Replica.Host, config.Config.OpenFGA.Replica.Port)
+		if fgaReplicaClientConn != nil {
+			defer fgaReplicaClientConn.Close()
 		}
-		if err != nil {
-			panic(err)
-		}
-
-	} else {
-		panic(err)
 	}
+	aclClient := acl.NewACLClient(fgaClient, fgaReplicaClient, redisClient)
 
 	repo := repository.NewRepository(db)
 

cmd/migration/main.go

@@ -74,7 +74,7 @@ func checkExist(databaseConfig *config.DatabaseConfig) error {
 func main() {
 	migrateFolder, _ := os.Getwd()
 
-	_ = config.Init()
+	_ = config.Init(config.ParseConfigFlag())
 	databaseConfig := config.Config.Database
 	if err := checkExist(&databaseConfig); err != nil {
 		panic(err)

cmd/model/main.go

@@ -149,7 +149,7 @@ func main() {
 	}()
 	grpc_zap.ReplaceGrpcLoggerV2(logger)
 
-	if err := config.Init(); err != nil {
+	if err := config.Init(config.ParseConfigFlag()); err != nil {
 		logger.Fatal(err.Error())
 	}
 

cmd/worker/main.go

@@ -16,11 +16,9 @@ import (
 	"go.temporal.io/sdk/interceptor"
 	"go.temporal.io/sdk/worker"
 
-	openfgaClient "github.com/openfga/go-sdk/client"
 	temporalClient "go.temporal.io/sdk/client"
 
 	"github.com/instill-ai/model-backend/config"
-	"github.com/instill-ai/model-backend/pkg/acl"
 	"github.com/instill-ai/model-backend/pkg/ray"
 	"github.com/instill-ai/model-backend/pkg/repository"
 	"github.com/instill-ai/x/temporal"
@@ -76,7 +74,7 @@ func initTemporalNamespace(ctx context.Context, client temporalClient.Client) {
 
 func main() {
 
-	if err := config.Init(); err != nil {
+	if err := config.Init(config.ParseConfigFlag()); err != nil {
 		log.Fatal(err.Error())
 	}
 
@@ -152,30 +150,7 @@ func main() {
 		initTemporalNamespace(ctx, tempClient)
 	}
 
-	fgaClient, err := openfgaClient.NewSdkClient(&openfgaClient.ClientConfiguration{
-		ApiScheme: "http",
-		ApiHost:   fmt.Sprintf("%s:%d", config.Config.OpenFGA.Host, config.Config.OpenFGA.Port),
-	})
-
-	if err != nil {
-		panic(err)
-	}
-
-	var aclClient acl.ACLClient
-	if stores, err := fgaClient.ListStores(context.Background()).Execute(); err == nil {
-		fgaClient.SetStoreId(stores.Stores[0].Id)
-		if models, err := fgaClient.ReadAuthorizationModels(context.Background()).Execute(); err == nil {
-			aclClient = acl.NewACLClient(fgaClient, &models.AuthorizationModels[0].Id)
-		}
-		if err != nil {
-			panic(err)
-		}
-
-	} else {
-		panic(err)
-	}
-
-	cw := modelWorker.NewWorker(repository.NewRepository(db), redisClient, rayService, &aclClient)
+	cw := modelWorker.NewWorker(repository.NewRepository(db), redisClient, rayService)
 
 	w := worker.New(tempClient, modelWorker.TaskQueue, worker.Options{})
 

config/config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-redis/redis/v9"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/yaml"
+	"github.com/knadh/koanf/providers/confmap"
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"
 )
@@ -45,6 +46,13 @@ type DatabaseConfig struct {
 	Password string `koanf:"password"`
 	Host     string `koanf:"host"`
 	Port     int    `koanf:"port"`
+	Replica  struct {
+		Username             string `koanf:"username"`
+		Password             string `koanf:"password"`
+		Host                 string `koanf:"host"`
+		Port                 int    `koanf:"port"`
+		ReplicationTimeFrame int    `koanf:"replicationtimeframe"` // in seconds
+	} `koanf:"replica"`
 	Name     string `koanf:"name"`
 	Version  uint   `koanf:"version"`
 	TimeZone string `koanf:"timezone"`
@@ -136,8 +144,13 @@ type LogConfig struct {
 
 // OpenFGA config
 type OpenFGAConfig struct {
-	Host string `koanf:"host"`
-	Port int    `koanf:"port"`
+	Host    string `koanf:"host"`
+	Port    int    `koanf:"port"`
+	Replica struct {
+		Host                 string `koanf:"host"`
+		Port                 int    `koanf:"port"`
+		ReplicationTimeFrame int    `koanf:"replicationtimeframe"` // in seconds
+	} `koanf:"replica"`
 }
 
 // Registry config
@@ -166,15 +179,18 @@ type AppConfig struct {
 var Config AppConfig
 
 // Init - Assign global config to decoded config struct
-func Init() error {
+func Init(filePath string) error {
 	k := koanf.New(".")
 	parser := yaml.Parser()
 
-	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-	fileRelativePath := fs.String("file", "config/config.yaml", "configuration file")
-	flag.Parse()
+	if err := k.Load(confmap.Provider(map[string]any{
+		"database.replica.replicationtimeframe": 60,
+		"openfga.replica.replicationtimeframe":  60,
+	}, "."), nil); err != nil {
+		log.Fatal(err.Error())
+	}
 
-	if err := k.Load(file.Provider(*fileRelativePath), parser); err != nil {
+	if err := k.Load(file.Provider(filePath), parser); err != nil {
 		log.Fatal(err.Error())
 	}
 
@@ -200,3 +216,15 @@ func Init() error {
 func ValidateConfig(_ *AppConfig) error {
 	return nil
 }
+
+var defaultConfigPath = "config/config.yaml"
+
+// ParseConfigFlag allows clients to specify the relative path to the file from
+// which the configuration will be loaded.
+func ParseConfigFlag() string {
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+	configPath := fs.String("file", defaultConfigPath, "configuration file")
+	flag.Parse()
+
+	return *configPath
+}

config/config.yaml

@@ -80,7 +80,7 @@ log:
     port: 8095
 openfga:
   host: openfga
-  port: 8080
+  port: 8081
 registry:
   host: registry
   port: 5000

go.mod

@@ -11,16 +11,17 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
-	github.com/iancoleman/strcase v0.2.0
-	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240505184837-87fec8c772eb
+	github.com/iancoleman/strcase v0.3.0
+	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240506211529-26ea3855f779
 	github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
 	github.com/instill-ai/x v0.4.0-alpha
 	github.com/knadh/koanf v1.5.0
 	github.com/mennanov/fieldmask-utils v1.1.0
-	github.com/openfga/go-sdk v0.3.3
+	github.com/openfga/api/proto v0.0.0-20240501220219-2b164f5813a7
 	github.com/pkg/errors v0.9.1
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/stretchr/testify v1.8.4
+	go.einride.tech/aip v0.67.1
 	go.opentelemetry.io/contrib/propagators/b3 v1.17.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
@@ -35,7 +36,7 @@ require (
 	golang.org/x/image v0.15.0
 	golang.org/x/net v0.23.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe
-	google.golang.org/grpc v1.61.1
+	google.golang.org/grpc v1.62.1
 	google.golang.org/protobuf v1.33.0
 	gorm.io/datatypes v1.1.0
 	gorm.io/driver/postgres v1.4.5
@@ -47,12 +48,10 @@ require (
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/tools v0.10.0 // indirect
 )
 
 require (
@@ -70,7 +69,7 @@ require (
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.4.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -102,8 +101,8 @@ require (
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/mysql v1.4.4 // indirect
 )