feat(ray): support containerized model deployment (#529)

Because

- Instill Model will support containerized model deployment

This commit

- add registry config
- add `container` model definitionn
- add `ray[serve]` for CLI support, ray-conda will be removed in the
future
- support containerized model deployment/undeployment

Resolves INS-3719
Resolves INS-3837

Dockerfile.dev

@@ -43,10 +43,6 @@ ENV PATH="$VENV/bin:$PATH"
 # RUN export PIP_DEFAULT_TIMEOUT=10000
 RUN pip install --upgrade pip setuptools wheel
 RUN pip install dvc[gs]==2.34.2
-# RUN pip install jsonschema pyyaml
-# RUN pip install --no-cache-dir opencv-contrib-python-headless transformers pillow torch torchvision onnxruntime dvc[gs]==2.34.2
-# RUN pip install --no-cache-dir ray[serve] scikit-image
-# RUN pip install --no-cache-dir instill-sdk==0.3.2rc7
 
 # -- set up Go
 COPY go.mod go.sum ./

Makefile

@@ -10,8 +10,8 @@ export
 
 .PHONY: dev
 dev:							## Run dev container
-	@docker compose ls -q | grep -q "instill-model" && true || \
-		(echo "Error: Run \"make latest PROFILE=model\" in model repository (https://github.com/instill-ai/model) in your local machine first." && exit 1)
+	@docker compose ls -q | grep -q "instill-core" && true || \
+		(echo "Error: Run \"make latest PROFILE=exclude-model\" in model repository (https://github.com/instill-ai/instill-core) in your local machine first." && exit 1)
 	@docker inspect --type container ${SERVICE_NAME} >/dev/null 2>&1 && echo "A container named ${SERVICE_NAME} is already running." || \
 		echo "Run dev container ${SERVICE_NAME}. To stop it, run \"make stop\"."
 	@docker run -d --rm \

config/config.go

@@ -148,6 +148,12 @@ type OpenFGAConfig struct {
 	Port int    `koanf:"port"`
 }
 
+// Registry config
+type RegistryConfig struct {
+	Host string `koanf:"host"`
+	Port int    `koanf:"port"`
+}
+
 // AppConfig defines
 type AppConfig struct {
 	Server                 ServerConfig       `koanf:"server"`
@@ -160,6 +166,7 @@ type AppConfig struct {
 	Temporal               TemporalConfig     `koanf:"temporal"`
 	Controller             ControllerConfig   `koanf:"controller"`
 	OpenFGA                OpenFGAConfig      `koanf:"openfga"`
+	Registry               RegistryConfig     `koanf:"registry"`
 	InitModel              InitModelConfig    `koanf:"initmodel"`
 	Log                    LogConfig          `koanf:"log"`
 }

config/config.yaml

@@ -89,3 +89,6 @@ log:
 openfga:
   host: openfga
   port: 8080
+registry:
+  host: registry
+  port: 5000

config/init/instill/seed/model_definitions.yaml

@@ -63,3 +63,28 @@
         instillUIOrder: 0
         minLength: 0
         maxLength: 1023
+- id: "container"
+  uid: "3b4265b7-7d8c-42f7-b5a9-13737d7dee1d"
+  title: "Container"
+  documentationUrl: "https://www.instill.tech/docs/import-models/local"
+  icon: "local.svg"
+  releaseStage: alpha
+  modelSpec:
+    $schema: "http://json-schema.org/draft-07/schema#"
+    title: "Containerized Model"
+    type: "object"
+    required:
+      - "task"
+    instillShortDescription: ""
+    additionalProperties: false
+    minProperties: 1
+    maxProperties: 1
+    properties:
+      task:
+        type: "string"
+        instillUIComponent: "textfield"
+        title: "Enter your model's Instill Task"
+        description: "Create and upload a containerized model which uses this Instill Task."
+        instillUIOrder: 0
+        minLength: 0
+        maxLength: 1023

go.mod

@@ -100,6 +100,6 @@ require (
 	google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/mysql v1.4.4 // indirect
 )

integration-test/grpc_query_model_definition.js

@@ -41,10 +41,10 @@ export function ListModelDefinitions(header) {
     });
     check(client.invoke('model.model.v1alpha.ModelPublicService/ListModelDefinitions', {}, header), {
       "ListModelDefinitions response status": (r) => r.status === grpc.StatusOK,
-      "ListModelDefinitions response modelDefinitions[0].name": (r) => r.message.modelDefinitions[0].name === "model-definitions/local",
+      "ListModelDefinitions response modelDefinitions[0].name": (r) => r.message.modelDefinitions[0].name === "model-definitions/container",
       "ListModelDefinitions response modelDefinitions[0].uid": (r) => r.message.modelDefinitions[0].uid !== undefined,
-      "ListModelDefinitions response modelDefinitions[0].id": (r) => r.message.modelDefinitions[0].id === "local",
-      "ListModelDefinitions response modelDefinitions[0].title": (r) => r.message.modelDefinitions[0].title === "Local",
+      "ListModelDefinitions response modelDefinitions[0].id": (r) => r.message.modelDefinitions[0].id === "container",
+      "ListModelDefinitions response modelDefinitions[0].title": (r) => r.message.modelDefinitions[0].title === "Container",
       "ListModelDefinitions response modelDefinitions[0].icon": (r) => r.message.modelDefinitions[0].icon !== undefined,
       "ListModelDefinitions response modelDefinitions[0].documentationUrl": (r) => r.message.modelDefinitions[0].documentationUrl !== undefined,
       "ListModelDefinitions response modelDefinitions[0].modelSpec": (r) => r.message.modelDefinitions[0].modelSpec !== undefined,

integration-test/rest_query_model_definition.js

@@ -22,17 +22,17 @@ export function ListModelDefinitions(header) {
         [`GET /v1alpha/model-definitions response next_page_token`]: (r) =>
           r.json().next_page_token !== undefined,
         [`GET /v1alpha/model-definitions response total_size`]: (r) =>
-          r.json().total_size == 2,
+          r.json().total_size == 3,
         [`GET /v1alpha/model-definitions response model_definitions.length`]: (r) =>
-          r.json().model_definitions.length === 2,
+          r.json().model_definitions.length === 3,
         [`GET /v1alpha/model-definitions response model_definitions[0].name`]: (r) =>
-          r.json().model_definitions[0].name === "model-definitions/local",
+          r.json().model_definitions[0].name === "model-definitions/container",
         [`GET /v1alpha/model-definitions response model_definitions[0].uid`]: (r) =>
           r.json().model_definitions[0].uid !== undefined,
         [`GET /v1alpha/model-definitions response model_definitions[0].id`]: (r) =>
-          r.json().model_definitions[0].id === "local",
+          r.json().model_definitions[0].id === "container",
         [`GET /v1alpha/model-definitions response model_definitions[0].title`]: (r) =>
-          r.json().model_definitions[0].title === "Local",
+          r.json().model_definitions[0].title === "Container",
         [`GET /v1alpha/model-definitions response model_definitions[0].documentation_url`]: (r) =>
           r.json().model_definitions[0].documentation_url === "https://www.instill.tech/docs/import-models/local",
         [`GET /v1alpha/model-definitions response model_definitions[0].icon`]: (r) =>
@@ -48,17 +48,17 @@ export function ListModelDefinitions(header) {
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response next_page_token`]: (r) =>
         r.json().next_page_token !== undefined,
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response total_size`]: (r) =>
-        r.json().total_size == 2,
+        r.json().total_size == 3,
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions.length`]: (r) =>
-        r.json().model_definitions.length === 2,
+        r.json().model_definitions.length === 3,
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].name`]: (r) =>
-        r.json().model_definitions[0].name === "model-definitions/local",
+        r.json().model_definitions[0].name === "model-definitions/container",
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].uid`]: (r) =>
         r.json().model_definitions[0].uid !== undefined,
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].id`]: (r) =>
-        r.json().model_definitions[0].id === "local",
+        r.json().model_definitions[0].id === "container",
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].title`]: (r) =>
-        r.json().model_definitions[0].title === "Local",
+        r.json().model_definitions[0].title === "Container",
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].documentation_url`]: (r) =>
         r.json().model_definitions[0].documentation_url === "https://www.instill.tech/docs/import-models/local",
       [`GET /v1alpha/model-definitions?view=VIEW_FULL response model_definitions[0].icon`]: (r) =>

pkg/datamodel/datamodel.go

@@ -129,6 +129,11 @@ type LocalModelConfiguration struct {
 	Tag     string `json:"tag,omitempty"`
 }
 
+type ContainerizedModelConfiguration struct {
+	Task string `json:"task,omitempty"`
+	Tag  string `json:"tag,omitempty"`
+}
+
 type ListModelQuery struct {
 	Owner string
 }

pkg/handler/create.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -270,6 +271,157 @@ func createGitHubModel(s service.Service, ctx context.Context, model *modelPB.Mo
 	}, nil
 }
 
+func createContainerizedModel(s service.Service, ctx context.Context, model *modelPB.Model, ns resource.Namespace, authUser *service.AuthUser, modelDefinition *datamodel.ModelDefinition) (*longrunningpb.Operation, error) {
+
+	eventName := "CreateContainerizedModel"
+
+	ctx, span := tracer.Start(ctx, eventName,
+		trace.WithSpanKind(trace.SpanKindServer))
+	defer span.End()
+
+	logUUID, _ := uuid.NewV4()
+
+	logger, _ := custom_logger.GetZapLogger(ctx)
+
+	var modelConfig datamodel.ContainerizedModelConfiguration
+	b, err := model.GetConfiguration().MarshalJSON()
+	if err != nil {
+		span.SetStatus(1, err.Error())
+		return &longrunningpb.Operation{}, status.Errorf(codes.InvalidArgument, err.Error())
+	}
+	if err := json.Unmarshal(b, &modelConfig); err != nil {
+		span.SetStatus(1, err.Error())
+		return &longrunningpb.Operation{}, status.Errorf(codes.InvalidArgument, err.Error())
+	}
+	if modelConfig.Task == "" {
+		span.SetStatus(1, "Invalid task")
+		return &longrunningpb.Operation{}, status.Errorf(codes.InvalidArgument, "Invalid Task")
+	}
+
+	modelConfig.Tag = "latest"
+
+	bModelConfig, _ := json.Marshal(modelConfig)
+
+	containerizedModel := s.PBToDBModel(ctx, ns, model)
+	containerizedModel.State = datamodel.ModelState(modelPB.Model_STATE_OFFLINE)
+	containerizedModel.Configuration = bModelConfig
+	containerizedModel.ModelDefinitionUID = modelDefinition.UID
+
+	modelRootDir := filepath.Join(config.Config.RayServer.ModelStore, ns.Permalink(), containerizedModel.ID)
+	err = os.MkdirAll(modelRootDir, os.ModePerm)
+	if err != nil {
+		st, err := sterr.CreateErrorResourceInfo(
+			codes.FailedPrecondition,
+			fmt.Sprintf("[handler] create a model error: %s", err.Error()),
+			"Model folder structure",
+			"",
+			"",
+			err.Error(),
+		)
+		if err != nil {
+			logger.Error(err.Error())
+		}
+		utils.RemoveModelRepository(config.Config.RayServer.ModelStore, ns.Permalink(), containerizedModel.ID)
+		span.SetStatus(1, st.Err().Error())
+		return &longrunningpb.Operation{}, st.Err()
+	}
+
+	modelMeta := utils.ModelMeta{
+		Tags: []string{"Containerized", "Experimental"},
+		Task: modelConfig.Task,
+	}
+
+	if val, ok := utils.Tasks[fmt.Sprintf("TASK_%v", strings.ToUpper(modelMeta.Task))]; ok {
+		containerizedModel.Task = datamodel.ModelTask(val)
+	} else {
+		if modelMeta.Task != "" {
+			st, err := sterr.CreateErrorResourceInfo(
+				codes.FailedPrecondition,
+				"[handler] create a model error: unsupported task",
+				"request body",
+				"request body contains unsupported task",
+				"",
+				"",
+			)
+			if err != nil {
+				logger.Error(err.Error())
+			}
+			utils.RemoveModelRepository(config.Config.RayServer.ModelStore, ns.Permalink(), containerizedModel.ID)
+			span.SetStatus(1, st.Err().Error())
+			return &longrunningpb.Operation{}, st.Err()
+		} else {
+			containerizedModel.Task = datamodel.ModelTask(commonPB.Task_TASK_UNSPECIFIED)
+		}
+	}
+
+	// TODO: properly support batch inference
+	maxBatchSize := 0
+	allowedMaxBatchSize := utils.GetSupportedBatchSize(containerizedModel.Task)
+
+	if maxBatchSize > allowedMaxBatchSize {
+		st, e := sterr.CreateErrorPreconditionFailure(
+			"[handler] create a model",
+			[]*errdetails.PreconditionFailure_Violation{
+				{
+					Type:        "MAX BATCH SIZE LIMITATION",
+					Subject:     "Create a model error",
+					Description: fmt.Sprintf("The max_batch_size in config.pbtxt exceeded the limitation %v, please try with a smaller max_batch_size", allowedMaxBatchSize),
+				},
+			})
+		if e != nil {
+			logger.Error(e.Error())
+		}
+		utils.RemoveModelRepository(config.Config.RayServer.ModelStore, ns.Permalink(), containerizedModel.ID)
+		span.SetStatus(1, st.Err().Error())
+		return &longrunningpb.Operation{}, st.Err()
+	}
+
+	wfID, err := s.CreateNamespaceModelAsync(ctx, ns, authUser, containerizedModel)
+	if err != nil {
+		st, err := sterr.CreateErrorResourceInfo(
+			codes.Internal,
+			fmt.Sprintf("[handler] create a model error: %s", err.Error()),
+			"Model service",
+			"",
+			"",
+			err.Error(),
+		)
+		if err != nil {
+			logger.Error(err.Error())
+		}
+		utils.RemoveModelRepository(config.Config.RayServer.ModelStore, ns.Permalink(), containerizedModel.ID)
+		span.SetStatus(1, st.Err().Error())
+		return &longrunningpb.Operation{}, st.Err()
+	}
+
+	// Manually set the custom header to have a StatusCreated http response for REST endpoint
+	if err := grpc.SetHeader(ctx, metadata.Pairs("x-http-code", strconv.Itoa(http.StatusCreated))); err != nil {
+		span.SetStatus(1, err.Error())
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+
+	logger.Info(string(custom_otel.NewLogMessage(
+		span,
+		logUUID.String(),
+		authUser.UID,
+		eventName,
+		custom_otel.SetEventResource(containerizedModel),
+		custom_otel.SetEventResult(&longrunningpb.Operation_Response{
+			Response: &anypb.Any{
+				Value: []byte(wfID),
+			},
+		}),
+	)))
+
+	return &longrunningpb.Operation{
+		Name: fmt.Sprintf("operations/%s", wfID),
+		Done: false,
+		Result: &longrunningpb.Operation_Response{
+			Response: &anypb.Any{},
+		},
+	}, nil
+}
+
 func createHuggingFaceModel(s service.Service, ctx context.Context, model *modelPB.Model, ns resource.Namespace, authUser *service.AuthUser, modelDefinition *datamodel.ModelDefinition) (*longrunningpb.Operation, error) {
 
 	eventName := "CreateHuggingFaceModel"

pkg/handler/public.go

@@ -490,6 +490,8 @@ func (h *PublicHandler) createNamespaceModel(ctx context.Context, req CreateName
 		return createArtiVCModel(h.service, ctx, modelToCreate, ns, authUser, modelDefinition)
 	case "huggingface":
 		return createHuggingFaceModel(h.service, ctx, modelToCreate, ns, authUser, modelDefinition)
+	case "container":
+		return createContainerizedModel(h.service, ctx, modelToCreate, ns, authUser, modelDefinition)
 	default:
 		span.SetStatus(1, fmt.Sprintf("model definition %v is not supported", modelDefinitionID))
 		return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("model definition %v is not supported", modelDefinitionID))
@@ -1238,11 +1240,6 @@ func (h *PublicHandler) watchNamespaceModel(ctx context.Context, req WatchNamesp
 	}
 
 	authUser, err := h.service.AuthenticateUser(ctx, false)
-	if err != nil {
-		span.SetStatus(1, err.Error())
-		return modelPB.Model_STATE_ERROR, err
-	}
-
 	if err != nil {
 		span.SetStatus(1, err.Error())
 		logger.Info(string(custom_otel.NewLogMessage(
@@ -1272,7 +1269,6 @@ func (h *PublicHandler) watchNamespaceModel(ctx context.Context, req WatchNamesp
 	}
 
 	state, _, err := h.service.GetResourceState(ctx, uuid.FromStringOrNil(pbModel.Uid))
-
 	if err != nil {
 		span.SetStatus(1, err.Error())
 		logger.Info(string(custom_otel.NewLogMessage(

pkg/ray/const.go

@@ -129,3 +129,28 @@ type TextGenerationChatOutput struct {
 type VisualQuestionAnsweringOutput struct {
 	Text []string
 }
+
+type ModelDeploymentConfig struct {
+	Applications []Application `yaml:"applications" json:"applications"`
+}
+
+type ApplicationWithAction struct {
+	Application Application
+	IsDeploy    bool
+}
+
+type Application struct {
+	Name        string     `yaml:"name" json:"name"`
+	ImportPath  string     `yaml:"import_path" json:"import_path"`
+	RoutePrefix string     `yaml:"route_prefix" json:"route_prefix"`
+	RuntimeEnv  RuntimeEnv `yaml:"runtime_env" json:"runtime_env"`
+}
+
+type RuntimeEnv struct {
+	Container Container `yaml:"container" json:"container"`
+}
+
+type Container struct {
+	Image      string   `yaml:"image" json:"image"`
+	RunOptions []string `yaml:"run_options" json:"run_options"`
+}