Make multimodal work w/ new projector backend

ollama · Dec 5, 2023 · 1c4fdb2 · 1c4fdb2
1 parent 34deadd
commit 1c4fdb2
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 14 deletions.
diff --git a/api/types.go b/api/types.go
@@ -202,10 +202,9 @@ type TokenResponse struct {
 }
 
 type GenerateResponse struct {
-	Model      string    `json:"model"`
-	MultiModal bool      `json:"multi_modal"`
-	CreatedAt  time.Time `json:"created_at"`
-	Response   string    `json:"response"`
+	Model     string    `json:"model"`
+	CreatedAt time.Time `json:"created_at"`
+	Response  string    `json:"response"`
 
 	ModelConfiguration ModelConfiguration `json:"model_configuration"`
 

diff --git a/cmd/cmd.go b/cmd/cmd.go
@@ -601,7 +601,13 @@ func generate(cmd *cobra.Command, opts generateOptions) error {
 
 	ctx := cmd.Context()
 	ctx = context.WithValue(ctx, generateContextKey("context"), latest.Context)
-	ctx = context.WithValue(ctx, "multimodal", latest.MultiModal)
+
+	for _, f := range latest.ModelConfiguration.ModelFamilies {
+		if f == "clip" {
+			ctx = context.WithValue(ctx, "multimodal", true)
+		}
+	}
+
 	cmd.SetContext(ctx)
 
 	return nil

diff --git a/llm/llama.go b/llm/llama.go
@@ -392,9 +392,6 @@ func newLlama(model string, adapters, projectors []string, runners []ModelRunner
 	if opts.UseNUMA {
 		params = append(params, "--numa")
 	}
-	if mmprojPath != "" {
-		params = append(params, "--mmproj", mmprojPath)
-	}
 
 	var runnerErr error
 

diff --git a/server/images.go b/server/images.go
@@ -303,8 +303,6 @@ func GetModel(name string) (*Model, error) {
 				return nil, err
 			}
 			model.License = append(model.License, string(bts))
-		case "application/vnd.ollama.image.mmproj":
-			model.MMProjPath = filename
 		}
 	}
 
@@ -830,10 +828,6 @@ TEMPLATE """{{ .Template }}"""
 SYSTEM """{{ .System }}"""
 {{- end }}
 
-{{- if .MMProjPath }}
-MMPROJ {{ .MMProjPath }}
-{{- end }}
-
 {{- range $adapter := .AdapterPaths }}
 ADAPTER {{ $adapter }}
 {{- end }}