From b76ebb2e6740eee83dd506847a4f2094af721014 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 15 Nov 2025 21:17:04 +0100 Subject: [PATCH] feat(importers): add transformers and vLLM Signed-off-by: Ettore Di Giacinto --- core/gallery/importers/importers.go | 6 +- core/gallery/importers/llama-cpp.go | 8 + core/gallery/importers/llama-cpp_test.go | 21 +- core/gallery/importers/transformers.go | 110 ++++++++++ core/gallery/importers/transformers_test.go | 219 ++++++++++++++++++++ core/gallery/importers/vllm.go | 98 +++++++++ core/gallery/importers/vllm_test.go | 181 ++++++++++++++++ core/http/views/model-editor.html | 45 +++- 8 files changed, 675 insertions(+), 13 deletions(-) create mode 100644 core/gallery/importers/transformers.go create mode 100644 core/gallery/importers/transformers_test.go create mode 100644 core/gallery/importers/vllm.go create mode 100644 core/gallery/importers/vllm_test.go diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 0be81ed06d0d..238aad6f1634 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -10,9 +10,11 @@ import ( hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" ) -var DefaultImporters = []Importer{ +var defaultImporters = []Importer{ &LlamaCPPImporter{}, &MLXImporter{}, + &VLLMImporter{}, + &TransformersImporter{}, } type Details struct { @@ -52,7 +54,7 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model Preferences: preferences, } - for _, importer := range DefaultImporters { + for _, importer := range defaultImporters { if importer.Match(details) { modelConfig, err = importer.Import(details) if err != nil { diff --git a/core/gallery/importers/llama-cpp.go b/core/gallery/importers/llama-cpp.go index f0e3b9e596f6..669faf79076c 100644 --- a/core/gallery/importers/llama-cpp.go +++ b/core/gallery/importers/llama-cpp.go @@ -80,10 +80,13 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) mmprojQuantsList = strings.Split(mmprojQuants, ",") } + embeddings, _ := preferencesMap["embeddings"].(string) + modelConfig := config.ModelConfig{ Name: name, Description: description, KnownUsecaseStrings: []string{"chat"}, + Options: []string{"use_jinja:true"}, Backend: "llama-cpp", TemplateConfig: config.TemplateConfig{ UseTokenizerTemplate: true, @@ -95,6 +98,11 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) }, } + if embeddings != "" && strings.ToLower(embeddings) == "true" || strings.ToLower(embeddings) == "yes" { + trueV := true + modelConfig.Embeddings = &trueV + } + cfg := gallery.ModelConfig{ Name: name, Description: description, diff --git a/core/gallery/importers/llama-cpp_test.go b/core/gallery/importers/llama-cpp_test.go index 5a3c4d74f44d..a9fe17335c1d 100644 --- a/core/gallery/importers/llama-cpp_test.go +++ b/core/gallery/importers/llama-cpp_test.go @@ -5,20 +5,21 @@ import ( "fmt" "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/mudler/LocalAI/core/gallery/importers" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("LlamaCPPImporter", func() { - var importer *importers.LlamaCPPImporter + var importer *LlamaCPPImporter BeforeEach(func() { - importer = &importers.LlamaCPPImporter{} + importer = &LlamaCPPImporter{} }) Context("Match", func() { It("should match when URI ends with .gguf", func() { - details := importers.Details{ + details := Details{ URI: "https://example.com/model.gguf", } @@ -28,7 +29,7 @@ var _ = Describe("LlamaCPPImporter", func() { It("should match when backend preference is llama-cpp", func() { preferences := json.RawMessage(`{"backend": "llama-cpp"}`) - details := importers.Details{ + details := Details{ URI: "https://example.com/model", Preferences: preferences, } @@ -38,7 +39,7 @@ var _ = Describe("LlamaCPPImporter", func() { }) It("should not match when URI does not end with .gguf and no backend preference", func() { - details := importers.Details{ + details := Details{ URI: "https://example.com/model.bin", } @@ -48,7 +49,7 @@ var _ = Describe("LlamaCPPImporter", func() { It("should not match when backend preference is different", func() { preferences := json.RawMessage(`{"backend": "mlx"}`) - details := importers.Details{ + details := Details{ URI: "https://example.com/model", Preferences: preferences, } @@ -59,7 +60,7 @@ var _ = Describe("LlamaCPPImporter", func() { It("should return false when JSON preferences are invalid", func() { preferences := json.RawMessage(`invalid json`) - details := importers.Details{ + details := Details{ URI: "https://example.com/model.gguf", Preferences: preferences, } @@ -72,7 +73,7 @@ var _ = Describe("LlamaCPPImporter", func() { Context("Import", func() { It("should import model config with default name and description", func() { - details := importers.Details{ + details := Details{ URI: "https://example.com/my-model.gguf", } @@ -89,7 +90,7 @@ var _ = Describe("LlamaCPPImporter", func() { It("should import model config with custom name and description from preferences", func() { preferences := json.RawMessage(`{"name": "custom-model", "description": "Custom description"}`) - details := importers.Details{ + details := Details{ URI: "https://example.com/my-model.gguf", Preferences: preferences, } @@ -106,7 +107,7 @@ var _ = Describe("LlamaCPPImporter", func() { It("should handle invalid JSON preferences", func() { preferences := json.RawMessage(`invalid json`) - details := importers.Details{ + details := Details{ URI: "https://example.com/my-model.gguf", Preferences: preferences, } diff --git a/core/gallery/importers/transformers.go b/core/gallery/importers/transformers.go new file mode 100644 index 000000000000..cd09c366d8ac --- /dev/null +++ b/core/gallery/importers/transformers.go @@ -0,0 +1,110 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +var _ Importer = &TransformersImporter{} + +type TransformersImporter struct{} + +func (i *TransformersImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return false + } + + b, ok := preferencesMap["backend"].(string) + if ok && b == "transformers" { + return true + } + + if details.HuggingFace != nil { + for _, file := range details.HuggingFace.Files { + if strings.Contains(file.Path, "tokenizer.json") || + strings.Contains(file.Path, "tokenizer_config.json") { + return true + } + } + } + + return false +} + +func (i *TransformersImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return gallery.ModelConfig{}, err + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = filepath.Base(details.URI) + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "Imported from " + details.URI + } + + backend := "transformers" + b, ok := preferencesMap["backend"].(string) + if ok { + backend = b + } + + modelType, ok := preferencesMap["type"].(string) + if !ok { + modelType = "AutoModelForCausalLM" + } + + quantization, ok := preferencesMap["quantization"].(string) + if !ok { + quantization = "" + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + KnownUsecaseStrings: []string{"chat"}, + Backend: backend, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{ + Model: details.URI, + }, + }, + TemplateConfig: config.TemplateConfig{ + UseTokenizerTemplate: true, + }, + } + modelConfig.ModelType = modelType + modelConfig.Quantization = quantization + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/transformers_test.go b/core/gallery/importers/transformers_test.go new file mode 100644 index 000000000000..a909e75c1726 --- /dev/null +++ b/core/gallery/importers/transformers_test.go @@ -0,0 +1,219 @@ +package importers_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("TransformersImporter", func() { + var importer *TransformersImporter + + BeforeEach(func() { + importer = &TransformersImporter{} + }) + + Context("Match", func() { + It("should match when backend preference is transformers", func() { + preferences := json.RawMessage(`{"backend": "transformers"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain tokenizer.json", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "tokenizer.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain tokenizer_config.json", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "tokenizer_config.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should not match when URI has no tokenizer files and no backend preference", func() { + details := Details{ + URI: "https://example.com/model.bin", + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should not match when backend preference is different", func() { + preferences := json.RawMessage(`{"backend": "llama-cpp"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should return false when JSON preferences are invalid", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + }) + + Context("Import", func() { + It("should import model config with default name and description", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("my-model")) + Expect(modelConfig.Description).To(Equal("Imported from https://huggingface.co/test/my-model")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: transformers")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: https://huggingface.co/test/my-model")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("type: AutoModelForCausalLM")) + }) + + It("should import model config with custom name and description from preferences", func() { + preferences := json.RawMessage(`{"name": "custom-model", "description": "Custom description"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("custom-model")) + Expect(modelConfig.Description).To(Equal("Custom description")) + }) + + It("should use custom model type from preferences", func() { + preferences := json.RawMessage(`{"type": "SentenceTransformer"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("type: SentenceTransformer")) + }) + + It("should use default model type when not specified", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("type: AutoModelForCausalLM")) + }) + + It("should use custom backend from preferences", func() { + preferences := json.RawMessage(`{"backend": "transformers"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: transformers")) + }) + + It("should use quantization from preferences", func() { + preferences := json.RawMessage(`{"quantization": "int8"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("quantization: int8")) + }) + + It("should handle invalid JSON preferences", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + _, err := importer.Import(details) + Expect(err).To(HaveOccurred()) + }) + + It("should extract filename correctly from URI with path", func() { + details := importers.Details{ + URI: "https://huggingface.co/test/path/to/model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("model")) + }) + + It("should include use_tokenizer_template in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("use_tokenizer_template: true")) + }) + + It("should include known_usecases in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- chat")) + }) + }) +}) diff --git a/core/gallery/importers/vllm.go b/core/gallery/importers/vllm.go new file mode 100644 index 000000000000..be544662a518 --- /dev/null +++ b/core/gallery/importers/vllm.go @@ -0,0 +1,98 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +var _ Importer = &VLLMImporter{} + +type VLLMImporter struct{} + +func (i *VLLMImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return false + } + + b, ok := preferencesMap["backend"].(string) + if ok && b == "vllm" { + return true + } + + if details.HuggingFace != nil { + for _, file := range details.HuggingFace.Files { + if strings.Contains(file.Path, "tokenizer.json") || + strings.Contains(file.Path, "tokenizer_config.json") { + return true + } + } + } + + return false +} + +func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return gallery.ModelConfig{}, err + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = filepath.Base(details.URI) + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "Imported from " + details.URI + } + + backend := "vllm" + b, ok := preferencesMap["backend"].(string) + if ok { + backend = b + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + KnownUsecaseStrings: []string{"chat"}, + Backend: backend, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{ + Model: details.URI, + }, + }, + TemplateConfig: config.TemplateConfig{ + UseTokenizerTemplate: true, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/vllm_test.go b/core/gallery/importers/vllm_test.go new file mode 100644 index 000000000000..b6eb5c953968 --- /dev/null +++ b/core/gallery/importers/vllm_test.go @@ -0,0 +1,181 @@ +package importers_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("VLLMImporter", func() { + var importer *VLLMImporter + + BeforeEach(func() { + importer = &VLLMImporter{} + }) + + Context("Match", func() { + It("should match when backend preference is vllm", func() { + preferences := json.RawMessage(`{"backend": "vllm"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain tokenizer.json", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "tokenizer.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain tokenizer_config.json", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "tokenizer_config.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should not match when URI has no tokenizer files and no backend preference", func() { + details := Details{ + URI: "https://example.com/model.bin", + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should not match when backend preference is different", func() { + preferences := json.RawMessage(`{"backend": "llama-cpp"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should return false when JSON preferences are invalid", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + }) + + Context("Import", func() { + It("should import model config with default name and description", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("my-model")) + Expect(modelConfig.Description).To(Equal("Imported from https://huggingface.co/test/my-model")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: vllm")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: https://huggingface.co/test/my-model")) + }) + + It("should import model config with custom name and description from preferences", func() { + preferences := json.RawMessage(`{"name": "custom-model", "description": "Custom description"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("custom-model")) + Expect(modelConfig.Description).To(Equal("Custom description")) + }) + + It("should use custom backend from preferences", func() { + preferences := json.RawMessage(`{"backend": "vllm"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: vllm")) + }) + + It("should handle invalid JSON preferences", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + _, err := importer.Import(details) + Expect(err).To(HaveOccurred()) + }) + + It("should extract filename correctly from URI with path", func() { + details := importers.Details{ + URI: "https://huggingface.co/test/path/to/model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("model")) + }) + + It("should include use_tokenizer_template in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("use_tokenizer_template: true")) + }) + + It("should include known_usecases in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- chat")) + }) + }) +}) diff --git a/core/http/views/model-editor.html b/core/http/views/model-editor.html index d5b5fce764d7..29f04e3a8990 100644 --- a/core/http/views/model-editor.html +++ b/core/http/views/model-editor.html @@ -130,6 +130,8 @@

+ +

Force a specific backend. Leave empty to auto-detect from URI. @@ -199,6 +201,39 @@

Preferred MMProj quantizations (comma-separated). Examples: fp16, fp32. Leave empty to use default (fp16).

+ + +
+ +

+ Enable embeddings support for this model. +

+
+ + +
+ + +

+ Model type for transformers backend. Examples: AutoModelForCausalLM, SentenceTransformer, Mamba, MusicgenForConditionalGeneration. Leave empty to use default (AutoModelForCausalLM). +

+
@@ -458,7 +493,9 @@

name: '', description: '', quantizations: '', - mmproj_quantizations: '' + mmproj_quantizations: '', + embeddings: false, + type: '' }, isSubmitting: false, currentJobId: null, @@ -527,6 +564,12 @@

if (this.commonPreferences.mmproj_quantizations && this.commonPreferences.mmproj_quantizations.trim()) { prefsObj.mmproj_quantizations = this.commonPreferences.mmproj_quantizations.trim(); } + if (this.commonPreferences.embeddings) { + prefsObj.embeddings = 'true'; + } + if (this.commonPreferences.type && this.commonPreferences.type.trim()) { + prefsObj.type = this.commonPreferences.type.trim(); + } // Add custom preferences (can override common ones) this.preferences.forEach(pref => {