diff --git a/core/gallery/importers/diffuser.go b/core/gallery/importers/diffuser.go new file mode 100644 index 000000000000..c702da3d3025 --- /dev/null +++ b/core/gallery/importers/diffuser.go @@ -0,0 +1,121 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "gopkg.in/yaml.v3" +) + +var _ Importer = &DiffuserImporter{} + +type DiffuserImporter struct{} + +func (i *DiffuserImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return false + } + + b, ok := preferencesMap["backend"].(string) + if ok && b == "diffusers" { + return true + } + + if details.HuggingFace != nil { + for _, file := range details.HuggingFace.Files { + if strings.Contains(file.Path, "model_index.json") || + strings.Contains(file.Path, "scheduler/scheduler_config.json") { + return true + } + } + } + + return false +} + +func (i *DiffuserImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + err = json.Unmarshal(preferences, &preferencesMap) + if err != nil { + return gallery.ModelConfig{}, err + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = filepath.Base(details.URI) + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "Imported from " + details.URI + } + + backend := "diffusers" + b, ok := preferencesMap["backend"].(string) + if ok { + backend = b + } + + pipelineType, ok := preferencesMap["pipeline_type"].(string) + if !ok { + pipelineType = "StableDiffusionPipeline" + } + + schedulerType, ok := preferencesMap["scheduler_type"].(string) + if !ok { + schedulerType = "" + } + + enableParameters, ok := preferencesMap["enable_parameters"].(string) + if !ok { + enableParameters = "negative_prompt,num_inference_steps" + } + + cuda := false + if cudaVal, ok := preferencesMap["cuda"].(bool); ok { + cuda = cudaVal + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + KnownUsecaseStrings: []string{"image"}, + Backend: backend, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{ + Model: details.URI, + }, + }, + Diffusers: config.Diffusers{ + PipelineType: pipelineType, + SchedulerType: schedulerType, + EnableParameters: enableParameters, + CUDA: cuda, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/diffuser_test.go b/core/gallery/importers/diffuser_test.go new file mode 100644 index 000000000000..38765e88bade --- /dev/null +++ b/core/gallery/importers/diffuser_test.go @@ -0,0 +1,246 @@ +package importers_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("DiffuserImporter", func() { + var importer *DiffuserImporter + + BeforeEach(func() { + importer = &DiffuserImporter{} + }) + + Context("Match", func() { + It("should match when backend preference is diffusers", func() { + preferences := json.RawMessage(`{"backend": "diffusers"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain model_index.json", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "model_index.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should match when HuggingFace details contain scheduler config", func() { + hfDetails := &hfapi.ModelDetails{ + Files: []hfapi.ModelFile{ + {Path: "scheduler/scheduler_config.json"}, + }, + } + details := Details{ + URI: "https://huggingface.co/test/model", + HuggingFace: hfDetails, + } + + result := importer.Match(details) + Expect(result).To(BeTrue()) + }) + + It("should not match when URI has no diffuser files and no backend preference", func() { + details := Details{ + URI: "https://example.com/model.bin", + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should not match when backend preference is different", func() { + preferences := json.RawMessage(`{"backend": "llama-cpp"}`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + + It("should return false when JSON preferences are invalid", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + result := importer.Match(details) + Expect(result).To(BeFalse()) + }) + }) + + Context("Import", func() { + It("should import model config with default name and description", func() { + details := Details{ + URI: "https://huggingface.co/test/my-diffuser-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("my-diffuser-model")) + Expect(modelConfig.Description).To(Equal("Imported from https://huggingface.co/test/my-diffuser-model")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: diffusers")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: https://huggingface.co/test/my-diffuser-model")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("pipeline_type: StableDiffusionPipeline")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("enable_parameters: negative_prompt,num_inference_steps")) + }) + + It("should import model config with custom name and description from preferences", func() { + preferences := json.RawMessage(`{"name": "custom-diffuser", "description": "Custom diffuser model"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("custom-diffuser")) + Expect(modelConfig.Description).To(Equal("Custom diffuser model")) + }) + + It("should use custom pipeline_type from preferences", func() { + preferences := json.RawMessage(`{"pipeline_type": "StableDiffusion3Pipeline"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("pipeline_type: StableDiffusion3Pipeline")) + }) + + It("should use default pipeline_type when not specified", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("pipeline_type: StableDiffusionPipeline")) + }) + + It("should use custom scheduler_type from preferences", func() { + preferences := json.RawMessage(`{"scheduler_type": "k_dpmpp_2m"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("scheduler_type: k_dpmpp_2m")) + }) + + It("should use cuda setting from preferences", func() { + preferences := json.RawMessage(`{"cuda": true}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("cuda: true")) + }) + + It("should use custom enable_parameters from preferences", func() { + preferences := json.RawMessage(`{"enable_parameters": "num_inference_steps,guidance_scale"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("enable_parameters: num_inference_steps,guidance_scale")) + }) + + It("should use custom backend from preferences", func() { + preferences := json.RawMessage(`{"backend": "diffusers"}`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: diffusers")) + }) + + It("should handle invalid JSON preferences", func() { + preferences := json.RawMessage(`invalid json`) + details := Details{ + URI: "https://huggingface.co/test/my-model", + Preferences: preferences, + } + + _, err := importer.Import(details) + Expect(err).To(HaveOccurred()) + }) + + It("should extract filename correctly from URI with path", func() { + details := importers.Details{ + URI: "https://huggingface.co/test/path/to/model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("model")) + }) + + It("should include known_usecases as image in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- image")) + }) + + It("should include diffusers configuration in config", func() { + details := Details{ + URI: "https://huggingface.co/test/my-model", + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("diffusers:")) + }) + }) +}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 283a3349a5e4..76020ca80cd8 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -20,6 +20,7 @@ var defaultImporters = []Importer{ &MLXImporter{}, &VLLMImporter{}, &TransformersImporter{}, + &DiffuserImporter{}, } type Details struct { diff --git a/core/http/views/model-editor.html b/core/http/views/model-editor.html index 734a327db205..8729d0829697 100644 --- a/core/http/views/model-editor.html +++ b/core/http/views/model-editor.html @@ -299,6 +299,7 @@

+

Force a specific backend. Leave empty to auto-detect from URI. @@ -401,6 +402,71 @@

Model type for transformers backend. Examples: AutoModelForCausalLM, SentenceTransformer, Mamba, MusicgenForConditionalGeneration. Leave empty to use default (AutoModelForCausalLM).

+ + +
+ + +

+ Pipeline type for diffusers backend. Examples: StableDiffusionPipeline, StableDiffusion3Pipeline, FluxPipeline. Leave empty to use default (StableDiffusionPipeline). +

+
+ + +
+ + +

+ Scheduler type for diffusers backend. Examples: k_dpmpp_2m, euler_a, ddim. Leave empty to use model default. +

+
+ + +
+ + +

+ Enabled parameters for diffusers backend (comma-separated). Leave empty to use default (negative_prompt,num_inference_steps). +

+
+ + +
+ +

+ Enable CUDA support for GPU acceleration with diffusers backend. +

+
@@ -658,7 +724,11 @@

quantizations: '', mmproj_quantizations: '', embeddings: false, - type: '' + type: '', + pipeline_type: '', + scheduler_type: '', + enable_parameters: '', + cuda: false }, isSubmitting: false, currentJobId: null, @@ -733,6 +803,18 @@

if (this.commonPreferences.type && this.commonPreferences.type.trim()) { prefsObj.type = this.commonPreferences.type.trim(); } + if (this.commonPreferences.pipeline_type && this.commonPreferences.pipeline_type.trim()) { + prefsObj.pipeline_type = this.commonPreferences.pipeline_type.trim(); + } + if (this.commonPreferences.scheduler_type && this.commonPreferences.scheduler_type.trim()) { + prefsObj.scheduler_type = this.commonPreferences.scheduler_type.trim(); + } + if (this.commonPreferences.enable_parameters && this.commonPreferences.enable_parameters.trim()) { + prefsObj.enable_parameters = this.commonPreferences.enable_parameters.trim(); + } + if (this.commonPreferences.cuda) { + prefsObj.cuda = true; + } // Add custom preferences (can override common ones) this.preferences.forEach(pref => { diff --git a/docs/go.mod b/docs/go.mod index b8e4544d7d36..4db20dd05bc5 100644 --- a/docs/go.mod +++ b/docs/go.mod @@ -1,8 +1,3 @@ module github.com/mudler/LocalAI/docs go 1.19 - -require ( - github.com/McShelby/hugo-theme-relearn v0.0.0-20251117214752-f69a085322cc // indirect - github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20400 // indirect -) diff --git a/docs/go.sum b/docs/go.sum index a891abc884ad..e69de29bb2d1 100644 --- a/docs/go.sum +++ b/docs/go.sum @@ -1,6 +0,0 @@ -github.com/McShelby/hugo-theme-relearn v0.0.0-20251117214752-f69a085322cc h1:8BvuabGtqXqhT4H01SS7s0zXea0B2R5ZOFEcPugMbNg= -github.com/McShelby/hugo-theme-relearn v0.0.0-20251117214752-f69a085322cc/go.mod h1:mKQQdxZNIlLvAj8X3tMq+RzntIJSr9z7XdzuMomt0IM= -github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20400 h1:L6+F22i76xmeWWwrtijAhUbf3BiRLmpO5j34bgl1ggU= -github.com/gohugoio/hugo-mod-bootstrap-scss/v5 v5.20300.20400/go.mod h1:uekq1D4ebeXgduLj8VIZy8TgfTjrLdSl6nPtVczso78= -github.com/gohugoio/hugo-mod-jslibs-dist/popperjs/v2 v2.21100.20000/go.mod h1:mFberT6ZtcchrsDtfvJM7aAH2bDKLdOnruUHl0hlapI= -github.com/twbs/bootstrap v5.3.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= diff --git a/go.mod b/go.mod index 22e835340ad0..8fc592986524 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( go.opentelemetry.io/otel/metric v1.38.0 go.opentelemetry.io/otel/sdk/metric v1.38.0 google.golang.org/grpc v1.76.0 + google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 oras.land/oras-go/v2 v2.6.0 @@ -65,7 +66,6 @@ require ( github.com/stretchr/testify v1.11.1 // indirect github.com/swaggo/files/v2 v2.0.2 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - google.golang.org/protobuf v1.36.10 // indirect ) require (