feat: siliconflow support audio image video

core/relay/adaptor/openai/gemini.go

@@ -3,7 +3,10 @@ package openai
 import (
 	"bytes"
 	"fmt"
+	"mime"
 	"net/http"
+	"net/url"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -685,19 +688,16 @@ func convertGeminiContentToOpenAI(
 			hasContent = true
 
 		case part.InlineData != nil:
-			// Handle image
-			imageURL := part.InlineData.Data
-			if !strings.HasPrefix(imageURL, "http") && !strings.HasPrefix(imageURL, "data:") {
-				// Base64 data
-				imageURL = "data:" + part.InlineData.MimeType + ";base64," + part.InlineData.Data
-			}
-
-			currentContentParts = append(currentContentParts, relaymodel.MessageContent{
-				Type: relaymodel.ContentTypeImageURL,
-				ImageURL: &relaymodel.ImageURL{
-					URL: imageURL,
-				},
-			})
+			currentContentParts = append(
+				currentContentParts,
+				convertGeminiInlineDataToOpenAIContent(part.InlineData),
+			)
+			hasContent = true
+		case part.FileData != nil:
+			currentContentParts = append(
+				currentContentParts,
+				convertGeminiFileDataToOpenAIContent(part.FileData),
+			)
 			hasContent = true
 		}
 	}
@@ -720,6 +720,96 @@ func convertGeminiContentToOpenAI(
 	return messages
 }
 
+func convertGeminiInlineDataToOpenAIContent(
+	inlineData *relaymodel.GeminiInlineData,
+) relaymodel.MessageContent {
+	dataURL := inlineData.Data
+	if !strings.HasPrefix(dataURL, "http") && !strings.HasPrefix(dataURL, "data:") {
+		dataURL = "data:" + inlineData.MimeType + ";base64," + inlineData.Data
+	}
+
+	switch {
+	case strings.HasPrefix(inlineData.MimeType, "audio/"):
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeInputAudio,
+			InputAudio: &relaymodel.InputAudio{
+				URL: dataURL,
+			},
+		}
+	case strings.HasPrefix(inlineData.MimeType, "video/"):
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeVideoURL,
+			VideoURL: &relaymodel.VideoURL{
+				URL: dataURL,
+			},
+		}
+	default:
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeImageURL,
+			ImageURL: &relaymodel.ImageURL{
+				URL: dataURL,
+			},
+		}
+	}
+}
+
+func convertGeminiFileDataToOpenAIContent(
+	fileData *relaymodel.GeminiFileData,
+) relaymodel.MessageContent {
+	mimeType := fileData.MimeType
+	if mimeType == "" {
+		mimeType = inferGeminiFileDataMimeType(fileData.FileURI)
+	}
+
+	switch {
+	case strings.HasPrefix(mimeType, "audio/"):
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeInputAudio,
+			InputAudio: &relaymodel.InputAudio{
+				URL: fileData.FileURI,
+			},
+		}
+	case strings.HasPrefix(mimeType, "video/"):
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeVideoURL,
+			VideoURL: &relaymodel.VideoURL{
+				URL: fileData.FileURI,
+			},
+		}
+	default:
+		return relaymodel.MessageContent{
+			Type: relaymodel.ContentTypeImageURL,
+			ImageURL: &relaymodel.ImageURL{
+				URL: fileData.FileURI,
+			},
+		}
+	}
+}
+
+func inferGeminiFileDataMimeType(fileURI string) string {
+	if after, ok := strings.CutPrefix(fileURI, "data:"); ok {
+		mediaType := after
+		if beforeParams, _, ok := strings.Cut(mediaType, ";"); ok {
+			return beforeParams
+		}
+
+		if beforeData, _, ok := strings.Cut(mediaType, ","); ok {
+			return beforeData
+		}
+	}
+
+	path := fileURI
+	if parsed, err := url.Parse(fileURI); err == nil && parsed.Path != "" {
+		path = parsed.Path
+	}
+
+	if ext := filepath.Ext(path); ext != "" {
+		return mime.TypeByExtension(strings.ToLower(ext))
+	}
+
+	return ""
+}
+
 // ConvertGeminiToResponsesRequest converts a Gemini request to Responses API format
 func ConvertGeminiToResponsesRequest(
 	meta *meta.Meta,

core/relay/adaptor/siliconflow/adaptor.go

@@ -1,7 +1,9 @@
 package siliconflow
 
 import (
+	"fmt"
 	"net/http"
+	"net/url"
 
 	"github.com/gin-gonic/gin"
 	"github.com/labring/aiproxy/core/model"
@@ -10,6 +12,7 @@ import (
 	"github.com/labring/aiproxy/core/relay/adaptor/registry"
 	"github.com/labring/aiproxy/core/relay/meta"
 	"github.com/labring/aiproxy/core/relay/mode"
+	"github.com/labring/aiproxy/core/relay/utils"
 )
 
 var _ adaptor.Adaptor = (*Adaptor)(nil)
@@ -30,25 +33,112 @@ func (a *Adaptor) DefaultBaseURL() string {
 
 func (a *Adaptor) Metadata() adaptor.Metadata {
 	return adaptor.Metadata{
-		Readme: "SiliconFlow API\nOpenAI-compatible chat, embeddings, audio, and rerank endpoints\nSupports Gemini-compatible request conversion",
+		Readme: "SiliconFlow API\nOpenAI-compatible chat, embeddings, image, audio, rerank, and video endpoints\nChat supports audio/video understanding request conversion",
 		Models: ModelList,
 	}
 }
 
-//
+func (a *Adaptor) SupportMode(meta *meta.Meta) bool {
+	m := adaptor.ModeFromMeta(meta)
+
+	return m == mode.ChatCompletions ||
+		m == mode.Completions ||
+		m == mode.Embeddings ||
+		m == mode.ImagesGenerations ||
+		m == mode.AudioSpeech ||
+		m == mode.AudioTranscription ||
+		m == mode.Rerank ||
+		m == mode.VideoGenerationsJobs ||
+		m == mode.VideoGenerationsGetJobs ||
+		m == mode.VideoGenerationsContent ||
+		m == mode.Videos ||
+		m == mode.VideosGet ||
+		m == mode.VideosContent ||
+		m == mode.Anthropic ||
+		m == mode.Gemini
+}
+
+func (a *Adaptor) GetRequestURL(
+	meta *meta.Meta,
+	_ adaptor.Store,
+	_ *gin.Context,
+) (adaptor.RequestURL, error) {
+	u := meta.Channel.BaseURL
+
+	var path string
+
+	switch meta.Mode {
+	case mode.ChatCompletions, mode.Anthropic, mode.Gemini:
+		path = "/chat/completions"
+	case mode.Completions:
+		path = "/completions"
+	case mode.Embeddings:
+		path = "/embeddings"
+	case mode.ImagesGenerations:
+		path = "/images/generations"
+	case mode.AudioSpeech:
+		path = "/audio/speech"
+	case mode.AudioTranscription:
+		path = "/audio/transcriptions"
+	case mode.Rerank:
+		path = "/rerank"
+	case mode.VideoGenerationsJobs, mode.Videos:
+		path = "/video/submit"
+	case mode.VideoGenerationsGetJobs, mode.VideoGenerationsContent,
+		mode.VideosGet, mode.VideosContent:
+		path = "/video/status"
+	default:
+		return adaptor.RequestURL{}, fmt.Errorf("unsupported mode: %s", meta.Mode)
+	}
+
+	requestURL, err := url.JoinPath(u, path)
+	if err != nil {
+		return adaptor.RequestURL{}, err
+	}
+
+	return adaptor.RequestURL{
+		Method: http.MethodPost,
+		URL:    requestURL,
+	}, nil
+}
 
 func (a *Adaptor) ConvertRequest(
 	meta *meta.Meta,
 	store adaptor.Store,
 	req *http.Request,
 ) (adaptor.ConvertResult, error) {
 	switch meta.Mode {
+	case mode.ChatCompletions:
+		return openai.ConvertChatCompletionsRequest(
+			meta,
+			req,
+			false,
+			patchChatMultimodalContent,
+		)
 	case mode.Embeddings:
 		if isVLEmbeddingModel(meta) {
 			return openai.ConvertEmbeddingsRequest(meta, req, false, patchVLEmbeddingsInput)
 		}
 
 		return a.Adaptor.ConvertRequest(meta, store, req)
+	case mode.ImagesGenerations:
+		return ConvertImageRequest(meta, req)
+	case mode.VideoGenerationsJobs:
+		return ConvertVideoRequest(meta, req)
+	case mode.Videos:
+		return ConvertVideoRequest(meta, req)
+	case mode.VideoGenerationsGetJobs:
+		return ConvertVideoStatusRequest(meta, req)
+	case mode.VideoGenerationsContent:
+		return ConvertVideoContentStatusRequest(meta, req)
+	case mode.VideosGet:
+		return ConvertVideosStatusRequest(meta, req)
+	case mode.VideosContent:
+		return ConvertVideosStatusRequest(meta, req)
+	case mode.Anthropic:
+		return openai.ConvertClaudeRequest(meta, req, patchSiliconFlowMultimodalContent)
+	case mode.Gemini:
+		return openai.ConvertGeminiRequest(meta, req, patchSiliconFlowMultimodalContent)
 	default:
 		return a.Adaptor.ConvertRequest(meta, store, req)
 	}
@@ -61,6 +151,24 @@ func (a *Adaptor) DoResponse(
 	resp *http.Response,
 ) (adaptor.DoResponseResult, adaptor.Error) {
 	switch meta.Mode {
+	case mode.ChatCompletions:
+		if utils.IsStreamResponse(resp) {
+			return openai.StreamHandler(meta, c, resp, nil)
+		}
+
+		return openai.Handler(meta, c, resp, nil)
+	case mode.Anthropic:
+		if utils.IsStreamResponse(resp) {
+			return openai.ClaudeStreamHandler(meta, c, resp)
+		}
+
+		return openai.ClaudeHandler(meta, c, resp)
+	case mode.Gemini:
+		if utils.IsStreamResponse(resp) {
+			return openai.GeminiStreamHandler(meta, c, resp)
+		}
+
+		return openai.GeminiHandler(meta, c, resp)
 	case mode.AudioSpeech:
 		if resp.StatusCode != http.StatusOK {
 			return adaptor.DoResponseResult{}, ErrorHandler(resp)
@@ -84,6 +192,14 @@ func (a *Adaptor) DoResponse(
 		}
 
 		return a.Adaptor.DoResponse(meta, store, c, resp)
+	case mode.ImagesGenerations:
+		return ImageHandler(meta, c, resp)
+	case mode.VideoGenerationsJobs, mode.Videos:
+		return VideoSubmitHandler(meta, store, c, resp)
+	case mode.VideoGenerationsGetJobs, mode.VideosGet:
+		return VideoStatusHandler(meta, store, c, resp)
+	case mode.VideoGenerationsContent, mode.VideosContent:
+		return VideoContentHandler(meta, c, resp)
 	default:
 		if !adaptor.IsSuccessfulResponseStatus(meta.Mode, resp.StatusCode) {
 			return adaptor.DoResponseResult{}, ErrorHandler(resp)