feat: add streaming endpoint support for openai

Signed-off-by: Praveen Yadav <pyadav9678@gmail.com>

README.md

@@ -7,7 +7,7 @@
 ## Supported Providers
 |                                               |   Provider     |   Provider Name  |   Support   |   Supported Endpoints    |
 |-----------------------------------------------|----------------|    :---:         |    :---:    |--------------------------|
-|<img src="assets/openai.png" width=16>         | OpenAI         |     openai       |      ✅     |  `/chat/completions`     |
+|<img src="assets/openai.png" width=16>         | OpenAI         |     openai       |      ✅     |  `/chat/completions`, `/chat/completions:stream`     |
 |<img src="assets/anyscale.png" width=16>       | Anyscale       |    anyscale      |      ✅     |  `/chat/completions`     |
 |<img src="assets/deepinfra.jpeg" width=16>     | Deepinfra      |    deepinfra     |      ✅     |  `/chat/completions`     |
 |<img src="assets/togetherai.svg" width=16>     | Together AI	   |    togetherai    |      ✅     |  `/chat/completions`     |

common/go.mod

@@ -4,14 +4,15 @@ go 1.22.1
 
 require (
 	github.com/mcuadros/go-defaults v1.2.0
-	github.com/missingstudio/common v0.0.0-20240311104829-52da85ba3b8d
+	github.com/missingstudio/common v0.0.0-20240314142641-67397ebf8b91
 	github.com/spf13/pflag v1.0.5
 	github.com/zeebo/assert v1.3.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jeremywohl/flatten v1.0.1 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -23,12 +24,14 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/viper v1.18.2 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

common/go.sum

@@ -6,8 +6,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs=
@@ -22,11 +21,9 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc=
 github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY=
-github.com/missingstudio/common v0.0.0-20240311104829-52da85ba3b8d h1:QTxP19QuVQuoy0O0A3twTGbY2jxX2zrilLGaoWvV3b8=
-github.com/missingstudio/common v0.0.0-20240311104829-52da85ba3b8d/go.mod h1:tuHBlzHR1vE/qyhQsg+tGTnCYRQ9Z4QmXWmOyRYjEW8=
+github.com/missingstudio/common v0.0.0-20240314142641-67397ebf8b91 h1:Dfv0HgmH4K5KQurLXCgVTZMHcNivnttHBEVxGQ8JYnU=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
@@ -55,8 +52,8 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
@@ -65,15 +62,13 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

gateway/core/chat/chat.go

@@ -75,3 +75,8 @@ type ChatCompletionResponse struct {
 	Cached            bool               `json:"cached,omitempty" default:"false"`
 	SystemFingerprint string             `json:"system_fingerprint,omitempty"`
 }
+
+type StreamChatData struct {
+	Data []byte
+	Chat *ChatCompletionResponse
+}

gateway/go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/net v0.22.0
 	golang.org/x/text v0.14.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -141,7 +142,6 @@ require (
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
 	google.golang.org/grpc v1.62.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

gateway/internal/api/routes.go

@@ -44,7 +44,7 @@ func (api *API) routes() *chi.Mux {
 		validateInterceptor,
 		otelconnectInterceptor,
 		interceptor.NewAPIKeyInterceptor(api.Logger, api.APIKeyService, false),
-		interceptor.HeadersInterceptor(),
+		interceptor.WithHeaderConfig(),
 		interceptor.RateLimiterInterceptor(api.RateLimiter),
 		interceptor.RetryInterceptor(),
 		interceptor.NewLoggingInterceptor(api.Logger),

gateway/internal/api/v1/chatcompletions.go

@@ -15,6 +15,7 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/router"
 	"github.com/missingstudio/common/errors"
 	llmv1 "github.com/missingstudio/protos/pkg/llm/v1"
+	"google.golang.org/genproto/googleapis/api/httpbody"
 )
 
 var (
@@ -69,7 +70,7 @@ func (s *V1Handler) ChatCompletions(
 		return nil, ErrChatCompletionNotSupported
 	}
 
-	resp, err := chatCompletionProvider.ChatCompletion(ctx, chatCompletionRequestSchema)
+	resp, err := chatCompletionProvider.ChatCompletions(ctx, chatCompletionRequestSchema)
 	if err != nil {
 		return nil, errors.New(err)
 	}
@@ -86,6 +87,60 @@ func (s *V1Handler) ChatCompletions(
 	return connect.NewResponse(chatCompletionResponseSchema), nil
 }
 
+func (s *V1Handler) StreamChatCompletions(ctx context.Context, req *connect.Request[llmv1.ChatCompletionRequest], stream *connect.ServerStream[httpbody.HttpBody]) error {
+	// Check if required headers are available
+	routerConfig := router.GetContextWithRouterConfig(ctx)
+	if routerConfig == nil {
+		return ErrRequiredHeaderNotExit
+	}
+
+	chatCompletionRequestSchema, err := s.createChatCompletionRequestSchema(req.Msg)
+	if err != nil {
+		return errors.New(err)
+	}
+
+	rsvc := router.NewRouterService(routerConfig)
+
+	providerConfig := rsvc.Next()
+	if providerConfig == nil {
+		return ErrNoProviderAbleToServe
+	}
+
+	authConfig := map[string]any{"auth": providerConfig.Auth}
+	connectionObj := provider.Provider{
+		Name:   providerConfig.Name,
+		Config: authConfig,
+	}
+
+	p, err := s.iProviderService.GetProvider(connectionObj)
+	if err != nil {
+		return errors.New(err)
+	}
+
+	// Validate provider configs
+	err = iProvider.Validate(p, authConfig)
+	if err != nil {
+		return errors.NewBadRequest(err.Error())
+	}
+
+	chatCompletionProvider, ok := p.(base.StreamChatCompletionsProvider)
+	if !ok {
+		return ErrChatCompletionNotSupported
+	}
+
+	byteChannel := make(chan []byte)
+	go sendDataStream(byteChannel, stream)
+	return chatCompletionProvider.StreamChatCompletions(ctx, chatCompletionRequestSchema, byteChannel)
+}
+
+func sendDataStream(byteChannel chan []byte, stream *connect.ServerStream[httpbody.HttpBody]) {
+	for data := range byteChannel {
+		_ = stream.Send(&httpbody.HttpBody{
+			Data: data,
+		})
+	}
+}
+
 func (s *V1Handler) createChatCompletionRequestSchema(req *llmv1.ChatCompletionRequest) (*chat.ChatCompletionRequest, error) {
 	payload, err := json.Marshal(req)
 	if err != nil {

gateway/internal/interceptor/headers.go

@@ -9,22 +9,44 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/router"
 )
 
-func HeadersInterceptor() connect.UnaryInterceptorFunc {
-	interceptor := func(next connect.UnaryFunc) connect.UnaryFunc {
-		return connect.UnaryFunc(func(
-			ctx context.Context,
-			req connect.AnyRequest,
-		) (connect.AnyResponse, error) {
-			config := req.Header().Get(constants.XMSConfig)
-
-			rc, err := router.NewRouterConfig(config, req.Header())
-			if err != nil {
-				return nil, errors.ErrRouterConfigNotValid
-			}
-
-			ctx = router.SetContextWithRouterConfig(ctx, rc)
-			return next(ctx, req)
-		})
-	}
-	return connect.UnaryInterceptorFunc(interceptor)
+var _ connect.Interceptor = &headersInterceptor{}
+
+type headersInterceptor struct{}
+
+func WithHeaderConfig() connect.Interceptor {
+	return &headersInterceptor{}
+}
+
+func (h *headersInterceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc {
+	return connect.UnaryFunc(func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
+		config := req.Header().Get(constants.XMSConfig)
+
+		rc, err := router.NewRouterConfig(config, req.Header())
+		if err != nil {
+			return nil, errors.ErrRouterConfigNotValid
+		}
+
+		ctx = router.SetContextWithRouterConfig(ctx, rc)
+		return next(ctx, req)
+	})
+}
+
+func (h *headersInterceptor) WrapStreamingClient(next connect.StreamingClientFunc) connect.StreamingClientFunc {
+	return connect.StreamingClientFunc(func(ctx context.Context, spec connect.Spec) connect.StreamingClientConn {
+		return next(ctx, spec)
+	})
+}
+
+func (h *headersInterceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) connect.StreamingHandlerFunc {
+	return connect.StreamingHandlerFunc(func(ctx context.Context, shc connect.StreamingHandlerConn) error {
+		config := shc.RequestHeader().Get(constants.XMSConfig)
+
+		rc, err := router.NewRouterConfig(config, shc.RequestHeader())
+		if err != nil {
+			return errors.ErrRouterConfigNotValid
+		}
+
+		ctx = router.SetContextWithRouterConfig(ctx, rc)
+		return next(ctx, shc)
+	})
 }

gateway/internal/provider/anyscale/anyscale.go

@@ -12,7 +12,7 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/requester"
 )
 
-func (anyscale *anyscaleProvider) ChatCompletion(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
+func (anyscale *anyscaleProvider) ChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
 	client := requester.NewHTTPClient()
 
 	rawPayload, err := json.Marshal(payload)

gateway/internal/provider/azure/azure.go

@@ -8,7 +8,7 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/provider/openai"
 )
 
-func (az *azureProvider) ChatCompletion(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
+func (az *azureProvider) ChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
 	return nil, errors.New("Not yet implemented")
 }
 

gateway/internal/provider/base/base.go

@@ -28,5 +28,9 @@ type Provider interface {
 var ProviderRegistry = map[string]func(provider.Provider) Provider{}
 
 type ChatCompletionProvider interface {
-	ChatCompletion(context.Context, *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error)
+	ChatCompletions(context.Context, *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error)
+}
+
+type StreamChatCompletionsProvider interface {
+	StreamChatCompletions(context.Context, *chat.ChatCompletionRequest, chan []byte) error
 }

gateway/internal/provider/deepinfra/deepinfra.go

@@ -12,7 +12,7 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/requester"
 )
 
-func (deepinfra *deepinfraProvider) ChatCompletion(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
+func (deepinfra *deepinfraProvider) ChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
 	client := requester.NewHTTPClient()
 
 	rawPayload, err := json.Marshal(payload)

gateway/internal/provider/openai/openai.go

@@ -1,11 +1,13 @@
 package openai
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/missingstudio/ai/gateway/core/chat"
 	"github.com/missingstudio/ai/gateway/core/provider"
@@ -28,9 +30,10 @@ var OpenAIModels = []string{
 	"gpt-3.5-turbo-instruct",
 }
 
-func (oai *openAIProvider) ChatCompletion(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
+func (oai *openAIProvider) ChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
 	client := requester.NewHTTPClient()
 
+	payload.Stream = false
 	rawPayload, err := json.Marshal(payload)
 	if err != nil {
 		return nil, fmt.Errorf("unable to marshal openai chat request payload: %w", err)
@@ -74,3 +77,48 @@ func (oai *openAIProvider) AddDefaultHeaders(req *http.Request, key string) *htt
 	req.Header.Add("Authorization", authorizationHeader)
 	return req
 }
+
+func (oai *openAIProvider) StreamChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest, stream chan []byte) error {
+	client := requester.NewHTTPClient()
+
+	payload.Stream = true
+	rawPayload, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("unable to marshal openai chat request payload: %w", err)
+	}
+
+	requestURL := fmt.Sprintf("%s%s", oai.config.BaseURL, oai.config.ChatCompletions)
+	req, err := http.NewRequestWithContext(ctx, "POST", requestURL, bytes.NewReader(rawPayload))
+	if err != nil {
+		return err
+	}
+
+	req = oai.AddDefaultHeaders(req, provider.AuthorizationHeader)
+	resp, err := client.SendRequestRaw(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		stream <- scanner.Bytes()
+
+		line := scanner.Text()
+		if strings.HasPrefix(line, "data:") {
+			event := strings.TrimPrefix(line, "data:")
+			event = strings.TrimSpace(strings.Trim(event, "\n"))
+			if strings.Contains(line, "[DONE]") {
+				break
+			}
+
+			var data chat.ChatCompletionResponse
+			if err := json.Unmarshal([]byte(event), &data); err != nil {
+				return err
+			}
+		}
+	}
+
+	close(stream)
+	return nil
+}

gateway/internal/provider/togetherai/togetherai.go

@@ -12,7 +12,7 @@ import (
 	"github.com/missingstudio/ai/gateway/internal/requester"
 )
 
-func (ta *togetherAIProvider) ChatCompletion(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
+func (ta *togetherAIProvider) ChatCompletions(ctx context.Context, payload *chat.ChatCompletionRequest) (*chat.ChatCompletionResponse, error) {
 	client := requester.NewHTTPClient()
 
 	rawPayload, err := json.Marshal(payload)