Fix exporting structs

decoder/decoder.go

@@ -14,15 +14,15 @@ var (
 	ErrDecoding = errors.New("decoding output error")
 )
 
-type jsonDecoder struct {
+type JSONDecoder struct {
 	output types.M
 }
 
-func NewJSONDecoder() *jsonDecoder {
-	return &jsonDecoder{}
+func NewJSONDecoder() *JSONDecoder {
+	return &JSONDecoder{}
 }
 
-func (d *jsonDecoder) Decode(input string) (types.M, error) {
+func (d *JSONDecoder) Decode(input string) (types.M, error) {
 	err := json.Unmarshal([]byte(input), &d.output)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", ErrDecoding, err)
@@ -33,18 +33,18 @@ func (d *jsonDecoder) Decode(input string) (types.M, error) {
 	}, nil
 }
 
-type regExDecoder struct {
+type RegExDecoder struct {
 	output types.M
 	regex  string
 }
 
-func NewRegExDecoder(regex string) *regExDecoder {
-	return &regExDecoder{
+func NewRegExDecoder(regex string) *RegExDecoder {
+	return &RegExDecoder{
 		regex: regex,
 	}
 }
 
-func (d *regExDecoder) Decode(input string) (types.M, error) {
+func (d *RegExDecoder) Decode(input string) (types.M, error) {
 	re, err := regexp.Compile(d.regex)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", ErrDecoding, err)

decoder/decoder_test.go

@@ -40,7 +40,7 @@ func TestJSONDecoder_Decode(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			d := &jsonDecoder{
+			d := &JSONDecoder{
 				output: tt.fields.output,
 			}
 			got, err := d.Decode(tt.args.input)
@@ -86,7 +86,7 @@ func TestRegExDecoder_Decode(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			d := &regExDecoder{
+			d := &RegExDecoder{
 				output: tt.fields.output,
 				regex:  tt.fields.regex,
 			}

embedder/huggingface/feature_extraction.go

@@ -16,7 +16,7 @@ type featureExtractionRequest struct {
 	Options options  `json:"options,omitempty"`
 }
 
-func (h *huggingFaceEmbedder) featureExtraction(ctx context.Context, text []string) ([]embedder.Embedding, error) {
+func (h *HuggingFaceEmbedder) featureExtraction(ctx context.Context, text []string) ([]embedder.Embedding, error) {
 
 	isTrue := true
 	request := featureExtractionRequest{

embedder/huggingface/http.go

@@ -11,7 +11,7 @@ import (
 
 const APIBaseURL = "https://api-inference.huggingface.co/pipeline/feature-extraction/"
 
-func (h *huggingFaceEmbedder) doRequest(ctx context.Context, jsonBody []byte, model string) ([]byte, error) {
+func (h *HuggingFaceEmbedder) doRequest(ctx context.Context, jsonBody []byte, model string) ([]byte, error) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, APIBaseURL+model, bytes.NewBuffer(jsonBody))
 	if err != nil {

embedder/huggingface/huggingface.go

@@ -11,28 +11,28 @@ const (
 	hfDefaultEmbedderModel = "sentence-transformers/all-MiniLM-L6-v2"
 )
 
-type huggingFaceEmbedder struct {
+type HuggingFaceEmbedder struct {
 	token string
 	model string
 }
 
-func New() *huggingFaceEmbedder {
-	return &huggingFaceEmbedder{
+func New() *HuggingFaceEmbedder {
+	return &HuggingFaceEmbedder{
 		token: os.Getenv("HUGGING_FACE_HUB_TOKEN"),
 		model: hfDefaultEmbedderModel,
 	}
 }
 
-func (h *huggingFaceEmbedder) WithToken(token string) *huggingFaceEmbedder {
+func (h *HuggingFaceEmbedder) WithToken(token string) *HuggingFaceEmbedder {
 	h.token = token
 	return h
 }
 
-func (h *huggingFaceEmbedder) WithModel(model string) *huggingFaceEmbedder {
+func (h *HuggingFaceEmbedder) WithModel(model string) *HuggingFaceEmbedder {
 	h.model = model
 	return h
 }
 
-func (h *huggingFaceEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
+func (h *HuggingFaceEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
 	return h.featureExtraction(ctx, texts)
 }

embedder/llamacpp/llamacpp.go

@@ -10,36 +10,36 @@ import (
 	"github.com/henomis/lingoose/embedder"
 )
 
-type llamaCppEmbedder struct {
+type LlamaCppEmbedder struct {
 	llamacppPath string
 	llamacppArgs []string
 	modelPath    string
 }
 
-func New() *llamaCppEmbedder {
-	return &llamaCppEmbedder{
+func New() *LlamaCppEmbedder {
+	return &LlamaCppEmbedder{
 		llamacppPath: "./llama.cpp/embedding",
 		modelPath:    "./llama.cpp/models/7B/ggml-model-q4_0.bin",
 		llamacppArgs: []string{},
 	}
 }
 
-func (l *llamaCppEmbedder) WithLlamaCppPath(llamacppPath string) *llamaCppEmbedder {
+func (l *LlamaCppEmbedder) WithLlamaCppPath(llamacppPath string) *LlamaCppEmbedder {
 	l.llamacppPath = llamacppPath
 	return l
 }
 
-func (l *llamaCppEmbedder) WithModel(modelPath string) *llamaCppEmbedder {
+func (l *LlamaCppEmbedder) WithModel(modelPath string) *LlamaCppEmbedder {
 	l.modelPath = modelPath
 	return l
 }
 
-func (l *llamaCppEmbedder) WithArgs(llamacppArgs []string) *llamaCppEmbedder {
+func (l *LlamaCppEmbedder) WithArgs(llamacppArgs []string) *LlamaCppEmbedder {
 	l.llamacppArgs = llamacppArgs
 	return l
 }
 
-func (o *llamaCppEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
+func (o *LlamaCppEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
 
 	embeddings := make([]embedder.Embedding, len(texts))
 	for i, text := range texts {
@@ -52,7 +52,7 @@ func (o *llamaCppEmbedder) Embed(ctx context.Context, texts []string) ([]embedde
 	return embeddings, nil
 }
 
-func (l *llamaCppEmbedder) embed(ctx context.Context, text string) (embedder.Embedding, error) {
+func (l *LlamaCppEmbedder) embed(ctx context.Context, text string) (embedder.Embedding, error) {
 
 	_, err := os.Stat(l.llamacppPath)
 	if err != nil {

embedder/openai/openai.go

@@ -57,26 +57,26 @@ var modelToString = map[Model]string{
 	AdaEmbeddingV2:        "text-embedding-ada-002",
 }
 
-type openAIEmbedder struct {
+type OpenAIEmbedder struct {
 	openAIClient *openai.Client
 	model        Model
 }
 
-func New(model Model) *openAIEmbedder {
+func New(model Model) *OpenAIEmbedder {
 	openAIKey := os.Getenv("OPENAI_API_KEY")
 
-	return &openAIEmbedder{
+	return &OpenAIEmbedder{
 		openAIClient: openai.NewClient(openAIKey),
 		model:        model,
 	}
 }
 
-func (o *openAIEmbedder) WithClient(client *openai.Client) *openAIEmbedder {
+func (o *OpenAIEmbedder) WithClient(client *openai.Client) *OpenAIEmbedder {
 	o.openAIClient = client
 	return o
 }
 
-func (o *openAIEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
+func (o *OpenAIEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
 	maxTokens := o.getMaxTokens()
 
 	embeddings, err := o.concurrentEmbed(ctx, texts, maxTokens)
@@ -87,7 +87,7 @@ func (o *openAIEmbedder) Embed(ctx context.Context, texts []string) ([]embedder.
 	return embeddings, nil
 }
 
-func (o *openAIEmbedder) concurrentEmbed(ctx context.Context, texts []string, maxTokens int) ([]embedder.Embedding, error) {
+func (o *OpenAIEmbedder) concurrentEmbed(ctx context.Context, texts []string, maxTokens int) ([]embedder.Embedding, error) {
 
 	type indexedEmbeddings struct {
 		index     int
@@ -139,7 +139,7 @@ func (o *openAIEmbedder) concurrentEmbed(ctx context.Context, texts []string, ma
 	return result, nil
 }
 
-func (o *openAIEmbedder) safeEmbed(ctx context.Context, text string, maxTokens int) (embedder.Embedding, error) {
+func (o *OpenAIEmbedder) safeEmbed(ctx context.Context, text string, maxTokens int) (embedder.Embedding, error) {
 
 	sanitizedText := text
 	if strings.HasSuffix(o.model.String(), "001") {
@@ -160,7 +160,7 @@ func (o *openAIEmbedder) safeEmbed(ctx context.Context, text string, maxTokens i
 
 }
 
-func (o *openAIEmbedder) chunkText(text string, maxTokens int) ([]string, error) {
+func (o *OpenAIEmbedder) chunkText(text string, maxTokens int) ([]string, error) {
 
 	tokens, err := o.textToTokens(text)
 	if err != nil {
@@ -185,7 +185,7 @@ func (o *openAIEmbedder) chunkText(text string, maxTokens int) ([]string, error)
 	return textChunks, nil
 }
 
-func (o *openAIEmbedder) getEmebeddingsForChunks(ctx context.Context, chunks []string) ([]embedder.Embedding, []float64, error) {
+func (o *OpenAIEmbedder) getEmebeddingsForChunks(ctx context.Context, chunks []string) ([]embedder.Embedding, []float64, error) {
 
 	chunkLens := []float64{}
 
@@ -202,7 +202,7 @@ func (o *openAIEmbedder) getEmebeddingsForChunks(ctx context.Context, chunks []s
 
 }
 
-func (t *openAIEmbedder) openAICreateEmebeddings(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
+func (t *OpenAIEmbedder) openAICreateEmebeddings(ctx context.Context, texts []string) ([]embedder.Embedding, error) {
 
 	resp, err := t.openAIClient.CreateEmbeddings(
 		ctx,

embedder/openai/openai_test.go

@@ -82,7 +82,7 @@ func Test_openAIEmbedder_splitText(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			o := &openAIEmbedder{
+			o := &OpenAIEmbedder{
 				openAIClient: tt.fields.openAIClient,
 				model:        tt.fields.model,
 			}

embedder/openai/token.go

@@ -4,7 +4,7 @@ import (
 	"github.com/pkoukk/tiktoken-go"
 )
 
-func (o *openAIEmbedder) textToTokens(text string) ([]int, error) {
+func (o *OpenAIEmbedder) textToTokens(text string) ([]int, error) {
 	tokenizer, err := tiktoken.EncodingForModel(o.model.String())
 	if err != nil {
 		return nil, err
@@ -13,7 +13,7 @@ func (o *openAIEmbedder) textToTokens(text string) ([]int, error) {
 	return tokenizer.Encode(text, nil, nil), nil
 }
 
-func (o *openAIEmbedder) getMaxTokens() int {
+func (o *OpenAIEmbedder) getMaxTokens() int {
 
 	if tiktoken.MODEL_TO_ENCODING[o.model.String()] == "cl100k_base" {
 		return 8191
@@ -22,7 +22,7 @@ func (o *openAIEmbedder) getMaxTokens() int {
 	return 2046
 }
 
-func (o *openAIEmbedder) tokensToText(tokens []int) (string, error) {
+func (o *OpenAIEmbedder) tokensToText(tokens []int) (string, error) {
 	tokenizer, err := tiktoken.EncodingForModel(o.model.String())
 	if err != nil {
 		return "", err

examples/chat/functions/main.go

@@ -12,13 +12,17 @@ import (
 	"github.com/henomis/lingoose/chat"
 	"github.com/henomis/lingoose/llm/openai"
 	"github.com/henomis/lingoose/prompt"
+	"github.com/henomis/lingoose/types"
 )
 
 func main() {
 	fmt.Printf("What's your name?\n> ")
 	reader := bufio.NewReader(os.Stdin)
 	name, _ := reader.ReadString('\n')
 
+	outputToken := 0
+	inputToken := 0
+
 	llmChat := chat.New(
 
 		chat.PromptMessage{
@@ -29,7 +33,16 @@ func main() {
 		},
 	)
 
-	llmOpenAI := openai.New(openai.GPT3Dot5Turbo0613, openai.DefaultOpenAITemperature, openai.DefaultOpenAIMaxTokens, true)
+	llmOpenAI := openai.New(openai.GPT3Dot5Turbo0613, openai.DefaultOpenAITemperature, openai.DefaultOpenAIMaxTokens, true).
+		WithCallback(func(response types.Meta) {
+			for k, v := range response {
+				if k == "CompletionTokens" {
+					outputToken += v.(int)
+				} else if k == "PromptTokens" {
+					inputToken += v.(int)
+				}
+			}
+		})
 
 	llmOpenAI.BindFunction(
 		GetNationalitiesForName,
@@ -62,6 +75,12 @@ func main() {
 		panic(err)
 	}
 
+	fmt.Printf("You used %d tokens (input=%d/output=%d)\n", inputToken+outputToken, inputToken, outputToken)
+
+	inputPrice := float64(inputToken) / 1000 * 0.0015
+	outputPrice := float64(outputToken) / 1000 * 0.002
+	fmt.Printf("You spent $%f\n", inputPrice+outputPrice)
+
 }
 
 type Query struct {

examples/embeddings/llamacpp/main.go

@@ -9,8 +9,8 @@ import (
 
 func main() {
 	llamacppEmbedder := llamacppembedder.New().
-		WithModel("/home/simone/GIT/lingoose/llama.cpp/models/7B/ggml-model-q4_0.bin").
-		WithLlamaCppPath("/home/simone/GIT/lingoose/llama.cpp/embedding")
+		WithModel("./llama.cpp/models/7B/ggml-model-q4_0.bin").
+		WithLlamaCppPath("./llama.cpp/embedding")
 
 	embeddings, err := llamacppEmbedder.Embed(context.Background(), []string{"hello", "world"})
 	if err != nil {

examples/transformer/visual-question-answering/main.go

@@ -9,7 +9,7 @@ import (
 
 func main() {
 
-	d := transformer.NewHFVisualQuestionAnswering("/Users/simone/Documents/gopher.png")
+	d := transformer.NewHFVisualQuestionAnswering("test.png")
 
 	response, err := d.Transform(context.Background(), "is it wearing glasses?", true)
 	if err != nil {

index/pinecone.go

@@ -253,9 +253,9 @@ func (p *Pinecone) createIndexIfRequired(ctx context.Context) error {
 
 func (p *Pinecone) batchUpsert(ctx context.Context, documents []document.Document) error {
 
-	for i := 0; i < len(documents); i += defaultPineconeBatchUpsertSize {
+	for i := 0; i < len(documents); i += p.batchUpsertSize {
 
-		batchEnd := i + defaultPineconeBatchUpsertSize
+		batchEnd := i + p.batchUpsertSize
 		if batchEnd > len(documents) {
 			batchEnd = len(documents)
 		}

llm/huggingface/conversational.go

@@ -37,7 +37,7 @@ type conversation struct {
 	PastUserInputs     []string `json:"past_user_inputs,omitempty"`
 }
 
-func (h *huggingFace) conversationalCompletion(ctx context.Context, prompt string) (string, error) {
+func (h *HuggingFace) conversationalCompletion(ctx context.Context, prompt string) (string, error) {
 
 	isTrue := true
 	request := conversationalRequest{

llm/huggingface/http.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 )
 
-func (h *huggingFace) doRequest(ctx context.Context, jsonBody []byte, model string) ([]byte, error) {
+func (h *HuggingFace) doRequest(ctx context.Context, jsonBody []byte, model string) ([]byte, error) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, APIBaseURL+model, bytes.NewBuffer(jsonBody))
 	if err != nil {