From 2227379e4f6883d9acf0c2ec8fb2ce09ad2d737a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20P=C3=A9rez=20Pacheco?= Date: Tue, 30 Apr 2024 20:05:25 +0200 Subject: [PATCH 1/5] Tokens in Evaluator Tests (#730) --- .../com/xebia/functional/xef/llm/Chat.kt | 57 +++++++++--- .../functional/xef/llm/models/Messages.kt | 18 ++++ .../functional/xef/evaluator/SuiteBuilder.kt | 1 + .../functional/xef/evaluator/models/Html.kt | 15 ++-- .../xef/evaluator/models/ItemResult.kt | 1 + .../xef/evaluator/models/Markdown.kt | 10 +++ .../xef/evaluator/models/TestModels.kt | 28 +++++- evaluator/src/main/resources/web/index.html | 13 --- evaluator/src/main/resources/web/script.js | 65 -------------- evaluator/src/main/resources/web/style.css | 87 ------------------- .../functional/xef/evaluator/TestExample.kt | 21 +++-- 11 files changed, 123 insertions(+), 193 deletions(-) create mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt delete mode 100644 evaluator/src/main/resources/web/index.html delete mode 100644 evaluator/src/main/resources/web/script.js delete mode 100644 evaluator/src/main/resources/web/style.css diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 3dc16adb8..60ab52f26 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -2,11 +2,17 @@ package com.xebia.functional.xef.llm import com.xebia.functional.openai.generated.api.Chat import com.xebia.functional.openai.generated.model.CreateChatCompletionRequest +import com.xebia.functional.openai.generated.model.CreateChatCompletionResponse +import com.xebia.functional.openai.generated.model.CreateChatCompletionResponseChoicesInner import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.llm.models.MessageWithUsage +import com.xebia.functional.xef.llm.models.MessagesUsage +import com.xebia.functional.xef.llm.models.MessagesWithUsage import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.PromptBuilder +import com.xebia.functional.xef.store.Memory import kotlinx.coroutines.flow.* @AiDsl @@ -54,9 +60,34 @@ suspend fun Chat.promptMessage(prompt: Prompt, scope: Conversation = Conversatio suspend fun Chat.promptMessages( prompt: Prompt, scope: Conversation = Conversation() -): List = +): List = promptResponse(prompt, scope) { it.message.content }.first + +@AiDsl +suspend fun Chat.promptMessageAndUsage( + prompt: Prompt, + scope: Conversation = Conversation() +): MessageWithUsage { + val response = promptMessagesAndUsage(prompt, scope) + val message = response.messages.firstOrNull() ?: throw AIError.NoResponse() + return MessageWithUsage(message, response.usage) +} + +@AiDsl +suspend fun Chat.promptMessagesAndUsage( + prompt: Prompt, + scope: Conversation = Conversation() +): MessagesWithUsage { + val response = promptResponse(prompt, scope) { it.message.content } + return MessagesWithUsage(response.first, response.second.usage?.let { MessagesUsage(it) }) +} + +private suspend fun Chat.promptResponse( + prompt: Prompt, + scope: Conversation = Conversation(), + block: suspend Chat.(CreateChatCompletionResponseChoicesInner) -> T? +): Pair, CreateChatCompletionResponse> = scope.metric.promptSpan(prompt) { - val promptMemories = prompt.messages.toMemory(scope) + val promptMemories: List = prompt.messages.toMemory(scope) val adaptedPrompt = PromptCalculator.adaptPromptToConversationAndModel(prompt, scope) adaptedPrompt.addMetrics(scope) @@ -72,13 +103,17 @@ suspend fun Chat.promptMessages( seed = adaptedPrompt.configuration.seed, ) - createChatCompletion(request) - .addMetrics(scope) - .choices - .addChoiceToMemory( - scope, - promptMemories, - prompt.configuration.messagePolicy.addMessagesToConversation - ) - .mapNotNull { it.message.content } + val createResponse: CreateChatCompletionResponse = createChatCompletion(request) + Pair( + createResponse + .addMetrics(scope) + .choices + .addChoiceToMemory( + scope, + promptMemories, + prompt.configuration.messagePolicy.addMessagesToConversation + ) + .mapNotNull { block(it) }, + createResponse + ) } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt new file mode 100644 index 000000000..1817d3f16 --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt @@ -0,0 +1,18 @@ +package com.xebia.functional.xef.llm.models + +import com.xebia.functional.openai.generated.model.CompletionUsage + +data class MessagesWithUsage(val messages: List, val usage: MessagesUsage?) + +data class MessageWithUsage(val message: String, val usage: MessagesUsage?) + +data class MessagesUsage(val completionTokens: Int, val promptTokens: Int, val totalTokens: Int) { + companion object { + operator fun invoke(usage: CompletionUsage) = + MessagesUsage( + completionTokens = usage.completionTokens, + promptTokens = usage.promptTokens, + totalTokens = usage.totalTokens + ) + } +} diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt index 1df855b86..94a33497b 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt @@ -44,6 +44,7 @@ data class SuiteSpec( output.description.value, item.context, output.value, + output.tokens, classification, success.contains(classification) ) diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt index d4eecb719..a2b1c377c 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt @@ -56,10 +56,12 @@ value class Html(val value: String) { const outputDiv = document.createElement('pre'); outputDiv.classList.add('output'); outputDiv.innerText = 'Output: ' + test.output; - outputDiv.addEventListener('click', function() { - this.classList.toggle('expanded'); - }); blockDiv.appendChild(outputDiv); + + const usageDiv = document.createElement('pre'); + usageDiv.classList.add('output'); + usageDiv.innerText = 'Usage: \n Completion Tokens: ' + test.usage?.completionTokens + '\n Prompt Tokens: ' + test.usage?.promptTokens + '\n Total Tokens: ' + test.usage?.totalTokens; + blockDiv.appendChild(usageDiv); const result = document.createElement('div'); result.classList.add('score', test.success ? 'score-passed' : 'score-failed'); @@ -123,16 +125,11 @@ value class Html(val value: String) { .output { color: #666; - cursor: pointer; - white-space: nowrap; + white-space: normal; overflow: hidden; text-overflow: ellipsis; } - .output.expanded { - white-space: normal; - } - .score { font-weight: bold; } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt index f78d6646a..19e2295e2 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt @@ -21,6 +21,7 @@ data class OutputResult( val description: String, val contextDescription: String, val output: String, + val usage: OutputTokens?, val result: E, val success: Boolean ) where E : AI.PromptClassifier, E : Enum diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt index 40a275b11..a998f08e2 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt @@ -28,6 +28,16 @@ value class Markdown(val value: String) { |
|${outputResult.output} |
+ |- Usage: + |
+ |${outputResult.usage?.let { usage -> + """ + |Completion Tokens: ${usage.completionTokens} + |Prompt Tokens: ${usage.promptTokens} + |Total Tokens: ${usage.totalTokens} + """.trimMargin() + } ?: "No usage information available"} + |
| |Result: ${if (outputResult.success) "✅ Success" else "❌ Failure"} (${outputResult.result}) """.trimMargin() diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt index 77f6157bb..ca9160674 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt @@ -1,17 +1,39 @@ package com.xebia.functional.xef.evaluator.models +import com.xebia.functional.xef.llm.models.MessageWithUsage +import com.xebia.functional.xef.llm.models.MessagesUsage import kotlin.jvm.JvmSynthetic import kotlinx.serialization.Serializable @Serializable data class OutputDescription(val value: String) @Serializable -data class OutputResponse(val description: OutputDescription, val value: String) { +data class OutputResponse( + val description: OutputDescription, + val tokens: OutputTokens?, + val value: String +) { companion object { @JvmSynthetic suspend operator fun invoke( description: OutputDescription, - block: suspend () -> String - ): OutputResponse = OutputResponse(description, block()) + block: suspend () -> MessageWithUsage + ): OutputResponse { + val response = block() + return OutputResponse(description, response.usage?.let { OutputTokens(it) }, response.message) + } + } +} + +@Serializable +data class OutputTokens( + val promptTokens: Int? = null, + val completionTokens: Int? = null, + val totalTokens: Int? = null +) { + companion object { + @JvmSynthetic + operator fun invoke(usage: MessagesUsage): OutputTokens = + OutputTokens(usage.promptTokens, usage.completionTokens, usage.totalTokens) } } diff --git a/evaluator/src/main/resources/web/index.html b/evaluator/src/main/resources/web/index.html deleted file mode 100644 index 2a3718117..000000000 --- a/evaluator/src/main/resources/web/index.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - Tests - - - - - -
- - diff --git a/evaluator/src/main/resources/web/script.js b/evaluator/src/main/resources/web/script.js deleted file mode 100644 index 35dc55966..000000000 --- a/evaluator/src/main/resources/web/script.js +++ /dev/null @@ -1,65 +0,0 @@ -document.addEventListener('DOMContentLoaded', function() { - - const container = document.getElementById('test-container'); - - const headerDiv = document.createElement('div'); - headerDiv.classList.add('test-block'); - - const header = document.createElement('h1'); - header.classList.add('test-header'); - header.textContent = "Suite test"; - - const suiteDescription = document.createElement('p'); - suiteDescription.textContent = 'Description: ' + testData.description; - - const model = document.createElement('p'); - model.textContent = 'Model: ' + testData.model; - - const metric = document.createElement('p'); - metric.textContent = 'Metric: ' + testData.metric; - - headerDiv.appendChild(header); - headerDiv.appendChild(suiteDescription); - headerDiv.appendChild(model); - headerDiv.appendChild(metric); - - container.appendChild(headerDiv); - - testData.items.forEach(block => { - const blockDiv = document.createElement('div'); - blockDiv.classList.add('test-block'); - - const title = document.createElement('h2'); - title.classList.add('test-title'); - title.textContent = 'Input: ' + block.description; - - blockDiv.appendChild(title); - - block.items.forEach(test => { - const itemDescription = document.createElement('div'); - itemDescription.textContent = 'Description: ' + test.description; - blockDiv.appendChild(itemDescription); - - const context = document.createElement('div'); - context.textContent = 'Context: ' + test.contextDescription; - blockDiv.appendChild(context); - - const outputDiv = document.createElement('pre'); - outputDiv.classList.add('output'); - outputDiv.innerText = 'Output: ' + test.output; - outputDiv.addEventListener('click', function() { - this.classList.toggle('expanded'); - }); - blockDiv.appendChild(outputDiv); - - const result = document.createElement('div'); - result.classList.add('score', test.success ? 'score-passed' : 'score-failed'); - result.textContent = 'Result: ' + test.result; - blockDiv.appendChild(result); - - blockDiv.appendChild(document.createElement('br')); - }); - container.appendChild(blockDiv); - }); - -}); diff --git a/evaluator/src/main/resources/web/style.css b/evaluator/src/main/resources/web/style.css deleted file mode 100644 index a14683826..000000000 --- a/evaluator/src/main/resources/web/style.css +++ /dev/null @@ -1,87 +0,0 @@ -body { - font-family: Arial, sans-serif; - margin: 0; - padding: 0; - background-color: #f4f4f4; -} - -#test-container { - width: 80%; - margin: 20px auto; - padding: 15px; - background-color: white; - border-radius: 8px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); -} - -.test-block { - margin-bottom: 20px; - border-bottom: 1px solid #eee; - padding-bottom: 20px; -} - -.test-title { - font-size: 1.2em; - color: #333; -} - -.input, .output { - margin: 5px 0; -} - -.input-passed { - margin-top: 25px; - color: green; - font-weight: bold; -} - -.input-failed { - margin-top: 25px; - color: red; - font-weight: bold; -} - -.output { - color: #666; - cursor: pointer; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} - -.output.expanded { - white-space: normal; -} - -.score { - font-weight: bold; -} - -.score-passed { - margin-bottom: 25px; - color: #008000; -} - -.score-failed { - margin-bottom: 25px; - color: red; -} - -.avg-score, .test-info { - font-size: 1.2em; - color: #d35400; - margin-top: 10px; -} - -.test-summary { - background-color: #e7e7e7; - padding: 15px; - margin-top: 20px; - border-radius: 8px; -} - -.test-summary h3 { - font-size: 1.1em; - color: #555; - margin-top: 0; -} diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt index fb8ce532b..d51a6710d 100644 --- a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt +++ b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt @@ -7,9 +7,10 @@ import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy import com.xebia.functional.xef.evaluator.models.OutputDescription import com.xebia.functional.xef.evaluator.models.OutputResponse -import com.xebia.functional.xef.llm.promptMessage +import com.xebia.functional.xef.llm.promptMessageAndUsage import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.PromptBuilder.Companion.user +import java.io.File object TestExample { @@ -31,10 +32,10 @@ object TestExample { context = "Contains information about a movie" ) { +OutputResponse(gpt35Description) { - Conversation { chat.promptMessage(Prompt(model) { +user(input) }) } + Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } } - +OutputResponse(description = fakeOutputs, value = "I don't know") + +OutputResponse(description = fakeOutputs, null, value = "I don't know") } +ItemSpec( @@ -42,13 +43,22 @@ object TestExample { context = "Contains instructions for making a cake" ) { +OutputResponse(gpt35Description) { - Conversation { chat.promptMessage(Prompt(model) { +user(input) }) } + Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } } - +OutputResponse(description = fakeOutputs, value = "The movie is Jurassic Park") + +OutputResponse(description = fakeOutputs, null, value = "The movie is Jurassic Park") } } val results = spec.evaluate(success = listOf(AnswerAccuracy.yes)) + + val outputPath = System.getProperty("user.dir") + "/build/testSuite" + File(outputPath).mkdir() + val fileHtml = File("$outputPath/test.html") + fileHtml.writeText(SuiteSpec.toHtml(results, "test.html").value) + + val fileMarkDown = File("$outputPath/test.md") + fileMarkDown.writeText(SuiteSpec.toMarkdown(results, "test.md").value) + results.items.forEach { println("==============") println(" ${it.description}") @@ -57,6 +67,7 @@ object TestExample { println() println(">> Output ${index + 1}") println("Description: ${item.description}") + println("Usage: ${item.usage}") println("Success: ${item.success}") println() println("AI Output:") From e1b68076a1ef7f3ee9505c9ca35601f41eb6bfbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20P=C3=A9rez=20Pacheco?= Date: Thu, 2 May 2024 18:44:06 +0200 Subject: [PATCH 2/5] Estimate price in Evaluator Tests (#731) --- .../functional/xef/evaluator/SuiteBuilder.kt | 7 ++- .../functional/xef/evaluator/models/Html.kt | 14 ++++-- .../xef/evaluator/models/Markdown.kt | 7 ++- .../xef/evaluator/models/ModelsPricing.kt | 44 +++++++++++++++++++ .../xef/evaluator/models/TestModels.kt | 33 ++++++++++++-- .../functional/xef/evaluator/TestExample.kt | 5 ++- 6 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt index 94a33497b..ff85f9e95 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt @@ -68,10 +68,9 @@ data class SuiteSpec( E : Enum = Html.get(Json.encodeToString(SuiteResults.serializer(serializer()), result), suiteName) - inline fun toMarkdown( - result: SuiteResults, - suiteName: String, - ): Markdown where E : AI.PromptClassifier, E : Enum = Markdown.get(result, suiteName) + inline fun toMarkdown(result: SuiteResults, suiteName: String): Markdown where + E : AI.PromptClassifier, + E : Enum = Markdown.get(result, suiteName) } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt index a2b1c377c..b2514dfbb 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt @@ -58,10 +58,12 @@ value class Html(val value: String) { outputDiv.innerText = 'Output: ' + test.output; blockDiv.appendChild(outputDiv); - const usageDiv = document.createElement('pre'); - usageDiv.classList.add('output'); - usageDiv.innerText = 'Usage: \n Completion Tokens: ' + test.usage?.completionTokens + '\n Prompt Tokens: ' + test.usage?.promptTokens + '\n Total Tokens: ' + test.usage?.totalTokens; - blockDiv.appendChild(usageDiv); + if (test.usage != undefined) { + const usageDiv = document.createElement('pre'); + usageDiv.classList.add('output'); + usageDiv.innerText = 'Usage: \n Prompt Tokens: ' + test.usage?.promptTokens + ' (~' + test.usage?.estimatePricePerToken + ' ' + test.usage?.currency + ')\n Completion Tokens: ' + test.usage?.completionTokens + ' (~' + test.usage?.estimatePriceCompletionToken + ' ' + test.usage?.currency + ')\n Total Tokens: ' + test.usage?.totalTokens + '\n Total Price: ~' + test.usage?.estimatePriceTotalToken + ' ' + test.usage?.currency; + blockDiv.appendChild(usageDiv); + } const result = document.createElement('div'); result.classList.add('score', test.success ? 'score-passed' : 'score-failed'); @@ -101,6 +103,10 @@ value class Html(val value: String) { border-bottom: 1px solid #eee; padding-bottom: 20px; } + + .test-block pre { + margin-bottom: 20px; + } .test-title { font-size: 1.2em; diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt index a998f08e2..92baaaf7b 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt @@ -32,9 +32,10 @@ value class Markdown(val value: String) { |
|${outputResult.usage?.let { usage -> """ - |Completion Tokens: ${usage.completionTokens} - |Prompt Tokens: ${usage.promptTokens} + |Prompt Tokens: ${usage.promptTokens} ${usage.estimatePricePerToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" } + |Completion Tokens: ${usage.completionTokens} ${usage.estimatePriceCompletionToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" } |Total Tokens: ${usage.totalTokens} + |Total Price: ${usage.estimatePriceTotalToken?.let { "${it.to2DecimalsString()} ${usage.currency ?: ""}" } ?: "Unknown"} """.trimMargin() } ?: "No usage information available"} |
@@ -50,5 +51,7 @@ value class Markdown(val value: String) { .trimMargin() return Markdown(content) } + + private fun Double.to2DecimalsString() = String.format("%.6f", this) } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt new file mode 100644 index 000000000..124148c84 --- /dev/null +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt @@ -0,0 +1,44 @@ +package com.xebia.functional.xef.evaluator.models + +data class ModelsPricing( + val modelName: String, + val currency: String, + val input: ModelsPricingItem, + val output: ModelsPricingItem +) { + + companion object { + + const val oneMillion = 1_000_000 + val oneThousand = 1_000 + + // The pricing for the models was updated the May 2st, 2024 + // Be sure to update the pricing for each model + + val gpt4Turbo = + ModelsPricing( + modelName = "gpt-4-turbo", + currency = "USD", + input = ModelsPricingItem(10.0, oneMillion), + output = ModelsPricingItem(30.0, oneMillion) + ) + + val gpt4 = + ModelsPricing( + modelName = "gpt-4-turbo", + currency = "USD", + input = ModelsPricingItem(30.0, oneMillion), + output = ModelsPricingItem(60.0, oneMillion) + ) + + val gpt3_5Turbo = + ModelsPricing( + modelName = "gpt-3.5-turbo", + currency = "USD", + input = ModelsPricingItem(0.5, oneMillion), + output = ModelsPricingItem(1.5, oneMillion) + ) + } +} + +data class ModelsPricingItem(val price: Double, val perTokens: Int) diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt index ca9160674..ccdcff017 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt @@ -17,10 +17,15 @@ data class OutputResponse( @JvmSynthetic suspend operator fun invoke( description: OutputDescription, + price: ModelsPricing?, block: suspend () -> MessageWithUsage ): OutputResponse { val response = block() - return OutputResponse(description, response.usage?.let { OutputTokens(it) }, response.message) + return OutputResponse( + description, + response.usage?.let { OutputTokens(it, price) }, + response.message + ) } } } @@ -28,12 +33,32 @@ data class OutputResponse( @Serializable data class OutputTokens( val promptTokens: Int? = null, + val estimatePricePerToken: Double? = null, val completionTokens: Int? = null, - val totalTokens: Int? = null + val estimatePriceCompletionToken: Double? = null, + val totalTokens: Int? = null, + val estimatePriceTotalToken: Double? = null, + val currency: String? ) { companion object { @JvmSynthetic - operator fun invoke(usage: MessagesUsage): OutputTokens = - OutputTokens(usage.promptTokens, usage.completionTokens, usage.totalTokens) + operator fun invoke(usage: MessagesUsage, price: ModelsPricing?): OutputTokens { + val estimateInputPrice = + price?.let { usage.promptTokens.let { (it * price.input.price) / price.input.perTokens } } + val estimateOutputPrice = + price?.let { + usage.completionTokens.let { (it * price.output.price) / price.output.perTokens } + } + val estimateTotalPrice = estimateInputPrice?.plus(estimateOutputPrice ?: 0.0) + return OutputTokens( + usage.promptTokens, + estimateInputPrice, + usage.completionTokens, + estimateOutputPrice, + usage.totalTokens, + estimateTotalPrice, + price?.currency + ) + } } } diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt index d51a6710d..999a6b8da 100644 --- a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt +++ b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt @@ -5,6 +5,7 @@ import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestMo import com.xebia.functional.xef.OpenAI import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy +import com.xebia.functional.xef.evaluator.models.ModelsPricing import com.xebia.functional.xef.evaluator.models.OutputDescription import com.xebia.functional.xef.evaluator.models.OutputResponse import com.xebia.functional.xef.llm.promptMessageAndUsage @@ -31,7 +32,7 @@ object TestExample { input = "Please provide a movie title, genre and director", context = "Contains information about a movie" ) { - +OutputResponse(gpt35Description) { + +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) { Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } } @@ -42,7 +43,7 @@ object TestExample { input = "Recipe for a chocolate cake", context = "Contains instructions for making a cake" ) { - +OutputResponse(gpt35Description) { + +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) { Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } } From 6a3fa589edd1ea0561696217625e92a38b1e39c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Monta=C3=B1ez?= Date: Thu, 2 May 2024 22:23:36 +0200 Subject: [PATCH 3/5] Enum description in tools (#732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * support enum descriptions * added example --------- Co-authored-by: José Carlos Montañez Co-authored-by: Raúl Raja Martínez --- .../xef/llm/models/functions/JsonSchema.kt | 35 +++++++++--------- .../xef/assistants/ToolsWithDescriptions.kt | 36 +++++++++++++++++++ 2 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 examples/src/main/kotlin/com/xebia/functional/xef/assistants/ToolsWithDescriptions.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt index 573caa7a4..e7211c2aa 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt @@ -307,6 +307,7 @@ private fun SerialDescriptor.createJsonSchema( } } +@OptIn(ExperimentalSerializationApi::class) private fun JsonObjectBuilder.applyJsonSchemaDefaults( descriptor: SerialDescriptor, annotations: List, @@ -322,25 +323,25 @@ private fun JsonObjectBuilder.applyJsonSchemaDefaults( } } - if (descriptor.kind == SerialKind.ENUM) { - this["enum"] = descriptor.elementNames - } - - if (annotations.isNotEmpty()) { - val multiplatformDescription = annotations.filterIsInstance() - val description = - if (multiplatformDescription.isEmpty()) { - try { - val jvmDescription = annotations.filterIsInstance() - jvmDescription.firstOrNull()?.value - } catch (e: Throwable) { - null + val additionalEnumDescription: String? = + if (descriptor.kind == SerialKind.ENUM) { + this["enum"] = descriptor.elementNames + descriptor.elementNames + .mapIndexed { index, name -> + "$name (${descriptor.getElementAnnotations(index).lastOfInstance()?.value})" } - } else { - multiplatformDescription.firstOrNull()?.value - } + .joinToString("\n - ") + } else null - this["description"] = description + if (annotations.isNotEmpty()) { + val description = annotations.filterIsInstance().firstOrNull()?.value + if (additionalEnumDescription != null) { + this["description"] = "$description\n - $additionalEnumDescription" + } else { + this["description"] = description + } + } else if (additionalEnumDescription != null) { + this["description"] = " - $additionalEnumDescription" } } diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/assistants/ToolsWithDescriptions.kt b/examples/src/main/kotlin/com/xebia/functional/xef/assistants/ToolsWithDescriptions.kt new file mode 100644 index 000000000..cb9a573e0 --- /dev/null +++ b/examples/src/main/kotlin/com/xebia/functional/xef/assistants/ToolsWithDescriptions.kt @@ -0,0 +1,36 @@ +package com.xebia.functional.xef.assistants + +import com.xebia.functional.xef.conversation.Description +import com.xebia.functional.xef.llm.assistants.Assistant +import com.xebia.functional.xef.llm.assistants.AssistantThread +import com.xebia.functional.xef.llm.assistants.RunDelta +import com.xebia.functional.xef.llm.assistants.Tool +import kotlinx.serialization.Serializable + +@Description("Natural numbers") +enum class NaturalWithDescriptions { + @Description("If the number is positive.") POSITIVE, + @Description("If the number is negative.") NEGATIVE +} + +@Serializable +data class SumInputWithDescription( + @Description("Left operand") val left: Int, + @Description("Right operand") val right: Int, + val natural: NaturalWithDescriptions +) + +class SumToolWithDescription : Tool { + override suspend fun invoke(input: SumInputWithDescription): Int { + return input.left + input.right + } +} + +suspend fun main() { + val toolConfig = Tool.toolOf(SumToolWithDescription()).functionObject + println(toolConfig.parameters) +} + +private suspend fun runAssistantAndDisplayResults(thread: AssistantThread, assistant: Assistant) { + thread.run(assistant).collect(RunDelta::printEvent) +} From 5fb134f50dc5fc03ff778abd535389ddb54c4389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Raja=20Mart=C3=ADnez?= Date: Thu, 2 May 2024 23:23:46 +0200 Subject: [PATCH 4/5] Update README with instruction to build locally (#725) * Add README instructions for building Xef * Include reasons why build may fail if you don't have docker --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 2b88706ad..64cdaa6e1 100644 --- a/README.md +++ b/README.md @@ -76,3 +76,17 @@ In [this](https://xef.ai/learn/quickstart/) small introduction we look at the ma ## 🚀 Examples You can also have a look at the [examples](https://github.com/xebia-functional/xef/tree/main/examples/src/main/kotlin/com/xebia/functional/xef/conversation) to have a feeling of how using the library looks like. + +## 🚧 Local Development + +To build the project locally, you can use the following commands: + +```shell +./gradlew downloadOpenAIAPI +./gradlew openaiClientGenerate +./gradlew build +``` + +The server and postgres tests may fail if you don't have [Docker](https://www.docker.com/) installed. +The server and postgres related tests depend on [Testcontainers](https://testcontainers.com/), which in turn depends on Docker. + From 1aef2237fa5f10748bcabd8947adc9cb12b2532f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Carlos=20Monta=C3=B1ez?= Date: Mon, 6 May 2024 16:31:39 +0200 Subject: [PATCH 5/5] fixed error in enum description (#733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Carlos Montañez --- .../xef/llm/models/functions/JsonSchema.kt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt index e7211c2aa..6fd44efd1 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt @@ -327,15 +327,21 @@ private fun JsonObjectBuilder.applyJsonSchemaDefaults( if (descriptor.kind == SerialKind.ENUM) { this["enum"] = descriptor.elementNames descriptor.elementNames - .mapIndexed { index, name -> - "$name (${descriptor.getElementAnnotations(index).lastOfInstance()?.value})" + .mapIndexedNotNull { index, name -> + val enumDescription = + descriptor.getElementAnnotations(index).lastOfInstance()?.value + if (enumDescription != null) { + "$name ($enumDescription)" + } else { + null + } } .joinToString("\n - ") } else null if (annotations.isNotEmpty()) { val description = annotations.filterIsInstance().firstOrNull()?.value - if (additionalEnumDescription != null) { + if (!additionalEnumDescription.isNullOrEmpty()) { this["description"] = "$description\n - $additionalEnumDescription" } else { this["description"] = description