-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* added classifier and sample metric * added comments and updated example description * removed non necessary object
- Loading branch information
Showing
5 changed files
with
95 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/metrics/AnswerAccuracy.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package com.xebia.functional.xef.evaluator.metrics | ||
|
||
import com.xebia.functional.xef.AI | ||
|
||
enum class AnswerAccuracy : AI.PromptClassifier { | ||
yes, | ||
no; | ||
|
||
override fun template(input: String, output: String, context: String): String { | ||
return """| | ||
|You are an expert en evaluating whether the `output` is consistent with the given `input` and `context`. | ||
| <input> | ||
| $input | ||
| </input> | ||
| <output> | ||
| $output | ||
| </output> | ||
| <context> | ||
| $context | ||
| </context> | ||
|Return one of the following: | ||
| - if the answer it's consistent: `yes` | ||
| - if the answer it's not consistent: `no` | ||
""" | ||
.trimMargin() | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
examples/src/main/kotlin/com/xebia/functional/xef/dsl/classify/AnswerAccuracy.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package com.xebia.functional.xef.dsl.classify | ||
|
||
import com.xebia.functional.openai.models.CreateChatCompletionRequestModel | ||
import com.xebia.functional.xef.AI | ||
import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy | ||
|
||
/** | ||
* This is a simple example of how to use the `AI.classify` function to classify the accuracy of an | ||
* answer. In this case, it's using the `AnswerAccuracy` enum class to classify if the answer is | ||
* consistent or not. | ||
* | ||
* You can extend the `AI.PromptClassifier` interface to create your own classification. Override | ||
* the `template` function to define the prompt to be used in the classification. | ||
*/ | ||
suspend fun main() { | ||
println( | ||
AI.classify<AnswerAccuracy>("Do I love Xef?", "I love Xef", "The answer responds the question") | ||
) | ||
println( | ||
AI.classify<AnswerAccuracy>( | ||
input = "Do I love Xef?", | ||
output = "I have three opened PRs", | ||
context = "The answer responds the question", | ||
model = CreateChatCompletionRequestModel.gpt_3_5_turbo_0125 | ||
) | ||
) | ||
} |