Skip to content

Commit

Permalink
add answer relevance to integration test
Browse files Browse the repository at this point in the history
  • Loading branch information
b.nativi committed Jun 26, 2024
1 parent 91b2f6a commit 155d343
Showing 1 changed file with 98 additions and 7 deletions.
105 changes: 98 additions & 7 deletions integration_tests/client/metrics/test_text_generation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def mock_ClientConnection_evaluate(
},
"parameters": {
"task_type": "text-generation",
"metrics_to_return": ["Coherence"],
"metrics_to_return": ["AnswerRelevance", "Coherence"],
"label_map": None,
"metric_params": None,
"convert_annotations_to_type": None,
Expand All @@ -63,6 +63,16 @@ def mock_ClientConnection_evaluate(
"created_at": "2024-06-20T17:27:53.017348Z",
"meta": {"duration": 2.038157},
"metrics": [
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid0",
"prediction": "Based on the provided context, John Adams and Alexander Hamilton did not get along. John Adams, during his presidency, had grown independent of his cabinet, often making decisions despite opposition from it. Hamilton, who was accustomed to being regularly consulted by Washington, sent Adams a detailed letter with policy suggestions after his inauguration, which Adams dismissively ignored.\n",
},
"value": 0.6666666666666666,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand All @@ -73,6 +83,16 @@ def mock_ClientConnection_evaluate(
"value": 4.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid1",
"prediction": "Yes, Lincoln won the election of 1860. He received the highest number of votes and a majority in the Electoral College, making him the 16th President of the United States. However, it's important to note that he won entirely due to his support in the North and West, as he did not receive any votes in 10 of the 15 Southern slave states.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand All @@ -83,6 +103,16 @@ def mock_ClientConnection_evaluate(
"value": 5.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid2",
"prediction": "If a turtle egg was kept warm, it would likely hatch into a baby turtle. The sex of the baby turtle would be determined by the incubation temperature, assuming the species is one of those that determine sex thermally. This is because many turtle species have the ability to move around inside their eggs to select the best temperature for development, which can influence their sexual destiny.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand Down Expand Up @@ -126,7 +156,7 @@ def mock_ClientConnection_get_evaluations(
},
"parameters": {
"task_type": "text-generation",
"metrics_to_return": ["Coherence"],
"metrics_to_return": ["AnswerRelevance", "Coherence"],
"label_map": None,
"metric_params": None,
"convert_annotations_to_type": None,
Expand All @@ -144,6 +174,16 @@ def mock_ClientConnection_get_evaluations(
"created_at": "2024-06-20T17:27:53.017348Z",
"meta": {"duration": 2.038157},
"metrics": [
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid0",
"prediction": "Based on the provided context, John Adams and Alexander Hamilton did not get along. John Adams, during his presidency, had grown independent of his cabinet, often making decisions despite opposition from it. Hamilton, who was accustomed to being regularly consulted by Washington, sent Adams a detailed letter with policy suggestions after his inauguration, which Adams dismissively ignored.\n",
},
"value": 0.6666666666666666,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand All @@ -154,6 +194,16 @@ def mock_ClientConnection_get_evaluations(
"value": 4.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid1",
"prediction": "Yes, Lincoln won the election of 1860. He received the highest number of votes and a majority in the Electoral College, making him the 16th President of the United States. However, it's important to note that he won entirely due to his support in the North and West, as he did not receive any votes in 10 of the 15 Southern slave states.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand All @@ -164,6 +214,16 @@ def mock_ClientConnection_get_evaluations(
"value": 5.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid2",
"prediction": "If a turtle egg was kept warm, it would likely hatch into a baby turtle. The sex of the baby turtle would be determined by the incubation temperature, assuming the species is one of those that determine sex thermally. This is because many turtle species have the ability to move around inside their eggs to select the best temperature for development, which can influence their sexual destiny.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
Expand Down Expand Up @@ -212,7 +272,7 @@ def test_llm_evaluation(

metrics_to_return = [
# MetricType.AnswerCorrectness,
# MetricType.AnswerRelevance,
MetricType.AnswerRelevance,
# MetricType.Bias,
MetricType.Coherence,
# MetricType.ContextPrecision,
Expand Down Expand Up @@ -253,41 +313,72 @@ def test_llm_evaluation(

# TODO Add metric checks as metrics are implemented.
expected_metrics = [
# Coherence results
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid0",
"prediction": PREDICTIONS[0],
"prediction": "Based on the provided context, John Adams and Alexander Hamilton did not get along. John Adams, during his presidency, had grown independent of his cabinet, often making decisions despite opposition from it. Hamilton, who was accustomed to being regularly consulted by Washington, sent Adams a detailed letter with policy suggestions after his inauguration, which Adams dismissively ignored.\n",
},
"value": 0.6666666666666666,
"label": None,
},
{
"type": "Coherence",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid0",
"prediction": "Based on the provided context, John Adams and Alexander Hamilton did not get along. John Adams, during his presidency, had grown independent of his cabinet, often making decisions despite opposition from it. Hamilton, who was accustomed to being regularly consulted by Washington, sent Adams a detailed letter with policy suggestions after his inauguration, which Adams dismissively ignored.\n",
},
"value": 4.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid1",
"prediction": PREDICTIONS[1],
"prediction": "Yes, Lincoln won the election of 1860. He received the highest number of votes and a majority in the Electoral College, making him the 16th President of the United States. However, it's important to note that he won entirely due to his support in the North and West, as he did not receive any votes in 10 of the 15 Southern slave states.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid1",
"prediction": "Yes, Lincoln won the election of 1860. He received the highest number of votes and a majority in the Electoral College, making him the 16th President of the United States. However, it's important to note that he won entirely due to his support in the North and West, as he did not receive any votes in 10 of the 15 Southern slave states.",
},
"value": 5.0,
"label": None,
},
{
"type": "AnswerRelevance",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid2",
"prediction": PREDICTIONS[2],
"prediction": "If a turtle egg was kept warm, it would likely hatch into a baby turtle. The sex of the baby turtle would be determined by the incubation temperature, assuming the species is one of those that determine sex thermally. This is because many turtle species have the ability to move around inside their eggs to select the best temperature for development, which can influence their sexual destiny.",
},
"value": 0.2,
"label": None,
},
{
"type": "Coherence",
"parameters": {
"dataset": "test_dataset",
"datum_uid": "uid2",
"prediction": "If a turtle egg was kept warm, it would likely hatch into a baby turtle. The sex of the baby turtle would be determined by the incubation temperature, assuming the species is one of those that determine sex thermally. This is because many turtle species have the ability to move around inside their eggs to select the best temperature for development, which can influence their sexual destiny.",
},
"value": 4.0,
"label": None,
},
]

# Check that returned metrics have the right format.
for m in metrics:
if m["type"] == "AnswerRelevance":
assert 0 <= m["value"] <= 1
if m["type"] == "Coherence":
assert m["value"] in [1, 2, 3, 4, 5]

Expand Down

0 comments on commit 155d343

Please sign in to comment.