Skip to content

Commit

Permalink
chore: add golden dataset for eval (#411)
Browse files Browse the repository at this point in the history 
Adding golden datasets that will be used in llm system evaluation.

The golden dataset is separated into multiple types of queries:
- queries that uses a specific tool
- airline related queries (no tool calling - answer is within prompt)
- assistant related question (no tool calling - answer is within prompt)
- out of context questions (no tool calling)
- multitool selections (agent selecting multiple tool before returning a
final answer to the user)
  • Loading branch information
@Yuan325
Yuan325 authored Jul 11, 2024
1 parent 6987280 commit cf071b8
Showing 1 changed file with 288 additions and 0 deletions.
288 changes: 288 additions & 0 deletions llm_demo/evaluation/eval_golden.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,288 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Any, Dict, List, Optional

from pydantic import BaseModel, Field


class ToolCall(BaseModel):
name: str
arguments: Dict[str, Any] = Field(
default={}, description="query arguments for tool call"
)


class EvalData(BaseModel):
category: Optional[str] = Field(default=None, description="evaluation category")
query: Optional[str] = Field(default=None, description="user query")
instruction: Optional[str] = Field(
default=None, description="instruction to llm system"
)
content: Optional[str] = Field(default=None)
tool_calls: Optional[List[ToolCall]] = Field(default=None)
context: Optional[str] = Field(
default=None, description="context given to llm in order to answer user query"
)
output: Optional[str] = Field(default=None)
prediction_tool_calls: Optional[List[ToolCall]] = Field(default=None)
prediction_output: Optional[str] = Field(default=None)
reset: bool = Field(
default=True, description="determine to reset the chat after invoke"
)


goldens = [
EvalData(
category="Search Airport Tool",
query="What is the airport located in San Francisco?",
tool_calls=[
ToolCall(
name="Search Airport",
arguments={"city": "San Francisco"},
),
],
),
EvalData(
category="Search Airport Tool",
query="Tell me more about Denver International Airport?",
tool_calls=[
ToolCall(
name="Search Airport",
arguments={
"city": "Denver",
"name": "Denver International Airport",
},
),
],
),
EvalData(
category="Search Flights By Flight Number Tool",
query="What is the departure gate for flight CY 922?",
tool_calls=[
ToolCall(
name="Search Flights By Flight Number",
arguments={
"airline": "CY",
"flight_number": "922",
},
),
],
),
EvalData(
category="Search Flights By Flight Number Tool",
query="What is flight CY 888 flying to?",
tool_calls=[
ToolCall(
name="Search Flights By Flight Number",
arguments={
"airline": "CY",
"flight_number": "888",
},
),
],
),
EvalData(
category="List Flights Tool",
query="What flights are headed to JFK tomorrow?",
tool_calls=[
ToolCall(
name="List Flights",
arguments={
"arrival_airport": "JFK",
"date": "2023-01-02",
},
),
],
),
EvalData(
category="List Flights Tool",
query="Is there any flight from SFO to DEN?",
output="I will need the date to retrieve relevant flights.",
),
EvalData(
category="Search Amenities Tool",
query="Are there any luxury shops?",
tool_calls=[
ToolCall(
name="Search Amenities",
arguments={
"query": "luxury shops",
},
),
],
),
EvalData(
category="Search Amenities Tool",
query="Where can I get coffee near gate A6?",
tool_calls=[
ToolCall(
name="Search Amenities",
arguments={
"query": "coffee near gate A6",
},
),
],
),
EvalData(
category="Search Policies Tool",
query="What is the flight cancellation policy?",
tool_calls=[
ToolCall(
name="Search Policies",
arguments={
"query": "flight cancellation policy",
},
),
],
),
EvalData(
category="Search Policies Tool",
query="How many checked bags can I bring?",
tool_calls=[
ToolCall(
name="Search Policies",
arguments={
"query": "checked bags",
},
),
],
),
EvalData(
category="Insert Ticket",
query="I would like to book flight CY 888 departing from SFO on 2024-01-01 at 6am.",
tool_calls=[
ToolCall(
name="Insert Ticket",
arguments={
"airline": "CY",
"flight_number": "888",
"departure_airport": "SFO",
"departure_time": "2024-01-01 06:00:00",
},
),
],
),
EvalData(
category="Insert Ticket",
query="What flights are headed from SFO to DEN today?",
tool_calls=[
ToolCall(
name="List Flights",
arguments={
"departure_airport": "SFO",
"arrival_airport": "DEN",
"date": "2024-01-01",
},
),
],
reset=False,
),
EvalData(
category="Insert Ticket",
query="I would like to book the first flight.",
tool_calls=[
ToolCall(
name="Insert Ticket",
arguments={
"airline": "CY",
"flight_number": "888",
"departure_airport": "SFO",
"arrival_airport": "DEN",
"departure_time": "2024-01-01 05:00:00",
"arrival_time": "2024-01-01 08:00:00",
},
),
],
),
EvalData(
category="List Tickets",
query="Do I have any tickets?",
tool_calls=[ToolCall(name="List Tickets")],
),
EvalData(
category="List Tickets",
query="When is my next flight?",
tool_calls=[ToolCall(name="List Tickets")],
),
EvalData(
category="Airline Related Question",
query="What is Cymbal Air?",
output="Cymbal Air is a passenger airline offering convenient flights to many cities around the world from its hub in San Francisco.",
),
EvalData(
category="Airline Related Question",
query="Where is the hub of cymbal air?",
output="The hub of Cymbal Air is in San Francisco.",
),
EvalData(
category="Assistant Related Question",
query="What can you help me with?",
output="I can help to book flights and answer a wide range of questions pertaining to travel on Cymbal Air, as well as amenities of San Francisco Airport.",
),
EvalData(
category="Assistant Related Question",
query="Can you help me book tickets?",
output="Yes, I can help with several tools such as search airports, list tickets, book tickets.",
),
EvalData(
category="Out-Of-Context Question",
query="Can you help me solve math problems?",
output="Sorry, I am not given the tools for this.",
),
EvalData(
category="Out-Of-Context Question",
query="Who is the CEO of Google?",
output="Sorry, I am not given the tools for this.",
),
EvalData(
category="Multitool Selections",
query="Where can I get a snack near the gate for flight CY 352?",
tool_calls=[
ToolCall(
name="Search Flights By Flight Number",
arguments={
"airline": "CY",
"flight_number": "352",
},
),
ToolCall(
name="Search Amenities",
arguments={
"query": "snack near gate A2.",
},
),
],
),
EvalData(
category="Multitool Selections",
query="What are some flights from SFO to Chicago tomorrow?",
tool_calls=[
ToolCall(
name="Search Airport",
arguments={
"city": "Chicago",
},
),
ToolCall(
name="List Flights",
arguments={
"departure_airport": "SFO",
"arrival_airport": "ORD",
"date": "2024-01-02",
},
),
],
),
]

0 comments on commit cf071b8

Please sign in to comment.