There are a few things to keep in mind while improving your system model:
1. Few shot prompting - Providing the model some predefined correct output samples (user-assistant message pair) so that it has an idea of what kind of output it should come up with. For example:
few_shot_user_1 = """I want the most expensive computer."""
few_shot_assistant_1 = """
[{
'category': 'Computers and Laptops',
'products': ['TechPro Ultrabook', 'BlueWave Gaming Laptop', 'PowerLite Convertible', 'TechPro Desktop', 'BlueWave Chromebook']
}]
"""
messages = [
{'role':'system', 'content': system_message},
{'role':'user', 'content': f"{delimiter}{few_shot_user_1}{delimiter}"},
{'role':'assistant', 'content': few_shot_assistant_1 },
{'role':'user', 'content': f"{delimiter}{user_input}{delimiter}"},
] 2. Regression Testing - If you modify your model in case it fails on some user prmomt, then make sure to check that modifying the model to fix the hard test cases does not negatively affect its performance on previous test cases.
3. Automated Testing for Development - You can do this by providing a set of sample ideal user prompt - assistant response pairs like below:
msg_ideal_pairs_set = [
# eg 0
{'customer_msg':"""Which TV can I buy if I'm on a budget?""",
'ideal_answer':{
'Televisions and Home Theater Systems':set(
['CineView 4K TV', 'SoundMax Home Theater', 'CineView 8K TV', 'SoundMax Soundbar', 'CineView OLED TV']
)}
},
# eg 1
{'customer_msg':"""I need a charger for my smartphone""",
'ideal_answer':{
'Smartphones and Accessories':set(
['MobiTech PowerCase', 'MobiTech Wireless Charger', 'SmartX EarBuds']
)}
},
# eg 2
{'customer_msg':f"""What computers do you have?""",
'ideal_answer':{
'Computers and Laptops':set(
['TechPro Ultrabook', 'BlueWave Gaming Laptop', 'PowerLite Convertible', 'TechPro Desktop', 'BlueWave Chromebook'
])
}
},
# eg 4
{'customer_msg':"""tell me about the CineView TV, the 8K one, Gamesphere console, the X one.
I'm on a budget, what computers do you have?""",
'ideal_answer':{
'Televisions and Home Theater Systems':set(
['CineView 8K TV']),
'Gaming Consoles and Accessories':set(
['GameSphere X']),
'Computers and Laptops':set(
['TechPro Ultrabook', 'BlueWave Gaming Laptop', 'PowerLite Convertible', 'TechPro Desktop', 'BlueWave Chromebook'])
}
},
# eg 7 # this will output a subset of the ideal answer
{'customer_msg':f"""What Gaming consoles would be good for my friend who is into racing games?""",
'ideal_answer':{
'Gaming Consoles and Accessories':set([
'GameSphere X',
'ProGamer Controller',
'GameSphere Y',
'ProGamer Racing Wheel',
'GameSphere VR Headset'
])}
},
# eg 8
{'customer_msg':f"""What could be a good present for my videographer friend?""",
'ideal_answer': {
'Cameras and Camcorders':set([
'FotoSnap DSLR Camera', 'ActionCam 4K', 'FotoSnap Mirrorless Camera', 'ZoomMaster Camcorder', 'FotoSnap Instant Camera'
])}
},
# eg 9
{'customer_msg':f"""I would like a hot tub time machine.""",
'ideal_answer': []
}
]And then evaluate your test cases by comparing them with the ideal answers using a sample function like this:
import json
def eval_response_with_ideal(response,
ideal,
debug=False):
if debug:
print("response")
print(response)
# json.loads() expects double quotes, not single quotes
json_like_str = response.replace("'",'"')
# parse into a list of dictionaries
l_of_d = json.loads(json_like_str)
# special case when response is empty list
if l_of_d == [] and ideal == []:
return 1
# otherwise, response is empty
# or ideal should be empty, there's a mismatch
elif l_of_d == [] or ideal == []:
return 0
correct = 0
if debug:
print("l_of_d is")
print(l_of_d)
for d in l_of_d:
cat = d.get('category')
prod_l = d.get('products')
if cat and prod_l:
# convert list to set for comparison
prod_set = set(prod_l)
# get ideal set of products
ideal_cat = ideal.get(cat)
if ideal_cat:
prod_set_ideal = set(ideal.get(cat))
else:
if debug:
print(f"did not find category {cat} in ideal")
print(f"ideal: {ideal}")
continue
if debug:
print("prod_set\n",prod_set)
print()
print("prod_set_ideal\n",prod_set_ideal)
if prod_set == prod_set_ideal:
if debug:
print("correct")
correct +=1
else:
print("incorrect")
print(f"prod_set: {prod_set}")
print(f"prod_set_ideal: {prod_set_ideal}")
if prod_set <= prod_set_ideal:
print("response is a subset of the ideal answer")
elif prod_set >= prod_set_ideal:
print("response is a superset of the ideal answer")
# count correct over total number of items in list
pc_correct = correct / len(l_of_d)
return pc_correct
response = find_category_and_product_v2(msg_ideal_pairs_set[7]["customer_msg"], products_and_category)
eval_response_with_ideal(response, msg_ideal_pairs_set[7]["ideal_answer"])
# prints the below output:
# incorrect
# prod_set: {'ProGamer Racing Wheel', 'GameSphere VR Headset', 'ProGamer Controller'}
# prod_set_ideal: {'GameSphere Y', 'ProGamer Racing Wheel', 'GameSphere X', 'GameSphere VR Headset', 'ProGamer Controller'}
# response is a subset of the ideal answer
# 0.04. Define a rubric - In case you are looking to evaluate text responses, then the below function would be helpful where we define a rubric. It specifies what we think the answer should get for us to consider it as a good answer.
cust_prod_info = {
'customer_msg': customer_msg,
'context': product_info
}
def eval_with_rubric(test_set, assistant_answer):
cust_msg = test_set['customer_msg']
context = test_set['context']
completion = assistant_answer
system_message = """\
You are an assistant that evaluates how well the customer service agent \
answers a user question by looking at the context that the customer service \
agent is using to generate its response.
"""
user_message = f"""\
You are evaluating a submitted answer to a question based on the context \
that the agent uses to answer the question.
Here is the data:
[BEGIN DATA]
************
[Question]: {cust_msg}
************
[Context]: {context}
************
[Submission]: {completion}
************
[END DATA]
Compare the factual content of the submitted answer with the context. \
Ignore any differences in style, grammar, or punctuation.
Answer the following questions:
- Is the Assistant response based only on the context provided? (Y or N)
- Does the answer include information that is not provided in the context? (Y or N)
- Is there any disagreement between the response and the context? (Y or N)
- Count how many questions the user asked. (output a number)
- For each question that the user asked, is there a corresponding answer to it?
Question 1: (Y or N)
Question 2: (Y or N)
...
Question N: (Y or N)
- Of the number of questions asked, how many of these questions were addressed by the answer? (output a number)
"""
messages = [
{'role': 'system', 'content': system_message},
{'role': 'user', 'content': user_message}
]
response = get_completion_from_messages(messages)
return response
customer_msg = f"""
tell me about the smartx pro phone and the fotosnap camera, the dslr one.
Also, what TVs or TV related products do you have?"""
products_by_category = utils.get_products_from_query(customer_msg)
category_and_product_list = utils.read_string_to_list(products_by_category)
product_info = utils.get_mentioned_product_info(category_and_product_list)
assistant_answer = utils.answer_user_msg(user_msg=customer_msg,product_info=product_info)
evaluation_output = eval_with_rubric(cust_prod_info, assistant_answer)
print(evaluation_output)This is the sample output for the above code:
If you want to evaluate it based on an expert's answer, the system message and the prompt can be changed like:
system_message = """\
You are an assistant that evaluates how well the customer service agent \
answers a user question by comparing the response to the ideal (expert) response
Output a single letter and nothing else.
"""
user_message = f"""\
You are comparing a submitted answer to an expert answer on a given question. Here is the data:
[BEGIN DATA]
************
[Question]: {cust_msg}
************
[Expert]: {ideal}
************
[Submission]: {completion}
************
[END DATA]
Compare the factual content of the submitted answer with the expert answer. Ignore any differences in style, grammar, or punctuation.
The submitted answer may either be a subset or superset of the expert answer, or it may conflict with it. Determine which case applies. Answer the question by selecting one of the following options:
(A) The submitted answer is a subset of the expert answer and is fully consistent with it.
(B) The submitted answer is a superset of the expert answer and is fully consistent with it.
(C) The submitted answer contains all the same details as the expert answer.
(D) There is a disagreement between the submitted answer and the expert answer.
(E) The answers differ, but these differences don't matter from the perspective of factuality.
choice_strings: ABCDE
"""
messages = [
{'role': 'system', 'content': system_message},
{'role': 'user', 'content': user_message}
]