From 8d616cc4aee937cf9a4fc1e1e366468548fd106e Mon Sep 17 00:00:00 2001 From: bkb2135 Date: Thu, 13 Jun 2024 20:36:28 +0000 Subject: [PATCH 1/4] Improve multi-turn prompting --- prompting/tasks/qa.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/prompting/tasks/qa.py b/prompting/tasks/qa.py index 2fa293087..ef8bf2bab 100644 --- a/prompting/tasks/qa.py +++ b/prompting/tasks/qa.py @@ -22,9 +22,9 @@ # Used to obtain the query (which is a followup question about the context) # TODO: we may not need the entire conversation history - we can sample a subset of it (first k messages, last k messages, etc.) FOLLOWUP_PROMPT_TEMPLATE = """ -Compose a single, specific question to continue the dialogue below. Adopt the persona of the original user, including their communication style and objectives. The question should be based on the previous exchanges and must not be answerable with a simple yes or no. +Compose a single, specific question to continue the dialogue below. Adopt the persona of the original user, reflecting their communication style and objectives. The question should be rooted in the previous exchanges and should not be answerable with a simple yes or no. -The question should require detailed knowledge of the conversation history for a correct response, emphasizing requests for clarification or additional details (e.g., 'What specific steps did you take?' or 'How did that situation resolve?'). Avoid referring to the subject by name and instead use indirect pronouns or descriptions (e.g., 'he,' 'she,' 'it'). Avoid answering the question yourself and refrain from providing new information not already discussed. +Ensure the question requires detailed knowledge of the conversation history for a correct response, focusing on requests for clarification or additional details (e.g., 'What specific steps did you take?', 'Are you sure?', 'How do you know that is true', or 'How did that situation resolve?'). Use indirect pronouns or descriptions to refer to subjects (e.g., 'he,' 'she,' 'it') instead of their names. Avoid answering the question yourself and do not introduce new information not already discussed. # Context: {context} @@ -33,6 +33,7 @@ {history} """ + # Used to obtain reference answer REFERENCE_PROMPT_TEMPLATE = """\ Answer the question you will receive in detail, utilizing the following context. @@ -45,18 +46,20 @@ """ # TODO: We also need a special followup reference prompt (or just merge both) -# Used to obtain reference answer +# TODO: We should FOLLOWUP_REFERENCE_PROMPT_TEMPLATE = """\ -Answer the question you will receive in detail, utilizing the following context and conversation history as required. - -#Context: -{context} +You are a helpful assistant. Answer the question below in detail, prioritizing the use of the provided conversation history. The context is available for additional information if needed, but it may not always be relevant. # Conversation History: {history} +# Context (optional): +{context} + # Question: {question} + +Ensure your answer references relevant parts of the conversation history. Use the context only if it provides additional necessary information. """ @dataclass From 3a3fab372c5451050b40dbb62f6392f338383aaa Mon Sep 17 00:00:00 2001 From: bkb2135 Date: Thu, 13 Jun 2024 22:01:43 +0000 Subject: [PATCH 2/4] Reiterate need for pronouns --- prompting/tasks/qa.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prompting/tasks/qa.py b/prompting/tasks/qa.py index ef8bf2bab..9f82cf840 100644 --- a/prompting/tasks/qa.py +++ b/prompting/tasks/qa.py @@ -24,8 +24,9 @@ FOLLOWUP_PROMPT_TEMPLATE = """ Compose a single, specific question to continue the dialogue below. Adopt the persona of the original user, reflecting their communication style and objectives. The question should be rooted in the previous exchanges and should not be answerable with a simple yes or no. -Ensure the question requires detailed knowledge of the conversation history for a correct response, focusing on requests for clarification or additional details (e.g., 'What specific steps did you take?', 'Are you sure?', 'How do you know that is true', or 'How did that situation resolve?'). Use indirect pronouns or descriptions to refer to subjects (e.g., 'he,' 'she,' 'it') instead of their names. Avoid answering the question yourself and do not introduce new information not already discussed. +Ensure the question requires detailed knowledge of the conversation history for a correct response, focusing on requests for clarification or additional details (e.g., 'What specific steps did you take?', 'Are you sure?', 'How do you know that is true', or 'How did that situation resolve?'). Use indirect pronouns or descriptions to refer to subjects instead of their names. Avoid answering the question yourself and do not introduce new information not already discussed. +When asking a followup question, you should use pronouns or descriptions to refer to subjects instead of their names. # Context: {context} From 3576010c246d1e77ce5d63221d9b75811a8bb9d2 Mon Sep 17 00:00:00 2001 From: bkb2135 Date: Thu, 13 Jun 2024 23:30:54 +0000 Subject: [PATCH 3/4] Stress importance of pronouns --- prompting/tasks/qa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/tasks/qa.py b/prompting/tasks/qa.py index 9f82cf840..133754ddd 100644 --- a/prompting/tasks/qa.py +++ b/prompting/tasks/qa.py @@ -26,7 +26,7 @@ Ensure the question requires detailed knowledge of the conversation history for a correct response, focusing on requests for clarification or additional details (e.g., 'What specific steps did you take?', 'Are you sure?', 'How do you know that is true', or 'How did that situation resolve?'). Use indirect pronouns or descriptions to refer to subjects instead of their names. Avoid answering the question yourself and do not introduce new information not already discussed. -When asking a followup question, you should use pronouns or descriptions to refer to subjects instead of their names. +When asking a followup question, you should use pronouns or descriptions to refer to subjects instead of their names. You absolutely must not repeat the subject of the question in the followup question. For example, if the question is "What is the capital of France?", the followup question should not be "What is the population of France?". Instead, it should be "How many people live there?" or "What is its population?". # Context: {context} From 49da0961e5d1bfcd3e65dff6b01254de843ae411 Mon Sep 17 00:00:00 2001 From: bkb2135 <98138173+bkb2135@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:35:42 -0400 Subject: [PATCH 4/4] SN1-12 --- prompting/tasks/qa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/tasks/qa.py b/prompting/tasks/qa.py index 133754ddd..d8a6e91a8 100644 --- a/prompting/tasks/qa.py +++ b/prompting/tasks/qa.py @@ -47,7 +47,7 @@ """ # TODO: We also need a special followup reference prompt (or just merge both) -# TODO: We should +# TODO: We should create followups using the specified llama3 chat template rather than feeding the message history through textually FOLLOWUP_REFERENCE_PROMPT_TEMPLATE = """\ You are a helpful assistant. Answer the question below in detail, prioritizing the use of the provided conversation history. The context is available for additional information if needed, but it may not always be relevant.