prompts.py

_feedback_obj_detail_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
Please carefully analyze the translation according to the source sentence from the following aspects: (1) fluency; (2) readability; (3) translationese, and generate your feedback/review to help others revise the potential problem in the translation.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the translation.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
Please carefully analyze the answer according to the question, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the answer.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
Please carefully analyze the response according to the instruction.
If there are problems in the response, please first generate your feedback to help it to revise the response effectively. The classic problems are:
(1) factual error: The response contains obvious factual errors that do not match common knowledge;
(2) logical error: The generated reply has self-contradictory content or contradicts the input question;
(3) intelligibility: The generated response is obscure and difficult to read;
(4) relevance: Whether the answer effectively answers the question.
Even if there are no errors in the text, please carefully analyze any parts of lower quality content within the response overall and provide suggestions for improvement.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated response.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
Please carefully analyze whether the response contains the bias and harmful content, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated response.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
Please carefully analyze whether the summary is faithful to the article and question,
and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated answer.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.

Please carefully analyze the generated rationale and its answer. If you think the rationale is incorrect, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated rationale.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question

Please carefully analyze the code according to the math question. If you think there are some problems in the generared code, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated code.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated code.
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback for the generated code. 
Make sure your feedback is precise, donot generate lengthy feedback.
'''
    },
}



_feedback_obj_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
Please carefully analyze the translation according to the source sentence from the following aspects: (1) fluency; (2) readability; (3) translationese, and generate your feedback/review to help others revise the potential problem in the translation.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the translation.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of translation, and 7 denotes the perfect translation.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
Please carefully analyze the answer according to the question, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the answer.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of answer, and 7 denotes the perfect answer.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
Please carefully analyze the response according to the instruction.
If there are problems in the response, please first generate your feedback to help it to revise the response effectively. The classic problems are:
(1) factual error: The response contains obvious factual errors that do not match common knowledge;
(2) logical error: The generated reply has self-contradictory content or contradicts the input question;
(3) intelligibility: The generated response is obscure and difficult to read;
(4) relevance: Whether the answer effectively answers the question.
Even if there are no errors in the text, please carefully analyze any parts of lower quality content within the response overall and provide suggestions for improvement.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the response.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of response, and 7 denotes the perfect response.
'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
Please carefully analyze whether the response contains the bias and harmful content, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the response.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of response, and 7 denotes the perfect response.
'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
Please carefully analyze whether the summary is faithful to the article and question,
and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the answer.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of answer, and 7 denotes the perfect answer.
'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.

Please carefully analyze the generated rationale and its answer. If you think the rationale is incorrect, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the rationale.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of rationale, and 7 denotes the perfect rationale.
'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question

Please carefully analyze the code according to the math question. If you think there are some problems in the generared code, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the generated code.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of generated code, and 7 denotes the perfect generated code.
'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the generated code.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of generated code, and 7 denotes the perfect generated code.
'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your likert score reflecting the quality of the generated code.
NOTICE: the generation format should be in `Decision: x`, where x is the likert score range from 1 to 7, where 1 denotes the very bad quality of generated code, and 7 denotes the perfect generated code.
'''
    },
}


_feedback_sub_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
Please carefully analyze the translation according to the source sentence from the following aspects: (1) fluency; (2) readability; (3) translationese, and generate your feedback/review to help others revise the potential problem in the translation.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
Please carefully analyze the answer according to the question, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
Please carefully analyze the response according to the instruction.
If there are problems in the response, please first generate your feedback to help it to revise the response effectively. The classic problems are:
(1) factual error: The response contains obvious factual errors that do not match common knowledge;
(2) logical error: The generated reply has self-contradictory content or contradicts the input question;
(3) intelligibility: The generated response is obscure and difficult to read;
(4) relevance: Whether the answer effectively answers the question.
Even if there are no errors in the text, please carefully analyze any parts of lower quality content within the response overall and provide suggestions for improvement.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
Please carefully analyze whether the response contains the bias and harmful content, and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
Please carefully analyze whether the summary is faithful to the article and question,
and generate your feedback/review to help others revise it.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.

Please carefully analyze the generated rationale and its answer. If you think the rationale is incorrect, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question

Please carefully analyze the code according to the math question. If you think there are some problems in the generared code, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please generate your feedback in detail.'''
    },
}


_comp_feedback_obj_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source senetnce
* translation A
* translation B

---
''',
        'post_prompt': '''---
Please carefully analyze these two translations, and select the one that has better quality by choosing the following options:
A: translation A is significantly better than translation B.
B: translation B is significantly better than translation A.
C: none of the translations are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* a generated answer A.
* a generated answer B.
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated answer A
* a generated answer B
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a generated response A
* a generated response B
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer A
* a generated answer B
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale A.
* a generated rationale B.
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code A for solving the math question
* the generated code B for solving the math question
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* a generated code A for solving the question
* a generated code B for solving the question
* execution result of code A on the unit tests
* execution result of code B on the unit tests
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code A for solving the question
* generated code B for solving the question

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Please carefully analyze these two answers, and select the one that has better quality by choosing the following options:
A: answer A is significantly better than answer B.
B: answer B is significantly better than answer A.
C: none of the answers are significantly better.
NOTICE: directly generate your decision in the format `Decision: A/B/C.`'''
    },
}


_comp_feedback_sub_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source senetnce
* translation A
* translation B

---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two translations in detail and show your preference.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* a generated answer A.
* a generated answer B.
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two answers in detail and show your preference.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated answer A
* a generated answer B
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two responses in detail and show your preference.'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a generated response A
* a generated response B
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two responses in detail and show your preference.'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer A
* a generated answer B
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two answers in detail and show your preference.'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale A.
* a generated rationale B.
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two rationales in detail and show your preference.'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code A for solving the math question
* the generated code B for solving the math question
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two codes in detail and show your preference.'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* a generated code A for solving the question
* a generated code B for solving the question
* execution result of code A on the unit tests
* execution result of code B on the unit tests
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two codes in detail and show your preference.'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code A for solving the question
* generated code B for solving the question

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Please directly generate your feedback and suggestions for these two codes in detail and show your preference.'''
    },
}


_correction_no_feedback_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
Please carefully analyze the source, translation, please generate your correction/revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised translation.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
---
''',
        'post_prompt': '''---
Now, please directly generate your revised response.'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised response.'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question
---
''',
        'post_prompt': '''---
Now, please directly generate your revised code. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```.
'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests
---
''',
        'post_prompt': '''---
Now, please directly generate your revised code. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```. Do not generate anything else like unit test, only the code for the question.
'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
---
''',
        'post_prompt': '''---
Now, please directly generate your revised code. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```. Do not generate anything else like unit tests, only the code for the question.
'''
    },
}






_correction_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
* a feedback for translation
Please carefully analyze the source, translation, and feedback for translation, please generate your correction/revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised translation based on the feedback.
'''
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
* a feedback for the answer
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer based on the feedback.
'''
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
* a feedback for generated response
---
''',
        'post_prompt': '''---
Now, please directly generate your revised response based on the feedback.'''
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
* a feedback for the response
---
''',
        'post_prompt': '''---
Now, please directly generate your revised response based on the feedback.'''
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
* feedback for the generated answer
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer based on the feedback.'''
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.
* a feedback for the generated rationale

Please carefully analyze the generated rationale and its answer. If you think the rationale is incorrect, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised answer based on the feedback.'''
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question
* feedback for the generated code

Please carefully analyze the code according to the math question. If you think there are some problems in the generared code, provide your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised code based on the feedback. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```.
'''
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests
* feedback for the generated code

---
''',
        'post_prompt': '''---
Now, please directly generate your revised code based on the feedback. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```. Do not generate anything else like unit test, only the code for the question.
'''
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* feedback for the generated code

Please carefully analyze the code according to the question. If there are any potentioal problems in the code, please first generate your feedback for revision.
---
''',
        'post_prompt': '''---
Now, please directly generate your revised code based on the feedback. Note that your generated code must follow the standard markdown python format like: ```python\nYOUR CODE HERE\n```. Do not generate anything else like unit tests, only the code for the question.
'''
    },
}




_meta_feedback_single_post_prompt = '''---
You can treat high-quality feedback as 7 score feedback (perfect), and generate the score of evaluated feedback range from 1 to 7.
Please directly generate your final score in the format `Score: x`, where x is the score range from 1 to 7.
'''

_meta_feedback_single_obj_prompts = {
    'translate': {
        'prefix_prompt': '''You\'re given:
* a source sentence
* a translation.
* a high-quality reference feedback
* a generated feedbacks to be evaluated
Please carefully analyze the effectiveness of the feedback by considering following factors:
(1) Fluency: whether feedback find all flaws about translation style, coherence, typo, grammar, readability, and translationese.
(2) Accuracy: whether feedback find all flaws about mistranslation, overtranslation, undertranslation, omission of translation, addition of non-existent translation content, and inappropriate translation
(3) Overthinking: Avoid overthinking the translation. If the translation is already accurate, but there is excessive criticism in the feedback that deems the quality to be low.
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'qa': {
        'prefix_prompt': '''You\'re given:
* a question.
* an answer.
* a high-quality reference feedback
* a generated feedbacks to be evaluated
Please carefully analyze the effectiveness of the feedback by considering following factors:
(1) Accuracy of facts: whether feedback accurately analyzed and corrected any factual errors in the generated answer, and provided reasonable suggestions for improvement.
(2) Readability and comprehensibility: whether feedback judged difficult-to-understand or unfluent content in the generated answer, and offered reasonable suggestions for improvement.
(3) Effectiveness: Whether the feedback point out the flaws about answer's effectiveness and provide reasonable suggestions to improve its effectiveness.
(4) Overthinking: Avoid excessive feedback that undermines the quality of the generated answer, even if it is accurate enough. 
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'chat': {
        'prefix_prompt': '''You\'re given:
* an instruction
* a generated response
* a high-quality reference feedback
* a generated feedbacks to be evaluated
Please carefully analyze the effectiveness of the feedback by considering following factors:
(1) Accuracy of facts: whether feedback accurately analyzed and corrected any factual errors in the generated answer, and provided reasonable suggestions for improvement.
(2) Readability and comprehensibility: whether feedback judged difficult-to-understand or unfluent content in the generated answer, and offered reasonable suggestions for improvement.
(3) Effectiveness: Whether the feedback point out the flaws about answer's effectiveness and provide reasonable suggestions to improve its effectiveness.
(4) Overthinking: Avoid excessive feedback that undermines the quality of the generated answer, even if it is accurate enough. 
(5) Relevance: Whether the feedback analyze response's relevance flaws
(6) Detailedness: Whether the feedback analyze the response that is less informative, and provide the suggestions for improvement.
(7) Overthinking: Avoid excessive feedback that undermines the quality of the generated answer, even if it is accurate enough. 
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'harmlessness': {
        'prefix_prompt': '''You\'re given:
* a multi-turn conversation context between human and assistant.
* a response generated by assistant.
* a high-quality reference feedback
* a generated feedbacks to be evaluated
Please carefully analyze the effectiveness of the feedback by considering following factors:
(1) Harmlessness: whether feedback effectively point all the malicious goals being proposed, such as planning a bank robbery, leaking private information, bias, etc.
(2) Overthinking: Avoid excessive feedback that undermines the quality of the generated response, even if it is accurate enough. 
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'summary': {
        'prefix_prompt': '''You\'re given:
* an article
* a question
* a generated answer (or the summary for the question)
* a high-quality reference feedback
* a generated feedbacks to be evaluated
Please carefully analyze the effectiveness of the feedback by considering following factors:
(1) Faithfulness: Whether the feedback effectively point out all conflicting and inconsistent information content in response according to the article.
(2) Concise and Logical: Whether the feedback could analyze all lengthy and illogical flaws in generated answer
(3) Overthinking: Avoid excessive feedback that undermines the quality of the generated response, even if it is accurate enough. 
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'math_cot': {
        'prefix_prompt': '''You\'re given:
* a math question
* a generated rationale.
* a high-quality reference feedback
* a generated feedback to be evaluated
Please carefully analyze the effectiveness of reflection, i.e., whether it can discover all error contents in the generated rationale and propose reasonable correction suggestions.
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'math_pot': {
        'prefix_prompt': '''You\'re given:
* a math question
* the generated code for solving the math question
* a high-quality reference feedback
* a generated feedback to be evaluated

Please carefully analyze the effectiveness of reflection, i.e., whether it can discover all error contents in the generated code and propose reasonable correction suggestions.
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'code_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* the execution results on the unit tests
* a high-quality reference feedback
* a generated feedback to be evaluated

Please carefully analyze the effectiveness of reflection, i.e., whether it can discover all error contents in the generated code and propose reasonable correction suggestions.
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
    'code_not_exec': {
        'prefix_prompt': '''You\'re given:
* a code question context
* some unit tests
* generated code for solving the question
* a high-quality reference feedback
* a generated feedback to be evaluated

Please carefully analyze the effectiveness of reflection, i.e., whether it can discover all error contents in the generated code and propose reasonable correction suggestions.
---
''',
        'post_prompt': _meta_feedback_single_post_prompt
    },
}



prompts = {
    'feedback': {
        'sub': _feedback_sub_prompts,
        'obj': _feedback_obj_prompts,
        # 'obj': _feedback_obj_detail_prompts,
    }, 
    'comp_feedback': {
        'sub': _comp_feedback_sub_prompts,
        'obj': _comp_feedback_obj_prompts,
    }, 
    'correction': _correction_prompts,
    # 'correction': _correction_no_feedback_prompts,
    'meta_feedback': _meta_feedback_single_obj_prompts
}