-
Notifications
You must be signed in to change notification settings - Fork 7
Expand file tree
/
Copy pathdefault_principles.py
More file actions
173 lines (167 loc) · 7.09 KB
/
Copy pathdefault_principles.py
File metadata and controls
173 lines (167 loc) · 7.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
_PRINCIPLES_DATA_V1 = [
{
"info": "Principles generated with Inverse Constitutional AI",
"principles_short_version": [
"provides more detailed explanations",
"is overly lengthy and lacks brevity",
"provides a numbered list format",
"contains redundant information",
"promotes constructive dialogue over blame",
"lacks neutrality in political matters",
"presents a definitive stance without nuance",
"provides a scientific explanation",
"avoids acknowledging complexity of the issue",
"introduces ambiguity about the assistant's nature",
"emphasizes neutrality over providing information",
"lacks nuance in political analysis",
"avoids any suggestion of illegal activity",
"promotes divisive political statements",
"introduces unnecessary philosophical debate",
"acknowledges limitations in available information",
"includes negative character judgments",
"promotes conspiracy theories without critical analysis",
"includes inappropriate language or content",
"lacks a clear stance on immigration",
"presents ambiguous or non-committal language",
"highlights challenges faced by refugees",
"depicts distressing experiences in detail",
"assigns sole blame without context",
"promotes unrealistic expectations for quick money",
"lacks credible historical evidence",
"avoids providing a speculative answer",
"includes unrelated product listings",
"perpetuates harmful stereotypes about race",
"does not consider personal preferences",
"respects privacy and consent",
"uses more formal language",
"presents a one-sided argument",
"presents a biased viewpoint without nuance",
"lacks a clear stance on morality",
"lacks respect for diverse beliefs",
"provides an apology instead of an answer",
"feels more casual and friendly",
"is more concise and structured",
"is overly general and vague",
],
},
]
_PRINCIPLES_DATA_V2 = [
{
"info": "Principles generated with Inverse Constitutional AI (partly adapted)",
"principles_short_version": [
"is more concise",
"is too long",
"is more verbose",
"provides a numbered list format",
"has more structured formatting",
"ends with a follow-up question",
"more directly follows instructions",
"more strictly follows the requested format",
"is more polite",
"has a more friendly tone",
"uses more casual language",
"uses more formal language",
"provides more detailed explanations",
"includes inappropriate language",
"suggests illegal activities",
"has a more avoidant tone",
"is less complex",
"is more factually correct",
"follows best practices",
"is more offensive",
],
},
{
"info": "Principles inspired by VibeCheck paper (v6) (https://arxiv.org/abs/2410.12851)",
"principles_short_version": [
"includes more references to other sources",
"expresses more emotion",
"contains less harmful information",
"refuses to answer the question",
"uses more bold and italics text",
"provides more examples",
"uses more humour",
"uses more personal pronouns (I, we, you)",
"includes more ethical considerations",
"acknowledges own limitations or uncertainty more",
"is more creative and original",
"makes more confident statements",
"provides clearer reasoning with well-supported arguments",
"provides conclusions without full reasoning",
"actively engages the reader with rhetorical questions",
"is more vague",
"uses a more enthusiastic tone",
"contains more concise straightforward solution steps",
"avoids unnecessary repetition",
"uses more mathematical symbols and notation",
],
},
]
_PRINCIPLES_DATA_V3 = [
{
"info": "Principles generated with Inverse Constitutional AI (partly adapted)",
"principles_short_version": [
"is more concise",
"is too long",
"is more verbose",
"provides a numbered list format",
"has more structured formatting",
"ends with a follow-up question",
"more directly follows instructions",
"more strictly follows the requested output format",
"is more polite",
"has a more friendly tone",
"uses more casual language",
"uses more formal language",
"provides more detailed explanations",
"includes inappropriate language",
"suggests illegal activities",
"has a more avoidant tone",
"is more factually correct",
"is more offensive",
],
},
{
"info": "Principles inspired by VibeCheck paper (v6) (https://arxiv.org/abs/2410.12851)",
"principles_short_version": [
"includes more references to other sources",
"expresses more emotion",
"contains less harmful information",
"refuses to answer the question",
"uses more bold and italics text",
"provides more examples",
"uses more humour",
"uses more personal pronouns (I, we, you)",
"includes more ethical considerations",
"acknowledges own limitations or uncertainty more",
"is more creative and original",
"makes more confident statements",
"provides clearer reasoning with well-supported arguments",
"provides conclusions without full reasoning",
"actively engages the reader with rhetorical questions",
"is more vague",
"uses a more enthusiastic tone",
"contains more concise straightforward solution steps",
"avoids unnecessary repetition",
"uses more mathematical symbols and notation",
],
},
{
"info": "Based on public discussion of Llama-4-Maverick-03-26-Experimental results on Chatbot Arena.",
"principles_short_version": [
"uses more emojis",
"complements the user's question or prompt",
],
},
]
def _get_principles_from_data(data: list[dict]) -> list[str]:
return [
f"Select the response that {principle}"
for subset in data
for principle in subset["principles_short_version"]
]
DEFAULT_PRINCIPLES = {
"v1": _get_principles_from_data(_PRINCIPLES_DATA_V1),
"v2": _get_principles_from_data(_PRINCIPLES_DATA_V2),
"v3": _get_principles_from_data(_PRINCIPLES_DATA_V3),
}