Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

Commit

Permalink
Model chat: space out annotation buckets (#4275)
Browse files Browse the repository at this point in the history 
* Layer changes

* Docs

* Fixes

* Empty
  • Loading branch information
@EricMichaelSmith
EricMichaelSmith committed Jan 4, 2022
1 parent 27f22d5 commit 22fd071
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 30 deletions.
4 changes: 3 additions & 1 deletion parlai/crowdsourcing/tasks/model_chat/README.md
View file
Expand Up @@ -15,7 +15,9 @@ Set `mephisto.blueprint.chat_data_folder` to the root folder that you want all r
## Passing in task config files

The following flags can be passed in to specify filepaths for overriding the text shown to the workers and the settings of the annotation categories. If they are not specified, the defaults in the `task_config/` folder will be used.
- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories. Set this flag to `""` to disable annotation of model responses.
- `mephisto.blueprint.annotations_config_path`: JSON file configuring annotation categories.
- Set this flag to `""` to disable annotation of model responses.
- If the text of the annotation categories is very long, you can set the `"show_line_breaks"` field (see `task_config/annotations_config.json`) to `true` in order to put a line break between the checkbox and text string of each category.
- `mephisto.blueprint.left_pane_text_path`: HTML to show on the left-hand pane of the chat window.
- `mephisto.blueprint.onboard_task_data_path`: JSON specifying parameters for testing workers during onboarding. Onboarding is only run if model responses will be annotated.
- `mephisto.blueprint.task_description_file`: HTML to show on the initial task-description page shown to the worker.
Expand Down
56 changes: 31 additions & 25 deletions parlai/crowdsourcing/tasks/model_chat/frontend/components/checkboxes.jsx
View file
Expand Up @@ -8,13 +8,13 @@

import React from "react";

function Checkboxes({
annotationBuckets,
turnIdx,
askReason,
annotations,
onUpdateAnnotations,
enabled=true,
function Checkboxes({
annotationBuckets,
turnIdx,
askReason,
annotations,
onUpdateAnnotations,
enabled=true,
}) {
var reasonComponent = (
<div>
Expand All @@ -30,33 +30,39 @@ function Checkboxes({
}
// TODO: add support for radio input type
let input_type = "checkbox";
const showLineBreaks = annotationBuckets.hasOwnProperty("show_line_breaks") ? annotationBuckets.show_line_breaks : false;
const numBuckets = Object.keys(annotationBuckets.config).length;
return (
<div key={'checkboxes_' + turnIdx}>
{
Object.keys(annotationBuckets.config).map(c => (
<span key={'span_' + c + '_' + turnIdx}>
<input
type={input_type}
id={c + '_' + turnIdx}
name={'checkbox_group_' + turnIdx}
onChange={(evt) => {
let newVal = evt.target.checked;
let oldAnnotations = Object.assign({}, annotations);
oldAnnotations[c] = newVal;
onUpdateAnnotations(oldAnnotations);
}}
disabled={!enabled}
/>
<span style={{ marginRight: '15px' }}>
{annotationBuckets.config[c].name}
Object.keys(annotationBuckets.config).map((c, checkboxIdx) => (
<>
<span key={'span_' + c + '_' + turnIdx}>
<input
type={input_type}
id={c + '_' + turnIdx}
name={'checkbox_group_' + turnIdx}
onChange={(evt) => {
let newVal = evt.target.checked;
let oldAnnotations = Object.assign({}, annotations);
oldAnnotations[c] = newVal;
onUpdateAnnotations(oldAnnotations);
}}
disabled={!enabled}
/>
<span style={{ marginRight: '15px' }}>
{annotationBuckets.config[c].name}
</span>
</span>
</span>
{(showLineBreaks && checkboxIdx < numBuckets - 1) ? <br></br> : ''}
</>
))
}
<div id={'checkbox_description_' + turnIdx} style={{ height: '24px' }}></div>
{reasonComponent}
</div>
)
}
// showLineBreaks: show a line break after every checkbox other than the final one

export { Checkboxes };
export { Checkboxes };
3 changes: 2 additions & 1 deletion parlai/crowdsourcing/tasks/model_chat/task_config/annotations_config.json
View file
Expand Up @@ -24,5 +24,6 @@
"name": "None, all good",
"description": "This response implies that there are no problems with the data"
}
}
},
"show_line_breaks": false
}
2 changes: 1 addition & 1 deletion tests/crowdsourcing/tasks/model_chat/expected_states/final_chat_data.json
View file
Expand Up @@ -132,7 +132,7 @@
"1"
],
"task_description": {
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n",
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n",
"model_nickname": "fixed_response",
"model_file": "/private/home/ems/GitHub/facebookresearch/ParlAI_dev/data/models/fixed_response/model",
"model_opt": {
Expand Down
4 changes: 2 additions & 2 deletions tests/crowdsourcing/tasks/model_chat/expected_states/state.json
View file
Expand Up @@ -357,7 +357,7 @@
"person2_seed_utterance": null,
"personas": null,
"task_description": {
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n",
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n",
"model_nickname": "fixed_response",
"model_opt": {
"init_opt": null,
Expand Down Expand Up @@ -515,7 +515,7 @@
"hit_ids": ["1"],
"assignment_ids": ["1"],
"task_description": {
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n }\n}\n",
"annotations_config": "{\n \"config\": {\n \"bucket_0\": {\n \"name\": \"Bucket 0\",\n \"description\": \"this response implies something...0\"\n },\n \"bucket_1\": {\n \"name\": \"Bucket 1\",\n \"description\": \"this response implies something...1\"\n },\n \"bucket_2\": {\n \"name\": \"Bucket 2\",\n \"description\": \"this response implies something...2\"\n },\n \"bucket_3\": {\n \"name\": \"Bucket 3\",\n \"description\": \"this response implies something...3\"\n },\n \"bucket_4\": {\n \"name\": \"Bucket 4\",\n \"description\": \"this response implies something...4\"\n },\n \"none_all_good\": {\n \"name\": \"None, all good\",\n \"description\": \"This response implies that there are no problems with the data\"\n }\n },\n \"show_line_breaks\": false\n}\n",
"model_nickname": "fixed_response",
"model_file": null,
"model_opt": {
Expand Down

0 comments on commit 22fd071

Please sign in to comment.