feat(labeler): allow user fixing the question (#159)

* feat(labeler): allow user fixing the question

* feat(labeler): allow user fixing the question

docs/components/labeler.md

@@ -116,8 +116,10 @@ The view section collects the configs determining how frontend renders the quest
 - `Field`: represents the field of `Document` your question data come from.
   - `Tags Key`: when you select `Field` as `.tags`, this textbox will show up, asking you to further specify which `.tags` key your question data comes from.
 - `Content Type`: you need to select the right content type to have the correct rendering on the the question data.
-- `Examples/View`: The maximum number of labeling examples on the frontend.
-- `TopK/Examples`: The maximum number of results for each example on the frontend.
+- `Questions/session`: The maximum number of labeling examples on the frontend.
+- `TopK/Question`: The maximum number of results for each example on the frontend.
+- `Start question`: The starting index of the question
+- `Keep same question`: If set, then `Start question` and `Questions/session` are locked. You will always get the same questions for labeling. 
 
 ````{tip}
 If your question panel looks like the image below, this means rendering is not setup correctly. You need to change `Field`, `Content Type` and `Tags Key` to correct the render setup.
@@ -129,6 +131,9 @@ If your question panel looks like the image below, this means rendering is not s
 
 ````
 
+```{tip}
+You can use `Keep same question` to debug your model: by fixing the query and observing how the model behaves after learning from your new labels.
+```
 
 #### Progress
 

finetuner/labeler/ui/index.html

@@ -22,7 +22,8 @@
         </div>
         <sidebar :labeler-config="labeler_config" :view-template="view_template" :tags="tags" :is-busy="is_busy"
             :progress-stats="progress_stats" :positive-rate="positive_rate" :negative-rate="negative_rate"
-            :advanced-config="advanced_config" :save-progress="saveProgress"></sidebar>
+            :advanced-config="advanced_config" :save-progress="saveProgress"
+                 :next-batch="next_batch"></sidebar>
         <div class="b-example-divider"></div>
         <div class="flex-grow-1 p-1 overflow-hidden">
             <div class="d-flex flex-column h-100">

finetuner/labeler/ui/js/components/sidebar.vue.js

@@ -9,6 +9,7 @@ const sidebar = {
         negativeRate: Number,
         advancedConfig: Object,
         saveProgress: Function,
+        nextBatch: Function,
     },
     template: `
     <div class="d-flex flex-column flex-shrink-0 p-3 sidebar">
@@ -61,18 +62,34 @@ const sidebar = {
                                 </select>
                             </div>
                         </div>
+                        <div class="row my-1">
+                            <label class="col-sm-6 col-form-label">Start question</label>
+                            <div class="col-sm-6">
+                                <input class="form-control" type="number" min="0"
+                                        v-model.number="labelerConfig.start_idx" v-on:input="nextBatch()" 
+                                        :disabled="labelerConfig.same_question">
+                            </div>
+                        </div>
                         <div class="row my-1">
                             <label class="col-sm-6 col-form-label">Questions/Session</label>
                             <div class="col-sm-6">
                                 <input class="form-control" type="number" min="1" max="9"
-                                        v-model.number="labelerConfig.example_per_view">
+                                        v-model.number="labelerConfig.example_per_view" v-on:input="nextBatch()"
+                                        :disabled="labelerConfig.same_question">
+                            </div>
+                        </div>
+                        <div class="row my-1">
+                            <label class="col-sm-6 col-form-label">Keep same question</label>
+                            <div class="col-sm-6 d-flex align-items-center justify-content-center">
+                                <input class="form-check-input"  type="checkbox"
+                                        v-model="labelerConfig.same_question" v-on:input="nextBatch(true, false)">
                             </div>
                         </div>
                         <div class="row my-1">
                             <label class="col-sm-6 col-form-label">TopK/Question</label>
                             <div class="col-sm-6">
                                 <input class="form-control" type="number" min="1" max="9"
-                                        v-model.number="labelerConfig.topk_per_example">
+                                        v-model.number="labelerConfig.topk_per_example" v-on:input="nextBatch()">
                             </div>
                         </div>
                     </div>

finetuner/labeler/ui/js/main.js

@@ -22,7 +22,8 @@ const app = new Vue({
         labeler_config: {
             content: 'uri',
             style: 'image',
-            example_per_view: 5,
+            example_per_view: 3,
+            same_question: false,
             topk_per_example: 9,
             tags: '',
             start_idx: 0,
@@ -122,7 +123,11 @@ const app = new Vue({
             });
             app.progress_stats.total.value++
             app.fit_doc(doc)
-            app.next_batch()
+            if (app.labeler_config.same_question) {
+                app.next_batch(true, false)
+            } else {
+                app.next_batch(false)
+            }
         },
         fit_doc: function (doc) {
             app.is_busy = true
@@ -155,13 +160,15 @@ const app = new Vue({
                 return doc.tags[app.labeler_config.tags]
             }
         },
-        next_batch: function () {
-            let end_idx = app.labeler_config.start_idx + (app.labeler_config.example_per_view - app.cur_batch.length)
-            if (end_idx === app.labeler_config.start_idx) {
+        next_batch: function (clear_exist=true, update_start_idx=true) {
+            if (clear_exist) {
+                app.cur_batch = []
+            }
+            let new_examples_to_query = Math.max(0, app.labeler_config.example_per_view - app.cur_batch.length)
+            let end_idx = app.labeler_config.start_idx + new_examples_to_query
+            if (end_idx <= app.labeler_config.start_idx) {
                 return
             }
-            let start_idx = app.labeler_config.start_idx
-            app.labeler_config.start_idx = end_idx
             app.is_busy = true
             app.is_conn_broken = false
             $.ajax({
@@ -170,7 +177,7 @@ const app = new Vue({
                 data: JSON.stringify({
                     data: [],
                     parameters: {
-                        'start': start_idx,
+                        'start': app.labeler_config.start_idx,
                         'end': end_idx,
                         'topk': app.labeler_config.topk_per_example,
                         'sample_size': app.advanced_config.sample_size.value
@@ -179,8 +186,15 @@ const app = new Vue({
                 contentType: "application/json; charset=utf-8",
                 dataType: "json",
             }).success(function (data, textStatus, jqXHR) {
+                if (update_start_idx) {
+                    app.labeler_config.start_idx = end_idx
+                }
+
                 app.cur_batch.push(...data['data'].docs)
-                app.tags = Object.keys(data.data.docs[0].tags)
+                try {
+                    app.tags = Object.keys(data.data.docs[0].tags)
+                } catch (e) {}
+
                 app.is_busy = false
                 app.progress_stats.this_session.value = app.cur_batch.length
             }).fail(function () {

finetuner/labeler/ui/main.css

@@ -2,6 +2,7 @@ body {
     min-height: 100vh;
     min-height: -webkit-fill-available;
     font-family: 'Poppins', sans-serif;
+    font-weight: lighter;
 }
 
 html {