feat(#1144): weak supervision for multilabel datasets (#1166)

(cherry picked from commit 1803531) feat(#1144): weak multi labels (#1190) (cherry picked from commit 1ba663f) feat(#1144): weak multi labels (#1190) (cherry picked from commit 1ba663f) fix: computation of correct/incorrect in weak multi label summary (#1237) (cherry picked from commit 2167296) feat(#1144): allow multilabel records in weak supervision (#1233) (cherry picked from commit 7deaf16) feat(#1144): allow all labels deselection in weak supervision (#1326) (cherry picked from commit 6e017d4)
argilla-io · Mar 28, 2022 · d832595 · d832595
1 parent 1e44b6f
commit d832595
Show file tree

Hide file tree

Showing 21 changed files with 1,556 additions and 686 deletions.
diff --git a/docs/reference/python/python_labeling.rst b/docs/reference/python/python_labeling.rst
@@ -17,13 +17,10 @@ Labeling tools for the text classification task.
    :exclude-members: RuleNotAppliedError
 
 .. automodule:: rubrix.labeling.text_classification.weak_labels
-   :members:
-   :exclude-members: WeakLabelsError, DuplicatedRuleNameError, NoRecordsFoundError, MultiLabelError, MissingLabelError
+   :members: WeakLabels, WeakMultiLabels
 
 .. automodule:: rubrix.labeling.text_classification.label_models
-   :members:
-   :exclude-members: TieBreakPolicy, LabelModelError, MissingAnnotationError, TooFewRulesError, NotFittedError
+   :members: Snorkel, FlyingSquid
 
 .. automodule:: rubrix.labeling.text_classification.label_errors
-   :members:
-   :exclude-members: SortBy, LabelErrorsException, NoRecordsError, MissingPredictionError
+   :members: find_label_errors
diff --git a/frontend/components/commons/sidebar/SidebarMenu.vue b/frontend/components/commons/sidebar/SidebarMenu.vue
@@ -128,11 +128,7 @@ export default {
       return this.dataset !== undefined;
     },
     showLabellingRules() {
-      return (
-        this.isDatasetView &&
-        !this.dataset.isMultiLabel &&
-        this.dataset.task === "TextClassification"
-      );
+      return this.isDatasetView && this.dataset.task === "TextClassification";
     },
   },
   watch: {

diff --git a/frontend/components/core/ReTableInfo.vue b/frontend/components/core/ReTableInfo.vue
@@ -110,6 +110,11 @@
                     >
                       {{ itemValue(item, column) | percent }}
                     </span>
+                    <span v-else-if="column.type === 'array'">
+                      <p v-for="(arrayItem, index) in itemValue(item, column)" :key="index">
+                        {{arrayItem}}{{index + 1 === itemValue(item, column).length ? '' : ','}}
+                      </p>
+                    </span>
                     <span v-else-if="column.type === 'object'">
                       <p
                         v-for="key in Object.keys(itemValue(item, column))"
@@ -448,9 +453,6 @@ export default {
       border-bottom: none;
       padding-top: 0;
       padding-bottom: 0.2em;
-      // &:hover {
-      //   background:transparent ;
-      // }
     }
     button:not(.re-button) {
       cursor: pointer;
@@ -486,8 +488,6 @@ export default {
       text-align: left;
       margin-right: 1.5em;
       flex: 1 1 0px;
-      // text-overflow: ellipsis;
-      // overflow: hidden;
       &:nth-last-of-type(-n + 2) {
         max-width: 120px;
       }
@@ -497,9 +497,6 @@ export default {
       &:nth-of-type(2) {
         min-width: 30%;
       }
-      // .task span {
-      //   display: flex;
-      // }
       &:first-child {
         flex-shrink: 0;
         min-width: 220px;
@@ -524,15 +521,6 @@ export default {
     margin-top: 5em;
     height: auto;
   }
-  // &__item:not(.disabled) {
-  //   &:hover,
-  //   &:focus {
-  //     background: $line-lighter-color;
-  //     #{$this}__header & {
-  //       background: transparent;
-  //     }
-  //   }
-  // }
   &__actions {
     position: absolute;
     right: 2em;
@@ -593,6 +581,13 @@ export default {
     font-weight: 600;
     word-break: break-word;
   }
+  .array {
+    p {
+      margin-top: 0;
+      margin-bottom: 0;
+      display: block;
+    }
+  }
   .text {
     color: $font-medium-color;
     p {

diff --git a/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue b/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue
@@ -61,10 +61,7 @@ export default {
   },
   watch: {
     async query(newValue) {
-      await this.updateCurrentRule({
-        query: newValue,
-        label: (this.currentRule || {}).label,
-      });
+      await this.dataset.setCurrentLabelingRule({ query: newValue });
     },
   },
   computed: {
@@ -91,19 +88,19 @@ export default {
     },
   },
   methods: {
-    async updateCurrentRule({ query, label }) {
+    async updateCurrentRule({ query, labels }) {
       if (!query) {
         return await this.dataset.clearCurrentLabelingRule();
       }
-      if (label) {
+      if (labels) {
         await this.dataset.setCurrentLabelingRule({
           query,
-          label,
+          labels,
         });
       } else {
         await this.dataset.setCurrentLabelingRule({
           query,
-          label: undefined,
+          labels: [],
         });
       }
       this.saved = false;

diff --git a/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue b/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue
@@ -46,7 +46,7 @@
         :id="label.class"
         :key="`${label.class}`"
         v-model="selectedLabelsVModel"
-        :allow-multiple="false"
+        :allow-multiple="dataset.isMultiLabel"
         :label="label"
         class="label-button"
         :data-title="label.class"
@@ -95,6 +95,7 @@
 import { mapActions } from "vuex";
 import { DatasetViewSettings } from "@/models/DatasetViewSettings";
 import { TextClassificationDataset } from "@/models/TextClassification";
+import _ from "lodash";
 import "assets/icons/info";
 
 export default {
@@ -130,23 +131,29 @@ export default {
       return this.dataset.getCurrentLabelingRuleMetrics() || {};
     },
 
-    selectedLabel() {
-      if (this.selectedLabelsVModel !== undefined) {
-        return this.selectedLabelsVModel[0];
+    selectedLabels() {
+      if (this.selectedLabelsVModel.length) {
+        return this.selectedLabelsVModel;
       }
     },
 
     ruleInfo() {
       // TODO: We can improve this
+      const storedRule =
+        this.currentRule &&
+        this.dataset.findRuleByQuery(this.currentRule.query);
+      const storedRuleLabels = storedRule && storedRule.labels;
+      const queryWithLabelsIsStored = _.isEqual(
+        _.sortBy(storedRuleLabels),
+        _.sortBy(this.selectedLabels)
+      );
       if (this.isSaved) {
         return "The rule was saved";
       }
-      if (
-        this.currentRule &&
-        this.selectedLabelsVModel.length &&
-        this.dataset.findRuleByQuery(this.currentRule.query, this.selectedLabel)
-      ) {
-        return "This query with this label are already saved as rule";
+      if (this.selectedLabels && queryWithLabelsIsStored) {
+        return `This query with ${
+          this.selectedLabels.length > 1 ? "these labels" : "this label"
+        } is already saved as rule`;
       }
     },
     coveredRecords() {
@@ -190,16 +197,21 @@ export default {
     },
   },
   watch: {
-    selectedLabel(newValue) {
-      // Here send description too --> update Rule
-      this.$emit("update-rule", {
-        query: this.query,
-        label: newValue,
-      });
+    selectedLabels: {
+      handler: function (newValue) {
+        if (!_.isEqual(_.sortBy(newValue), _.sortBy(this.currentRule.labels))) {
+          // Here send description too --> update Rule
+          this.$emit("update-rule", {
+            query: this.query,
+            labels: newValue,
+          });
+        }
+      },
+      deep: true,
     },
     currentRule(newValue) {
-      if (newValue && newValue.label) {
-        this.selectedLabelsVModel = [newValue.label];
+      if (newValue && newValue.labels) {
+        this.selectedLabelsVModel = [...newValue.labels];
       }
     },
   },
@@ -211,7 +223,7 @@ export default {
       this.collapseLabels();
       this.$emit("save-rule", {
         query: this.currentRule.query,
-        label: this.selectedLabel,
+        labels: this.selectedLabels,
       });
     },
     expandLabels() {

diff --git a/frontend/components/text-classifier/labeling-rules/RulesManagement.vue b/frontend/components/text-classifier/labeling-rules/RulesManagement.vue
@@ -79,7 +79,7 @@ export default {
           class: "table-info__title",
           type: "action",
         },
-        { name: "Label", field: "label", class: "text" },
+        { name: "Labels", field: "labels", class: "array", type: "array" },
         {
           name: "Coverage",
           field: "coverage",
@@ -158,7 +158,7 @@ export default {
             name: r.description,
             query: r.query,
             kind: "select",
-            label: r.label,
+            labels: r.labels,
             ...this.metricsForRule(r),
             created_at: r.created_at,
           };
@@ -244,7 +244,7 @@ export default {
 <style lang="scss" scoped>
 .rules-management {
   padding-left: 4em;
-  padding-top: 7em;
+  padding-top: 2em;
   overflow: auto;
   height: 100vh;
   &__container {

diff --git a/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue b/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue
@@ -173,38 +173,6 @@ export default {
     formatNumber(value) {
       return isNaN(value) ? "-" : this.$options.filters.percent(value);
     },
-    async getMetrics() {
-      this.metricsTotal = await this.getRulesMetrics({
-        dataset: this.dataset,
-      });
-    },
-    async getMetricsByRules() {
-      const responses = await Promise.all(
-        this.rules.map((rule) => {
-          return this.getRuleruleMetrics({
-            dataset: this.dataset,
-            query: rule.query,
-            label: rule.label,
-          });
-        })
-      );
-
-      responses.forEach((response, idx) => {
-        this.metricsByRules[this.rules[idx].query] = response;
-      });
-    },
-    async getruleMetrics() {
-      if (this.query !== undefined) {
-        const response = await this.getRuleruleMetrics({
-          dataset: this.dataset,
-          query: this.query,
-          label: this.activeLabel,
-        });
-        this.ruleMetrics = response;
-      } else {
-        this.ruleMetrics = {};
-      }
-    },
     getValuesByMetricType(type) {
       return Object.keys(this.metricsByRules).map((key) => {
         return this.metricsByRules[key][type] || 0;