From d832595cc5e63ff435d2d81b6c94c17b0abcb550 Mon Sep 17 00:00:00 2001
From: Francisco Aranda <francisco@recogn.ai>
Date: Sun, 20 Feb 2022 22:14:02 +0100
Subject: [PATCH] feat(#1144): weak supervision for multilabel datasets (#1166)

(cherry picked from commit 18035318c967c9f5d81da95b858e18891c638917)

feat(#1144): weak multi labels (#1190)

(cherry picked from commit 1ba663f0e66636809139f4613481c57e54957660)

feat(#1144): weak multi labels (#1190)

(cherry picked from commit 1ba663f0e66636809139f4613481c57e54957660)

fix: computation of correct/incorrect in weak multi label summary (#1237)

(cherry picked from commit 2167296a42c810ec100b5f53f0f0d835282dd0be)

feat(#1144): allow multilabel records in weak supervision (#1233)

(cherry picked from commit 7deaf165a62a9dd2c2f8f3f16351ac0a8c200004)

feat(#1144): allow all labels deselection in weak supervision (#1326)

(cherry picked from commit 6e017d46289c7cd7d60da22ec581edcda916179b)
---
 docs/reference/python/python_labeling.rst     |   9 +-
 .../commons/sidebar/SidebarMenu.vue           |   6 +-
 frontend/components/core/ReTableInfo.vue      |  29 +-
 .../labeling-rules/RuleDefinition.vue         |  13 +-
 .../labeling-rules/RuleLabelsDefinition.vue   |  50 +-
 .../labeling-rules/RulesManagement.vue        |   6 +-
 .../labeling-rules/RulesMetrics.vue           |  32 -
 frontend/models/TextClassification.js         |  50 +-
 .../client/sdk/text_classification/models.py  |   3 +-
 .../labeling/text_classification/__init__.py  |   2 +-
 .../labeling/text_classification/rule.py      |  22 +-
 .../text_classification/weak_labels.py        | 627 ++++++++++--
 src/rubrix/server/commons/es_wrapper.py       |   6 +-
 src/rubrix/server/commons/helpers.py          |   2 +-
 .../tasks/text_classification/api/api.py      |   8 +-
 .../tasks/text_classification/api/model.py    |  21 +-
 .../service/labeling_service.py               | 109 +-
 .../text_classification/service/service.py    |  26 +-
 .../text_classification/test_label_models.py  |   2 +-
 .../text_classification/test_weak_labels.py   | 933 ++++++++++++------
 .../text_classification/test_api_rules.py     | 286 ++++--
 21 files changed, 1556 insertions(+), 686 deletions(-)

diff --git a/docs/reference/python/python_labeling.rst b/docs/reference/python/python_labeling.rst
index 35d684d9d4..59180b21a8 100644
--- a/docs/reference/python/python_labeling.rst
+++ b/docs/reference/python/python_labeling.rst
@@ -17,13 +17,10 @@ Labeling tools for the text classification task.
    :exclude-members: RuleNotAppliedError
 
 .. automodule:: rubrix.labeling.text_classification.weak_labels
-   :members:
-   :exclude-members: WeakLabelsError, DuplicatedRuleNameError, NoRecordsFoundError, MultiLabelError, MissingLabelError
+   :members: WeakLabels, WeakMultiLabels
 
 .. automodule:: rubrix.labeling.text_classification.label_models
-   :members:
-   :exclude-members: TieBreakPolicy, LabelModelError, MissingAnnotationError, TooFewRulesError, NotFittedError
+   :members: Snorkel, FlyingSquid
 
 .. automodule:: rubrix.labeling.text_classification.label_errors
-   :members:
-   :exclude-members: SortBy, LabelErrorsException, NoRecordsError, MissingPredictionError
+   :members: find_label_errors
diff --git a/frontend/components/commons/sidebar/SidebarMenu.vue b/frontend/components/commons/sidebar/SidebarMenu.vue
index f76410002b..bbb85da56d 100644
--- a/frontend/components/commons/sidebar/SidebarMenu.vue
+++ b/frontend/components/commons/sidebar/SidebarMenu.vue
@@ -128,11 +128,7 @@ export default {
       return this.dataset !== undefined;
     },
     showLabellingRules() {
-      return (
-        this.isDatasetView &&
-        !this.dataset.isMultiLabel &&
-        this.dataset.task === "TextClassification"
-      );
+      return this.isDatasetView && this.dataset.task === "TextClassification";
     },
   },
   watch: {
diff --git a/frontend/components/core/ReTableInfo.vue b/frontend/components/core/ReTableInfo.vue
index b9cb1091e7..fa14679455 100644
--- a/frontend/components/core/ReTableInfo.vue
+++ b/frontend/components/core/ReTableInfo.vue
@@ -110,6 +110,11 @@
                     >
                       {{ itemValue(item, column) | percent }}
                     </span>
+                    <span v-else-if="column.type === 'array'">
+                      <p v-for="(arrayItem, index) in itemValue(item, column)" :key="index">
+                        {{arrayItem}}{{index + 1 === itemValue(item, column).length ? '' : ','}}
+                      </p>
+                    </span>
                     <span v-else-if="column.type === 'object'">
                       <p
                         v-for="key in Object.keys(itemValue(item, column))"
@@ -448,9 +453,6 @@ export default {
       border-bottom: none;
       padding-top: 0;
       padding-bottom: 0.2em;
-      // &:hover {
-      //   background:transparent ;
-      // }
     }
     button:not(.re-button) {
       cursor: pointer;
@@ -486,8 +488,6 @@ export default {
       text-align: left;
       margin-right: 1.5em;
       flex: 1 1 0px;
-      // text-overflow: ellipsis;
-      // overflow: hidden;
       &:nth-last-of-type(-n + 2) {
         max-width: 120px;
       }
@@ -497,9 +497,6 @@ export default {
       &:nth-of-type(2) {
         min-width: 30%;
       }
-      // .task span {
-      //   display: flex;
-      // }
       &:first-child {
         flex-shrink: 0;
         min-width: 220px;
@@ -524,15 +521,6 @@ export default {
     margin-top: 5em;
     height: auto;
   }
-  // &__item:not(.disabled) {
-  //   &:hover,
-  //   &:focus {
-  //     background: $line-lighter-color;
-  //     #{$this}__header & {
-  //       background: transparent;
-  //     }
-  //   }
-  // }
   &__actions {
     position: absolute;
     right: 2em;
@@ -593,6 +581,13 @@ export default {
     font-weight: 600;
     word-break: break-word;
   }
+  .array {
+    p {
+      margin-top: 0;
+      margin-bottom: 0;
+      display: block;
+    }
+  }
   .text {
     color: $font-medium-color;
     p {
diff --git a/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue b/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue
index 87327443a3..9e2c233443 100644
--- a/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue
+++ b/frontend/components/text-classifier/labeling-rules/RuleDefinition.vue
@@ -61,10 +61,7 @@ export default {
   },
   watch: {
     async query(newValue) {
-      await this.updateCurrentRule({
-        query: newValue,
-        label: (this.currentRule || {}).label,
-      });
+      await this.dataset.setCurrentLabelingRule({ query: newValue });
     },
   },
   computed: {
@@ -91,19 +88,19 @@ export default {
     },
   },
   methods: {
-    async updateCurrentRule({ query, label }) {
+    async updateCurrentRule({ query, labels }) {
       if (!query) {
         return await this.dataset.clearCurrentLabelingRule();
       }
-      if (label) {
+      if (labels) {
         await this.dataset.setCurrentLabelingRule({
           query,
-          label,
+          labels,
         });
       } else {
         await this.dataset.setCurrentLabelingRule({
           query,
-          label: undefined,
+          labels: [],
         });
       }
       this.saved = false;
diff --git a/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue b/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue
index c342c86e8c..a2fb8652fa 100644
--- a/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue
+++ b/frontend/components/text-classifier/labeling-rules/RuleLabelsDefinition.vue
@@ -46,7 +46,7 @@
         :id="label.class"
         :key="`${label.class}`"
         v-model="selectedLabelsVModel"
-        :allow-multiple="false"
+        :allow-multiple="dataset.isMultiLabel"
         :label="label"
         class="label-button"
         :data-title="label.class"
@@ -95,6 +95,7 @@
 import { mapActions } from "vuex";
 import { DatasetViewSettings } from "@/models/DatasetViewSettings";
 import { TextClassificationDataset } from "@/models/TextClassification";
+import _ from "lodash";
 import "assets/icons/info";
 
 export default {
@@ -130,23 +131,29 @@ export default {
       return this.dataset.getCurrentLabelingRuleMetrics() || {};
     },
 
-    selectedLabel() {
-      if (this.selectedLabelsVModel !== undefined) {
-        return this.selectedLabelsVModel[0];
+    selectedLabels() {
+      if (this.selectedLabelsVModel.length) {
+        return this.selectedLabelsVModel;
       }
     },
 
     ruleInfo() {
       // TODO: We can improve this
+      const storedRule =
+        this.currentRule &&
+        this.dataset.findRuleByQuery(this.currentRule.query);
+      const storedRuleLabels = storedRule && storedRule.labels;
+      const queryWithLabelsIsStored = _.isEqual(
+        _.sortBy(storedRuleLabels),
+        _.sortBy(this.selectedLabels)
+      );
       if (this.isSaved) {
         return "The rule was saved";
       }
-      if (
-        this.currentRule &&
-        this.selectedLabelsVModel.length &&
-        this.dataset.findRuleByQuery(this.currentRule.query, this.selectedLabel)
-      ) {
-        return "This query with this label are already saved as rule";
+      if (this.selectedLabels && queryWithLabelsIsStored) {
+        return `This query with ${
+          this.selectedLabels.length > 1 ? "these labels" : "this label"
+        } is already saved as rule`;
       }
     },
     coveredRecords() {
@@ -190,16 +197,21 @@ export default {
     },
   },
   watch: {
-    selectedLabel(newValue) {
-      // Here send description too --> update Rule
-      this.$emit("update-rule", {
-        query: this.query,
-        label: newValue,
-      });
+    selectedLabels: {
+      handler: function (newValue) {
+        if (!_.isEqual(_.sortBy(newValue), _.sortBy(this.currentRule.labels))) {
+          // Here send description too --> update Rule
+          this.$emit("update-rule", {
+            query: this.query,
+            labels: newValue,
+          });
+        }
+      },
+      deep: true,
     },
     currentRule(newValue) {
-      if (newValue && newValue.label) {
-        this.selectedLabelsVModel = [newValue.label];
+      if (newValue && newValue.labels) {
+        this.selectedLabelsVModel = [...newValue.labels];
       }
     },
   },
@@ -211,7 +223,7 @@ export default {
       this.collapseLabels();
       this.$emit("save-rule", {
         query: this.currentRule.query,
-        label: this.selectedLabel,
+        labels: this.selectedLabels,
       });
     },
     expandLabels() {
diff --git a/frontend/components/text-classifier/labeling-rules/RulesManagement.vue b/frontend/components/text-classifier/labeling-rules/RulesManagement.vue
index 9461fc58af..513ebb2707 100644
--- a/frontend/components/text-classifier/labeling-rules/RulesManagement.vue
+++ b/frontend/components/text-classifier/labeling-rules/RulesManagement.vue
@@ -79,7 +79,7 @@ export default {
           class: "table-info__title",
           type: "action",
         },
-        { name: "Label", field: "label", class: "text" },
+        { name: "Labels", field: "labels", class: "array", type: "array" },
         {
           name: "Coverage",
           field: "coverage",
@@ -158,7 +158,7 @@ export default {
             name: r.description,
             query: r.query,
             kind: "select",
-            label: r.label,
+            labels: r.labels,
             ...this.metricsForRule(r),
             created_at: r.created_at,
           };
@@ -244,7 +244,7 @@ export default {
 <style lang="scss" scoped>
 .rules-management {
   padding-left: 4em;
-  padding-top: 7em;
+  padding-top: 2em;
   overflow: auto;
   height: 100vh;
   &__container {
diff --git a/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue b/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue
index ab6847c04e..24312d12a9 100644
--- a/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue
+++ b/frontend/components/text-classifier/labeling-rules/RulesMetrics.vue
@@ -173,38 +173,6 @@ export default {
     formatNumber(value) {
       return isNaN(value) ? "-" : this.$options.filters.percent(value);
     },
-    async getMetrics() {
-      this.metricsTotal = await this.getRulesMetrics({
-        dataset: this.dataset,
-      });
-    },
-    async getMetricsByRules() {
-      const responses = await Promise.all(
-        this.rules.map((rule) => {
-          return this.getRuleruleMetrics({
-            dataset: this.dataset,
-            query: rule.query,
-            label: rule.label,
-          });
-        })
-      );
-
-      responses.forEach((response, idx) => {
-        this.metricsByRules[this.rules[idx].query] = response;
-      });
-    },
-    async getruleMetrics() {
-      if (this.query !== undefined) {
-        const response = await this.getRuleruleMetrics({
-          dataset: this.dataset,
-          query: this.query,
-          label: this.activeLabel,
-        });
-        this.ruleMetrics = response;
-      } else {
-        this.ruleMetrics = {};
-      }
-    },
     getValuesByMetricType(type) {
       return Object.keys(this.metricsByRules).map((key) => {
         return this.metricsByRules[key][type] || 0;
diff --git a/frontend/models/TextClassification.js b/frontend/models/TextClassification.js
index fb1edb58e9..bef32b8378 100644
--- a/frontend/models/TextClassification.js
+++ b/frontend/models/TextClassification.js
@@ -17,6 +17,7 @@
 
 import { ObservationDataset, USER_DATA_METADATA_KEY } from "./Dataset";
 import { BaseRecord, BaseSearchQuery, BaseSearchResults } from "./Common";
+import _ from "lodash";
 
 class TextClassificationRecord extends BaseRecord {
   inputs;
@@ -90,6 +91,7 @@ class TextClassificationDataset extends ObservationDataset {
       // Some kind of metrics fields
       globalResults: this.attr({}),
       // labeling rules fields
+      isMultiLabel: this.boolean(false),
       rules: this.attr(null),
       rulesOveralMetrics: this.attr(null),
       perRuleQueryMetrics: this.attr(null),
@@ -105,6 +107,7 @@ class TextClassificationDataset extends ObservationDataset {
   async initialize() {
     const { labels } = await this.fetchMetricSummary("dataset_labels");
     const entity = this.getTaskDatasetClass();
+    const isMultiLabel = this.results.records.some((r) => r.multi_label);
     await entity.insertOrUpdate({
       where: this.id,
       data: [
@@ -112,6 +115,7 @@ class TextClassificationDataset extends ObservationDataset {
           owner: this.owner,
           name: this.name,
           _labels: labels,
+          isMultiLabel,
         },
       ],
     });
@@ -191,10 +195,10 @@ class TextClassificationDataset extends ObservationDataset {
     return overalMetrics;
   }
 
-  async _persistRule({ query, label, description }) {
-    const { response } = await TextClassificationDataset.api().post(
+  async _persistRule({ query, labels, description }) {
+    let { response } = await TextClassificationDataset.api().post(
       `/datasets/${this.task}/${this.name}/labeling/rules`,
-      { query, label, description },
+      { query, labels, description },
       {
         // Ignore errors related to rule not found
         validateStatus: function (status) {
@@ -203,17 +207,21 @@ class TextClassificationDataset extends ObservationDataset {
       }
     );
     if (response.status === 409) {
-      await TextClassificationDataset.api().patch(
+      const apiResult = await TextClassificationDataset.api().patch(
         `/datasets/${this.task}/${this.name}/labeling/rules/${query}`,
-        { label, description }
+        { labels, description }
       );
+      response = apiResult.response;
     }
+
+    return response.data;
   }
 
-  async _fetchRuleMetrics({ query, label }) {
+  async _fetchRuleMetrics({ query, labels }) {
     var url = `/datasets/${this.task}/${this.name}/labeling/rules/${query}/metrics`;
-    if (label !== undefined) {
-      url += `?label=${label}`;
+    if (labels !== undefined) {
+      const urlLabels = labels.map((label) => `label=${label}`);
+      url += `?${urlLabels.join("&")}`;
     }
     const { response } = await TextClassificationDataset.api().get(url);
 
@@ -265,10 +273,6 @@ class TextClassificationDataset extends ObservationDataset {
     });
   }
 
-  get isMultiLabel() {
-    return this.results.records.some((r) => r.multi_label);
-  }
-
   get labels() {
     const { labels } = (this.metadata || {})[USER_DATA_METADATA_KEY] || {};
     const aggregations = this.globalResults.aggregations;
@@ -322,16 +326,16 @@ class TextClassificationDataset extends ObservationDataset {
     return this.activeRuleMetrics === null ? undefined : this.activeRuleMetrics;
   }
 
-  async setCurrentLabelingRule({ query, label }) {
+  async setCurrentLabelingRule({ query, labels }) {
     if (
       this.currentLabelingRule &&
       query === this.currentLabelingRule.query &&
-      label === this.currentLabelingRule.label
+      _.isEqual(_.sortBy(labels), _.sortBy(this.currentLabelingRule.labels))
     ) {
       return;
     }
 
-    let rule = this.findRuleByQuery(query, label);
+    let rule = this.findRuleByQuery(query, labels);
     let ruleMetrics = this.getMetricsByRule(rule);
 
     await TextClassificationDataset.insertOrUpdate({
@@ -340,24 +344,27 @@ class TextClassificationDataset extends ObservationDataset {
         name: this.name,
         activeRule: rule || {
           query,
-          label,
+          labels,
           description: query,
         },
         activeRuleMetrics:
-          ruleMetrics || (await this._fetchRuleMetrics({ query, label })),
+          ruleMetrics || (await this._fetchRuleMetrics({ query, labels })),
       },
     });
   }
 
   async storeLabelingRule(activeRule) {
-    await this._persistRule(activeRule);
+    let rule = await this._persistRule(activeRule);
 
     const rules = this.rules.filter((rule) => rule.query !== activeRule.query);
     const perRuleQueryMetrics = {
       ...this.perRuleQueryMetrics,
       [activeRule.query]: this.activeRuleMetrics,
     };
-    let rule = await this._getRule({ query: activeRule.query });
+    if (rule === undefined) {
+      rule = await this._getRule({ query: activeRule.query });
+    }
+
     rules.push(rule);
 
     const overalMetrics = await this._fetchOveralMetrics(
@@ -413,9 +420,10 @@ class TextClassificationDataset extends ObservationDataset {
     });
   }
 
-  findRuleByQuery(query, label = undefined) {
+  findRuleByQuery(query, labels = undefined) {
     for (let rule of this.labelingRules || []) {
-      if (rule.query === query && [rule.label, undefined].includes(label)) {
+      const labelsAreEqual = _.isEqual(_.sortBy(rule.labels), _.sortBy(labels));
+      if (rule.query === query && (!labels || labelsAreEqual)) {
         return rule;
       }
     }
diff --git a/src/rubrix/client/sdk/text_classification/models.py b/src/rubrix/client/sdk/text_classification/models.py
index 8c8a25deb7..0fef3febb8 100644
--- a/src/rubrix/client/sdk/text_classification/models.py
+++ b/src/rubrix/client/sdk/text_classification/models.py
@@ -182,7 +182,8 @@ class LabelingRule(BaseModel):
 
     """
 
-    label: str
+    label: str = None
+    labels: List[str] = Field(default_factory=list)
     query: str
     description: Optional[str] = None
     author: str
diff --git a/src/rubrix/labeling/text_classification/__init__.py b/src/rubrix/labeling/text_classification/__init__.py
index b083a134f6..42bf473e30 100644
--- a/src/rubrix/labeling/text_classification/__init__.py
+++ b/src/rubrix/labeling/text_classification/__init__.py
@@ -16,4 +16,4 @@
 from .label_errors import find_label_errors
 from .label_models import FlyingSquid, Snorkel
 from .rule import Rule, load_rules
-from .weak_labels import WeakLabels
+from .weak_labels import WeakLabels, WeakMultiLabels
diff --git a/src/rubrix/labeling/text_classification/rule.py b/src/rubrix/labeling/text_classification/rule.py
index 7a000d979f..57e959ea8c 100644
--- a/src/rubrix/labeling/text_classification/rule.py
+++ b/src/rubrix/labeling/text_classification/rule.py
@@ -16,7 +16,7 @@
 
 import rubrix as rb
 from rubrix import TextClassificationRecord
-from rubrix import _client_instance as client
+from rubrix.client import api
 from rubrix.client.sdk.text_classification.models import LabelingRule
 
 
@@ -25,7 +25,7 @@ class Rule:
 
     Args:
         query: An ElasticSearch query with the `query string syntax <https://rubrix.readthedocs.io/en/stable/reference/webapp/search_records.html>`_.
-        label: The label associated to the query.
+        label: The label associated to the query. Can also be a list of labels.
         name: An optional name for the rule to be used as identifier in the
             `rubrix.labeling.text_classification.WeakLabels` class. By default, we will use the ``query`` string.
 
@@ -42,7 +42,7 @@ class Rule:
     def __init__(
         self,
         query: str,
-        label: str,
+        label: Union[str, List[str]],
         name: Optional[str] = None,
         author: Optional[str] = None,
     ):
@@ -58,7 +58,7 @@ def query(self) -> str:
         return self._query
 
     @property
-    def label(self) -> str:
+    def label(self) -> Union[str, List[str]]:
         """The rule label"""
         return self._label
 
@@ -99,8 +99,7 @@ def metrics(self, dataset: str) -> Dict[str, Union[int, float]]:
         Returns:
             The rule metrics.
         """
-        current_client = client()
-        metrics = current_client.rule_metrics_for_dataset(
+        metrics = api.active_api().rule_metrics_for_dataset(
             dataset=dataset,
             rule=LabelingRule(
                 query=self.query, label=self.label, author=self.author or "None"
@@ -117,14 +116,16 @@ def metrics(self, dataset: str) -> Dict[str, Union[int, float]]:
             "precision": metrics.precision if metrics.precision is not None else None,
         }
 
-    def __call__(self, record: TextClassificationRecord) -> Optional[str]:
+    def __call__(
+        self, record: TextClassificationRecord
+    ) -> Optional[Union[str, List[str]]]:
         """Check if the given record is among the matching ids from the ``self.apply`` call.
 
         Args:
             record: The record to be labelled.
 
         Returns:
-            A label if the record id is among the matching ids, otherwise None.
+            A label or list of labels if the record id is among the matching ids, otherwise None.
 
         Raises:
             RuleNotAppliedError: If the rule was not applied to the dataset before.
@@ -151,12 +152,11 @@ def load_rules(dataset: str) -> List[Rule]:
     Returns:
         A list of rules defined in the given dataset.
     """
-    current_client = client()
-    rules = current_client.fetch_dataset_labeling_rules(dataset)
+    rules = api.active_api().fetch_dataset_labeling_rules(dataset)
     return [
         Rule(
             query=rule.query,
-            label=rule.label,
+            label=rule.label or rule.labels,
             name=rule.description,
             author=rule.author,
         )
diff --git a/src/rubrix/labeling/text_classification/weak_labels.py b/src/rubrix/labeling/text_classification/weak_labels.py
index 0a2f400d7d..6403c75af1 100644
--- a/src/rubrix/labeling/text_classification/weak_labels.py
+++ b/src/rubrix/labeling/text_classification/weak_labels.py
@@ -26,8 +26,10 @@
 from rubrix.labeling.text_classification.rule import Rule, load_rules
 
 
-class WeakLabels:
-    """Computes the weak labels of a dataset by applying a given list of rules.
+class WeakLabelsBase:
+    """Base class for the weak label classes.
+
+    Tries to facilitate the implementations, avoids code duplications.
 
     Args:
         dataset: Name of the dataset to which the rules will be applied.
@@ -37,42 +39,11 @@ class WeakLabels:
         query: An optional ElasticSearch query with the
             `query string syntax <https://rubrix.readthedocs.io/en/stable/reference/webapp/search_records.html>`_
             to filter the dataset before applying the rules.
-        label2int: An optional dict, mapping the labels to integers. Remember that the return type ``None`` means
-            abstention (e.g. ``{None: -1}``). By default, we will build a mapping on the fly when applying the rules.
 
     Raises:
         NoRulesFoundError: When you do not provide rules, and the dataset has no rules either.
         DuplicatedRuleNameError: When you provided multiple rules with the same name.
         NoRecordsFoundError: When the filtered dataset is empty.
-        MultiLabelError: When trying to get weak labels for a multi-label text classification task.
-        MissingLabelError: When provided with a ``label2int`` dict, and a
-            weak label or annotation label is not present in its keys.
-
-    Examples:
-        Get the weak label matrix from a dataset with rules:
-
-        >>> weak_labels = WeakLabels(dataset="my_dataset")
-        >>> weak_labels.matrix()
-        >>> weak_labels.summary()
-
-        Get the weak label matrix from rules defined in Python:
-
-        >>> def awesome_rule(record: TextClassificationRecord) -> str:
-        ...     return "Positive" if "awesome" in record.inputs["text"] else None
-        >>> another_rule = Rule(query="good OR best", label="Positive")
-        >>> weak_labels = WeakLabels(rules=[awesome_rule, another_rule], dataset="my_dataset")
-        >>> weak_labels.matrix()
-        >>> weak_labels.summary()
-
-        Use the WeakLabels object with snorkel's LabelModel:
-
-        >>> from snorkel.labeling.model import LabelModel
-        >>> label_model = LabelModel()
-        >>> label_model.fit(L_train=weak_labels.matrix(has_annotation=False))
-        >>> label_model.score(L=weak_labels.matrix(has_annotation=True), Y=weak_labels.annotation())
-        >>> label_model.predict(L=weak_labels.matrix(has_annotation=False))
-
-        For a builtin integration with Snorkel, see `rubrix.labeling.text_classification.Snorkel`.
     """
 
     def __init__(
@@ -81,7 +52,6 @@ def __init__(
         rules: Optional[List[Callable]] = None,
         ids: Optional[List[Union[int, str]]] = None,
         query: Optional[str] = None,
-        label2int: Optional[Dict[Optional[str], int]] = None,
     ):
         if not isinstance(dataset, str):
             raise TypeError(
@@ -90,7 +60,7 @@ def __init__(
         self._dataset = dataset
 
         self._rules = rules or load_rules(dataset)
-        if self._rules == []:
+        if not self._rules:
             raise NoRulesFoundError(
                 f"No rules were found in the given dataset '{dataset}'"
             )
@@ -132,20 +102,197 @@ def __init__(
                 + (f" with ids {ids}." if ids else ".")
             )
 
+    @property
+    def rules(self) -> List[Callable]:
+        """The rules (labeling functions) that were used to produce the weak labels."""
+        return self._rules
+
+    def records(
+        self, has_annotation: Optional[bool] = None
+    ) -> List[TextClassificationRecord]:
+        """Returns the records corresponding to the weak label matrix
+
+        Args:
+            has_annotation: If True, return only the records that have an annotation. If False, return only the records
+                that have NO annotation. By default, we return all the records.
+
+        Returns:
+            A list of records, or optionally just a part of them.
+        """
+        if has_annotation is True:
+            return [rec for rec in self._records if rec.annotation is not None]
+        if has_annotation is False:
+            return [rec for rec in self._records if rec.annotation is None]
+
+        return list(self._records)
+
+    def matrix(self, has_annotation: Optional[bool] = None) -> np.ndarray:
+        """Returns the weak label matrix, or optionally just a part of it.
+
+        Args:
+            has_annotation: If True, return only the part of the matrix that has a corresponding annotation.
+                If False, return only the part of the matrix that has NOT a corresponding annotation.
+                By default, we return the whole weak label matrix.
+
+        Returns:
+            The weak label matrix, or optionally just a part of it.
+        """
+        raise NotImplementedError
+
+    def annotation(
+        self,
+        include_missing: bool = False,
+    ) -> np.ndarray:
+        """Returns the annotation labels.
+
+        Args:
+            include_missing: If True, returns an array of the length of the record list (``self.records()``).
+
+        Returns:
+            The annotation labels.
+        """
+        raise NotImplementedError
+
+    def summary(
+        self,
+        normalize_by_coverage: bool = False,
+        annotation: Optional[np.ndarray] = None,
+    ) -> pd.DataFrame:
+        """Returns a summary statistics for each rule:
+
+        Args:
+            normalize_by_coverage: Normalize the overlaps and conflicts by the respective coverage.
+            annotation: An optional array holding the annotations.
+                By default, we will use ``self.annotation(include_missing=True)``.
+
+        Returns:
+            The summary statistics for each rule in a pandas DataFrame.
+        """
+        raise NotImplementedError
+
+    def show_records(
+        self,
+        labels: Optional[List[str]] = None,
+        rules: Optional[List[Union[str, int]]] = None,
+    ) -> pd.DataFrame:
+        """Shows records in a pandas DataFrame, optionally filtered by weak labels and non-abstaining rules.
+
+        If you provide both ``labels`` and ``rules``, we take the intersection of both filters.
+
+        Args:
+            labels: All of these labels are in the record's weak labels. If None, do not filter by labels.
+            rules: All of these rules did not abstain for the record. If None, do not filter by rules.
+                You can refer to the rules by their (function) name or by their index in the ``self.rules`` list.
+
+        Returns:
+            The optionally filtered records as a pandas DataFrame.
+        """
+        raise NotImplementedError
+
+    def _compute_overlaps_conflicts(
+        self,
+        has_weak_label: np.ndarray,
+        has_overlaps_or_conflicts: np.ndarray,
+        coverage: np.ndarray,
+        normalize_by_coverage: bool,
+    ) -> np.ndarray:
+        """Helper method to compute the overlaps/conflicts and optionally normalize them by the respective coverage
+
+        Args:
+            has_weak_label: 2D boolean matrix (i, j) that indicates if the record i has a weak label from rule j.
+            has_overlaps_or_conflicts: Array that indicates if the record has overlapping/conflicting weak labels.
+            coverage: Array of coverages for each rule
+            normalize_by_coverage: Normalize the overlaps/conflicts by the respective coverage.
+
+        Returns:
+            Array of fractions of overlaps/conflicts for each rule, optionally normalized by their coverages.
+        """
+        overlaps_or_conflicts = (
+            has_weak_label
+            * np.repeat(has_overlaps_or_conflicts, len(self._rules)).reshape(
+                has_weak_label.shape
+            )
+        ).sum(axis=0) / len(self._records)
+        # total
+        overlaps_or_conflicts = np.append(
+            overlaps_or_conflicts, has_overlaps_or_conflicts.sum() / len(self._records)
+        )
+
+        if normalize_by_coverage:
+            overlaps_or_conflicts /= coverage
+            return np.nan_to_num(overlaps_or_conflicts)
+
+        return overlaps_or_conflicts
+
+
+class WeakLabels(WeakLabelsBase):
+    """Computes the weak labels of a single-label text classification dataset by applying a given list of rules.
+
+    Args:
+        dataset: Name of the dataset to which the rules will be applied.
+        rules: A list of rules (labeling functions). They must return a string, or ``None`` in case of abstention.
+            If None, we will use the rules of the dataset (Default).
+        ids: An optional list of record ids to filter the dataset before applying the rules.
+        query: An optional ElasticSearch query with the
+            `query string syntax <https://rubrix.readthedocs.io/en/stable/reference/webapp/search_records.html>`_
+            to filter the dataset before applying the rules.
+        label2int: An optional dict, mapping the labels to integers. Remember that the return type ``None`` means
+            abstention (e.g. ``{None: -1}``). By default, we will build a mapping on the fly when applying the rules.
+
+    Raises:
+        NoRulesFoundError: When you do not provide rules, and the dataset has no rules either.
+        DuplicatedRuleNameError: When you provided multiple rules with the same name.
+        NoRecordsFoundError: When the filtered dataset is empty.
+        MultiLabelError: When trying to get weak labels for a multi-label text classification task.
+        MissingLabelError: When provided with a ``label2int`` dict, and a
+            weak label or annotation label is not present in its keys.
+
+    Examples:
+        >>> # Get the weak label matrix from a dataset with rules:
+        >>> weak_labels = WeakLabels(dataset="my_dataset")
+        >>> weak_labels.matrix()
+        >>> weak_labels.summary()
+        >>>
+        >>> # Get the weak label matrix from rules defined in Python:
+        >>> def awesome_rule(record: TextClassificationRecord) -> str:
+        ...     return "Positive" if "awesome" in record.inputs["text"] else None
+        >>> another_rule = Rule(query="good OR best", label="Positive")
+        >>> weak_labels = WeakLabels(dataset="my_dataset", rules=[awesome_rule, another_rule])
+        >>> weak_labels.matrix()
+        >>> weak_labels.summary()
+        >>>
+        >>> # Use the WeakLabels object with snorkel's LabelModel:
+        >>> from snorkel.labeling.model import LabelModel
+        >>> label_model = LabelModel()
+        >>> label_model.fit(L_train=weak_labels.matrix(has_annotation=False))
+        >>> label_model.score(L=weak_labels.matrix(has_annotation=True), Y=weak_labels.annotation())
+        >>> label_model.predict(L=weak_labels.matrix(has_annotation=False))
+        >>>
+        >>> # For a builtin integration with Snorkel, see `rubrix.labeling.text_classification.Snorkel`.
+    """
+
+    def __init__(
+        self,
+        dataset: str,
+        rules: Optional[List[Callable]] = None,
+        ids: Optional[List[Union[int, str]]] = None,
+        query: Optional[str] = None,
+        label2int: Optional[Dict[Optional[str], int]] = None,
+    ):
+        super().__init__(dataset=dataset, rules=rules, ids=ids, query=query)
+
         if self._records[0].multi_label:
             raise MultiLabelError(
-                "Multi-label text classification is not yet supported."
+                "For multi-label text classification, use the `rb.labeling.text_classification.WeakMultiLabels` class."
             )
 
         # apply rules -> create the weak label matrix, annotation array, final label2int mapping
-        self._matrix, self._annotation_array, self._label2int = self._apply_rules(
-            label2int
-        )
+        self._matrix, self._annotation, self._label2int = self._apply_rules(label2int)
         self._int2label = {v: k for k, v in self._label2int.items()}
 
     def _apply_rules(
         self, label2int: Optional[Dict[str, int]]
-    ) -> Tuple[np.ndarray, np.ndarray, Dict[str, int]]:
+    ) -> Tuple[np.ndarray, np.ndarray, Dict[Optional[str], int]]:
         """Apply the rules to the dataset.
 
         Args:
@@ -177,7 +324,7 @@ def _apply_rules(
         for n, record in tqdm(
             enumerate(self._records), total=len(self._records), desc="Applying rules"
         ):
-            # First: fill annotation array
+            # FIRST: fill annotation array
             try:
                 annotation = _label2int[record.annotation]
             except KeyError as error:
@@ -192,9 +339,15 @@ def _apply_rules(
 
             annotation_array[n] = annotation
 
-            # Second: fill weak label matrix
+            # SECOND: fill weak label matrix
             for m, rule in enumerate(self._rules):
                 weak_label = rule(record)
+                if isinstance(weak_label, list):
+                    if len(weak_label) != 1:
+                        raise MultiLabelError(
+                            "For rules that do not return exactly 1 label, use the `WeakMultiLabels` class."
+                        )
+                    weak_label = weak_label[0]
 
                 try:
                     weak_label = _label2int[weak_label]
@@ -211,11 +364,6 @@ def _apply_rules(
 
         return weak_label_matrix, annotation_array, _label2int
 
-    @property
-    def rules(self) -> List[Callable]:
-        """The rules (labeling functions) that were used to produce the weak labels."""
-        return self._rules
-
     @property
     def label2int(self) -> Dict[Optional[str], int]:
         """The dictionary that maps weak/annotation labels to integers."""
@@ -238,31 +386,12 @@ def matrix(self, has_annotation: Optional[bool] = None) -> np.ndarray:
             The weak label matrix, or optionally just a part of it.
         """
         if has_annotation is True:
-            return self._matrix[self._annotation_array != self._label2int[None]]
+            return self._matrix[self._annotation != self._label2int[None]]
         if has_annotation is False:
-            return self._matrix[self._annotation_array == self._label2int[None]]
+            return self._matrix[self._annotation == self._label2int[None]]
 
         return self._matrix
 
-    def records(
-        self, has_annotation: Optional[bool] = None
-    ) -> List[TextClassificationRecord]:
-        """Returns the records corresponding to the weak label matrix.
-
-        Args:
-            has_annotation: If True, return only the records that have an annotation. If False, return only the records
-                that have NO annotation. By default, we return all the records.
-
-        Returns:
-            A list of records, or optionally just a part of them.
-        """
-        if has_annotation is True:
-            return [rec for rec in self._records if rec.annotation is not None]
-        if has_annotation is False:
-            return [rec for rec in self._records if rec.annotation is None]
-
-        return self._records
-
     def annotation(
         self,
         include_missing: bool = False,
@@ -272,7 +401,7 @@ def annotation(
 
         Args:
             include_missing: If True, returns an array of the length of the record list (``self.records()``).
-                For this we will fill the array with the ``self.label2int[None]`` integer for records without an annotation.
+                For this, we will fill the array with the ``self.label2int[None]`` integer for records without an annotation.
             exclude_missing_annotations: DEPRECATED
 
         Returns:
@@ -287,9 +416,9 @@ def annotation(
             include_missing = not exclude_missing_annotations
 
         if include_missing:
-            return self._annotation_array
+            return self._annotation
 
-        return self._annotation_array[self._annotation_array != self._label2int[None]]
+        return self._annotation[self._annotation != self._label2int[None]]
 
     def summary(
         self,
@@ -311,11 +440,12 @@ def summary(
         Args:
             normalize_by_coverage: Normalize the overlaps and conflicts by the respective coverage.
             annotation: An optional array with ints holding the annotations.
-                By default we will use ``self.annotation(exclude_missing_annotations=False)``.
+                By default, we will use ``self.annotation(include_missing=True)``.
 
         Returns:
             The summary statistics for each rule in a pandas DataFrame.
         """
+        annotation = annotation if annotation is not None else self._annotation
         has_weak_label = self._matrix != self._label2int[None]
 
         # polarity (label)
@@ -358,14 +488,9 @@ def summary(
         index = list(self._rules_name2index.keys()) + ["total"]
 
         # only add annotated_coverage, correct, incorrect and precision if we have annotations
-        if (
-            any(self._annotation_array != self._label2int[None])
-            or annotation is not None
-        ):
+        has_annotation = annotation != self._label2int[None]
+        if any(has_annotation):
             # annotated coverage
-            has_annotation = (
-                annotation if annotation is not None else self._annotation_array
-            ) != self._label2int[None]
             annotated_coverage = (
                 has_weak_label[has_annotation].sum(axis=0) / has_annotation.sum()
             )
@@ -378,7 +503,7 @@ def summary(
             # correct/incorrect
             correct, incorrect = self._compute_correct_incorrect(
                 has_weak_label,
-                annotation if annotation is not None else self._annotation_array,
+                annotation if annotation is not None else self._annotation,
             )
 
             # precision
@@ -408,31 +533,6 @@ def summary(
             index=index,
         )
 
-    def _compute_overlaps_conflicts(
-        self,
-        has_weak_label: np.ndarray,
-        has_overlaps_or_conflicts: np.ndarray,
-        coverage: np.ndarray,
-        normalize_by_coverage: bool,
-    ) -> np.ndarray:
-        """Helper method to compute the overlaps/conflicts and optionally normalize them by the respective coverage"""
-        overlaps_or_conflicts = (
-            has_weak_label
-            * np.repeat(has_overlaps_or_conflicts, len(self._rules)).reshape(
-                has_weak_label.shape
-            )
-        ).sum(axis=0) / len(self._records)
-        # total
-        overlaps_or_conflicts = np.append(
-            overlaps_or_conflicts, has_overlaps_or_conflicts.sum() / len(self._records)
-        )
-
-        if normalize_by_coverage:
-            overlaps_or_conflicts /= coverage
-            return np.nan_to_num(overlaps_or_conflicts)
-
-        return overlaps_or_conflicts
-
     def _compute_correct_incorrect(
         self, has_weak_label: np.ndarray, annotation: np.ndarray
     ) -> Tuple[np.ndarray, np.ndarray]:
@@ -517,18 +617,331 @@ def change_mapping(self, label2int: Dict[str, int]):
                 )
             # compute masks
             label_masks[label] = self._matrix == self._label2int[label]
-            annotation_masks[label] = self._annotation_array == self._label2int[label]
+            annotation_masks[label] = self._annotation == self._label2int[label]
 
         # swap integers
         for label in self._label2int:
             self._matrix[label_masks[label]] = label2int[label]
-            self._annotation_array[annotation_masks[label]] = label2int[label]
+            self._annotation[annotation_masks[label]] = label2int[label]
 
         # update mapping dicts
         self._label2int = label2int.copy()
         self._int2label = {val: key for key, val in self._label2int.items()}
 
 
+class WeakMultiLabels(WeakLabelsBase):
+    """Computes the weak labels of a multi-label text classification dataset by applying a given list of rules.
+
+    Args:
+        dataset: Name of the dataset to which the rules will be applied.
+        rules: A list of rules (labeling functions). They must return a string, or ``None`` in case of abstention.
+            If None, we will use the rules of the dataset (Default).
+        ids: An optional list of record ids to filter the dataset before applying the rules.
+        query: An optional ElasticSearch query with the
+            `query string syntax <https://rubrix.readthedocs.io/en/stable/reference/webapp/search_records.html>`_
+            to filter the dataset before applying the rules.
+
+    Raises:
+        NoRulesFoundError: When you do not provide rules, and the dataset has no rules either.
+        DuplicatedRuleNameError: When you provided multiple rules with the same name.
+        NoRecordsFoundError: When the filtered dataset is empty.
+
+    Examples:
+        >>> # Get the 3 dimensional weak label matrix from a multi-label classification dataset with rules:
+        >>> weak_labels = WeakMultiLabels(dataset="my_dataset")
+        >>> weak_labels.matrix()
+        >>> weak_labels.summary()
+        >>>
+        >>> # Get the 3 dimensional weak label matrix from rules defined in Python:
+        >>> def awesome_rule(record: TextClassificationRecord) -> str:
+        ...     return ["Positive", "Slang"] if "next level" in record.inputs["text"] else None
+        >>> another_rule = Rule(query="amped OR psyched", label=["Positive", "Slang"])
+        >>> weak_labels = WeakMultiLabels(dataset="my_dataset", rules=[awesome_rule, another_rule])
+        >>> weak_labels.matrix()
+        >>> weak_labels.summary()
+    """
+
+    def __init__(
+        self,
+        dataset: str,
+        rules: Optional[List[Callable]] = None,
+        ids: Optional[List[Union[int, str]]] = None,
+        query: Optional[str] = None,
+    ):
+        super().__init__(dataset=dataset, rules=rules, ids=ids, query=query)
+
+        # apply rules -> create the weak label matrix (3D), annotation (2D), label list
+        self._matrix, self._annotation, self._labels = self._apply_rules()
+
+    def _apply_rules(self) -> Tuple[np.ndarray, np.ndarray, List[str]]:
+        # call apply on the ElasticSearch rules
+        for rule in tqdm(self._rules, desc="Preparing rules"):
+            if isinstance(rule, Rule):
+                rule.apply(self._dataset)
+
+        # we make two passes over the records:
+        # FIRST: Get labels from rules and annotations
+        annotations, weak_labels = [], []
+        for record in tqdm(
+            self._records, total=len(self._records), desc="Applying rules"
+        ):
+            annotations.append(
+                record.annotation
+                if isinstance(record.annotation, list)
+                else [record.annotation]
+            )
+            weak_labels.append([np.atleast_1d(rule(record)) for rule in self._rules])
+
+        annotation_set = {ann for anns in annotations for ann in anns}
+        weak_label_set = {
+            wl for wl_record in weak_labels for wl_rule in wl_record for wl in wl_rule
+        }
+        labels = sorted(list(annotation_set.union(weak_label_set) - {None}))
+
+        # create weak label matrix (3D), annotation matrix
+        weak_label_matrix = np.empty(
+            (len(self._records), len(self._rules), len(labels)), dtype=np.short
+        )
+        annotation_matrix = np.empty((len(self._records), len(labels)), dtype=np.short)
+
+        # SECOND: Fill arrays with weak labels
+        for n, annotation_n, weak_label_n in tqdm(
+            zip(range(len(annotations)), annotations, weak_labels),
+            desc="Filling weak label matrix",
+        ):
+            # first: fill annotation matrix
+            if annotation_n == [None]:
+                # "abstain" is an array with -1
+                annotation_matrix[n] = -1 * np.ones(len(labels), dtype=np.short)
+            else:
+                annotation_matrix[n] = np.array(
+                    [1 if label in annotation_n else 0 for label in labels],
+                    dtype=np.short,
+                )
+
+            # second: fill weak label matrix (3D)
+            for m, weak_labels_m in enumerate(weak_label_n):
+                if weak_labels_m.tolist() == [None]:
+                    weak_label_matrix[n, m] = -1 * np.ones(len(labels))
+                else:
+                    weak_label_matrix[n, m] = np.array(
+                        [1 if label in weak_labels_m else 0 for label in labels],
+                        dtype=np.short,
+                    )
+
+        return weak_label_matrix, annotation_matrix, labels
+
+    @property
+    def labels(self) -> List[str]:
+        """The labels of the multi-label text classification dataset."""
+        return self._labels
+
+    def matrix(self, has_annotation: Optional[bool] = None) -> np.ndarray:
+        """Returns the 3 dimensional weak label matrix, or optionally just a part of it.
+
+        It has the dimensions ("nr of record" x "nr of rules" x "nr of labels").
+        It holds a 1 or 0 in case a rule votes for a label or not. If the rule abstains, it holds a -1 for each label.
+
+        Args:
+            has_annotation: If True, return only the part of the matrix that has a corresponding annotation.
+                If False, return only the part of the matrix that has NOT a corresponding annotation.
+                By default, we return the whole weak label matrix.
+
+        Returns:
+            The 3 dimensional weak label matrix, or optionally just a part of it.
+        """
+        if has_annotation is True:
+            return self._matrix[self._annotation.sum(1) >= 0]
+        if has_annotation is False:
+            return self._matrix[self._annotation.sum(1) < 0]
+
+        return self._matrix
+
+    def annotation(
+        self,
+        include_missing: bool = False,
+    ) -> np.ndarray:
+        """Returns the annotation labels as a matrix of integers.
+
+        It has the dimensions ("nr of record" x "nr of labels").
+        It holds a 1 or 0 to indicate if the record is annotated with the corresponding label.
+        In case there is no annotation for the record, it holds a -1 for each label.
+
+        Args:
+            include_missing: If True, returns a matrix of the length of the record list (``self.records()``).
+                For this, we will fill the matrix with -1 for records without an annotation.
+
+        Returns:
+            The annotation labels as a matrix of integers.
+        """
+        if include_missing:
+            return self._annotation
+
+        return self._annotation[self._annotation.sum(1) >= 0]
+
+    def summary(
+        self,
+        normalize_by_coverage: bool = False,
+        annotation: Optional[np.ndarray] = None,
+    ) -> pd.DataFrame:
+        """Returns following summary statistics for each rule:
+
+        - **label**: Set of unique labels returned by the rule, excluding "None" (abstain).
+        - **coverage**: Fraction of the records labeled by the rule.
+        - **annotated_coverage**: Fraction of annotated records labeled by the rule (if annotations are available).
+        - **overlaps**: Fraction of the records labeled by the rule together with at least one other rule.
+        - **correct**: Number of labels the rule labeled correctly (if annotations are available).
+        - **incorrect**: Number of labels the rule labeled incorrectly or missed (if annotations are available).
+        - **precision**: Fraction of correct labels given by the rule (if annotations are available). The precision does not penalize the rule for abstains.
+
+        Args:
+            normalize_by_coverage: Normalize the overlaps by the respective coverage.
+            annotation: An optional matrix with ints holding the annotations (see ``self.annotation``).
+                By default, we will use ``self.annotation(include_missing=True)``.
+
+        Returns:
+            The summary statistics for each rule in a pandas DataFrame.
+        """
+        annotation = annotation if annotation is not None else self._annotation
+        has_weak_label = self._matrix.sum(2) >= 0
+
+        # polarity (label)
+        polarity = [
+            set(
+                [
+                    self._labels[i]
+                    # get indices of votes
+                    for i in np.nonzero(
+                        # remove abstentions
+                        self._matrix[:, m, :][self._matrix[:, m, :].sum(1) >= 0]
+                    )[1]
+                ]
+            )
+            for m in range(len(self._rules))
+        ]
+        polarity.append(set().union(*polarity))
+
+        # coverage
+        coverage = has_weak_label.sum(axis=0) / len(self._records)
+        coverage = np.append(
+            coverage,
+            (has_weak_label.sum(axis=1) > 0).sum() / len(self._records),
+        )
+
+        # overlaps
+        has_overlaps = has_weak_label.sum(axis=1) > 1
+        overlaps = self._compute_overlaps_conflicts(
+            has_weak_label, has_overlaps, coverage, normalize_by_coverage
+        )
+
+        # index for the summary
+        index = list(self._rules_name2index.keys()) + ["total"]
+
+        # only add annotated_coverage, correct, incorrect and precision if we have annotations
+        has_annotation = annotation.sum(1) >= 0
+        if any(has_annotation):
+            # annotated coverage
+            annotated_coverage = (
+                has_weak_label[has_annotation].sum(axis=0) / has_annotation.sum()
+            )
+            annotated_coverage = np.append(
+                annotated_coverage,
+                (has_weak_label[has_annotation].sum(axis=1) > 0).sum()
+                / has_annotation.sum(),
+            )
+
+            # correct/incorrect
+            correct, incorrect = self._compute_correct_incorrect(annotation)
+
+            # precision
+            precision = correct / (correct + incorrect)
+
+            return pd.DataFrame(
+                {
+                    "label": polarity,
+                    "coverage": coverage,
+                    "annotated_coverage": annotated_coverage,
+                    "overlaps": overlaps,
+                    "correct": correct,
+                    "incorrect": incorrect,
+                    "precision": precision,
+                },
+                index=index,
+            )
+
+        return pd.DataFrame(
+            {
+                "label": polarity,
+                "coverage": coverage,
+                "overlaps": overlaps,
+            },
+            index=index,
+        )
+
+    def _compute_correct_incorrect(
+        self, annotation: np.ndarray
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        """Helper method to compute the correctly and incorrectly predicted annotations by the rules"""
+        # transform annotation to tensor
+        annotation = np.repeat(annotation, len(self._rules), axis=0).reshape(
+            self._matrix.shape
+        )
+
+        # correct, we don't want to count the "correct non predictions"
+        correct = ((annotation == self._matrix) & (self._matrix == 1)).sum(2).sum(0)
+
+        # incorrect, we don't want to count the "misses", since we focus on precision, not recall
+        incorrect = (
+            ((annotation != self._matrix) & (self._matrix == 1) & (annotation != -1))
+            .sum(2)
+            .sum(0)
+        )
+
+        # add totals at the end
+        return np.append(correct, correct.sum()), np.append(incorrect, incorrect.sum())
+
+    def show_records(
+        self,
+        labels: Optional[List[str]] = None,
+        rules: Optional[List[Union[str, int]]] = None,
+    ) -> pd.DataFrame:
+        """Shows records in a pandas DataFrame, optionally filtered by weak labels and non-abstaining rules.
+
+        If you provide both ``labels`` and ``rules``, we take the intersection of both filters.
+
+        Args:
+            labels: All of these labels are in the record's weak labels. If None, do not filter by labels.
+            rules: All of these rules did not abstain for the record. If None, do not filter by rules.
+                You can refer to the rules by their (function) name or by their index in the ``self.rules`` list.
+
+        Returns:
+            The optionally filtered records as a pandas DataFrame.
+        """
+        # get labels mask
+        if labels is not None:
+            labels = [self._labels.index(label) for label in labels]
+            idx_by_labels = np.all(((self._matrix == 1).sum(1) > 0)[:, labels], axis=1)
+        else:
+            idx_by_labels = np.ones_like(self._records).astype(bool)
+
+        # get rule mask
+        if rules is not None:
+            rules = [
+                self._rules_name2index[rule] if isinstance(rule, str) else rule
+                for rule in rules
+            ]
+            idx_by_rules = (self._matrix[:, rules, :].sum(axis=2) >= 0).sum(
+                axis=1
+            ) == len(rules)
+        else:
+            idx_by_rules = np.ones_like(self._records).astype(bool)
+
+        # apply mask
+        filtered_records = np.array(self._records)[idx_by_labels & idx_by_rules]
+
+        return pd.DataFrame(map(lambda x: x.dict(), filtered_records))
+
+
 class WeakLabelsError(Exception):
     pass
 
diff --git a/src/rubrix/server/commons/es_wrapper.py b/src/rubrix/server/commons/es_wrapper.py
index 39903c3d7d..f79b267386 100644
--- a/src/rubrix/server/commons/es_wrapper.py
+++ b/src/rubrix/server/commons/es_wrapper.py
@@ -416,9 +416,11 @@ def update_document(
 
         """
         if partial_update:
-            self.__client__.update(index=index, id=doc_id, body={"doc": document})
+            self.__client__.update(
+                index=index, id=doc_id, body={"doc": document}, refresh=True
+            )
         else:
-            self.__client__.index(index=index, id=doc_id, body=document)
+            self.__client__.index(index=index, id=doc_id, body=document, refresh=True)
 
     def open_index(self, index: str):
         """
diff --git a/src/rubrix/server/commons/helpers.py b/src/rubrix/server/commons/helpers.py
index 554e64f885..e5c0c401ac 100644
--- a/src/rubrix/server/commons/helpers.py
+++ b/src/rubrix/server/commons/helpers.py
@@ -84,7 +84,7 @@ def unflatten_dict(
 
     """
     resultDict = {}
-
+    stop_keys = stop_keys or []
     for key, value in data.items():
         if key is not None:
             parts = key.split(sep)
diff --git a/src/rubrix/server/tasks/text_classification/api/api.py b/src/rubrix/server/tasks/text_classification/api/api.py
index 4d47867fa1..85ea0aef99 100644
--- a/src/rubrix/server/tasks/text_classification/api/api.py
+++ b/src/rubrix/server/tasks/text_classification/api/api.py
@@ -326,7 +326,9 @@ async def create_rule(
 async def compute_rule_metrics(
     name: str,
     query: str,
-    label: Optional[str] = Query(None, description="Label related to query rule"),
+    labels: Optional[List[str]] = Query(
+        None, description="Label related to query rule", alias="label"
+    ),
     common_params: CommonTaskQueryParams = Depends(),
     service: TextClassificationService = Depends(
         TextClassificationService.get_instance
@@ -342,7 +344,7 @@ async def compute_rule_metrics(
     )
 
     return service.compute_rule_metrics(
-        Dataset.parse_obj(dataset), rule_query=query, label=label
+        Dataset.parse_obj(dataset), rule_query=query, labels=labels
     )
 
 
@@ -459,7 +461,7 @@ async def update_rule(
     rule = service.update_labeling_rule(
         Dataset.parse_obj(dataset),
         rule_query=query,
-        label=update.label,
+        labels=update.labels,
         description=update.description,
     )
     return rule
diff --git a/src/rubrix/server/tasks/text_classification/api/model.py b/src/rubrix/server/tasks/text_classification/api/model.py
index 3fbabef41a..d6d2288feb 100644
--- a/src/rubrix/server/tasks/text_classification/api/model.py
+++ b/src/rubrix/server/tasks/text_classification/api/model.py
@@ -37,11 +37,30 @@
 
 
 class UpdateLabelingRule(BaseModel):
-    label: str = Field(description="The label associated with the rule")
+    label: Optional[str] = Field(
+        default=None, description="@Deprecated::The label associated with the rule."
+    )
+    labels: List[str] = Field(
+        default_factory=list,
+        description="For multi label problems, a list of labels. "
+        "It will replace the `label` field",
+    )
     description: Optional[str] = Field(
         None, description="A brief description of the rule"
     )
 
+    @root_validator
+    def initialize_labels(cls, values):
+        label = values.get("label", None)
+        labels = values.get("labels", [])
+
+        if label:
+            labels.append(label)
+            values["labels"] = list(set(labels))
+
+        assert len(labels) >= 1, f"No labels was provided in rule {values}"
+        return values
+
 
 class CreateLabelingRule(UpdateLabelingRule):
     """
diff --git a/src/rubrix/server/tasks/text_classification/service/labeling_service.py b/src/rubrix/server/tasks/text_classification/service/labeling_service.py
index 49ae2ddd10..0b59f7b130 100644
--- a/src/rubrix/server/tasks/text_classification/service/labeling_service.py
+++ b/src/rubrix/server/tasks/text_classification/service/labeling_service.py
@@ -3,21 +3,17 @@
 from fastapi import Depends
 from pydantic import BaseModel, Field
 
-from rubrix.server.commons.errors import (
-    EntityAlreadyExistsError,
-    EntityNotFoundError,
-)
+from rubrix.server.commons.errors import EntityAlreadyExistsError, EntityNotFoundError
 from rubrix.server.commons.es_helpers import filters
+from rubrix.server.commons.helpers import unflatten_dict
 from rubrix.server.datasets.dao import DatasetsDAO
 from rubrix.server.datasets.model import BaseDatasetDB
-from ..api.model import (
-    LabelingRule,
-    TextClassificationDatasetDB,
-)
+
 from ...commons import EsRecordDataFieldNames
 from ...commons.dao.dao import DatasetRecordsDAO
 from ...commons.dao.model import RecordSearch
 from ...commons.metrics.model.base import ElasticsearchMetric
+from ..api.model import LabelingRule, TextClassificationDatasetDB
 
 
 class DatasetLabelingRulesMetric(ElasticsearchMetric):
@@ -55,7 +51,7 @@ class LabelingRulesMetric(ElasticsearchMetric):
     def aggregation_request(
         self,
         rule_query: str,
-        label: Optional[str],
+        labels: Optional[List[str]],
     ) -> Dict[str, Any]:
 
         annotated_records_filter = filters.exists_field(
@@ -70,43 +66,72 @@ def aggregation_request(
             ),
         }
 
-        if label is not None:
-            rule_label_annotated_filter = filters.annotated_as([label])
-            aggr_filters.update(
-                {
-                    "correct_records": filters.boolean_filter(
-                        filter_query=annotated_records_filter,
-                        should_filters=[
-                            rule_query_filter,
-                            rule_label_annotated_filter,
-                        ],
-                        minimum_should_match=2,
-                    ),
-                    "incorrect_records": filters.boolean_filter(
-                        filter_query=annotated_records_filter,
-                        must_query=rule_query_filter,
-                        must_not_query=rule_label_annotated_filter,
-                    ),
-                }
-            )
+        if labels is not None:
+            for label in labels:
+                rule_label_annotated_filter = filters.annotated_as([label])
+                encoded_label = self._encode_label_name(label)
+                aggr_filters.update(
+                    {
+                        f"{encoded_label}.correct_records": filters.boolean_filter(
+                            filter_query=annotated_records_filter,
+                            should_filters=[
+                                rule_query_filter,
+                                rule_label_annotated_filter,
+                            ],
+                            minimum_should_match=2,
+                        ),
+                        f"{encoded_label}.incorrect_records": filters.boolean_filter(
+                            filter_query=annotated_records_filter,
+                            must_query=rule_query_filter,
+                            must_not_query=rule_label_annotated_filter,
+                        ),
+                    }
+                )
 
         return {self.id: {"filters": {"filters": aggr_filters}}}
 
+    @staticmethod
+    def _encode_label_name(label: str) -> str:
+        return label.replace(".", "@@@")
+
+    @staticmethod
+    def _decode_label_name(label: str) -> str:
+        return label.replace("@@@", ".")
+
     def aggregation_result(self, aggregation_result: Dict[str, Any]) -> Dict[str, Any]:
         if self.id in aggregation_result:
             aggregation_result = aggregation_result[self.id]
 
-        correct = aggregation_result.get("correct_records", 0)
-        incorrect = aggregation_result.get("incorrect_records", 0)
-        annotated = correct + incorrect
-
+        aggregation_result = unflatten_dict(aggregation_result)
         results = {
-            **aggregation_result,
-            "correct_records": correct,
-            "incorrect_records": incorrect,
+            "covered_records": aggregation_result.pop("covered_records"),
+            "annotated_covered_records": aggregation_result.pop(
+                "annotated_covered_records"
+            ),
         }
-        if annotated > 0:
-            results["precision"] = correct / annotated
+
+        all_correct = []
+        all_incorrect = []
+        all_precision = []
+        for label, metrics in aggregation_result.items():
+            correct = metrics.get("correct_records", 0)
+            incorrect = metrics.get("incorrect_records", 0)
+            annotated = correct + incorrect
+            metrics["annotated"] = annotated
+            if annotated > 0:
+                precision = correct / annotated
+                metrics["precision"] = precision
+                all_precision.append(precision)
+
+            all_correct.append(correct)
+            all_incorrect.append(incorrect)
+            results[self._decode_label_name(label)] = metrics
+
+        results["correct_records"] = sum(all_correct)
+        results["incorrect_records"] = sum(all_incorrect)
+        if len(all_precision) > 0:
+            results["precision"] = sum(all_precision) / len(all_precision)
+
         return results
 
 
@@ -118,9 +143,9 @@ class DatasetLabelingRulesSummary(BaseModel):
 class LabelingRuleSummary(BaseModel):
     covered_records: int
     annotated_covered_records: int
-    correct_records: int
-    incorrect_records: int
-    precision: Optional[float]
+    correct_records: int = Field(default=0)
+    incorrect_records: int = Field(default=0)
+    precision: Optional[float] = None
 
 
 class LabelingService:
@@ -181,7 +206,7 @@ def compute_rule_metrics(
         self,
         dataset: BaseDatasetDB,
         rule_query: str,
-        label: Optional[str] = None,
+        labels: Optional[List[str]] = None,
     ) -> Tuple[int, int, LabelingRuleSummary]:
         """Computes metrics for given rule query and optional label against a set of rules"""
 
@@ -192,7 +217,7 @@ def compute_rule_metrics(
             search=RecordSearch(
                 include_default_aggregations=False,
                 aggregations=self.__rule_metrics__.aggregation_request(
-                    rule_query=rule_query, label=label
+                    rule_query=rule_query, labels=labels
                 ),
             ),
         )
diff --git a/src/rubrix/server/tasks/text_classification/service/service.py b/src/rubrix/server/tasks/text_classification/service/service.py
index 6e7bbb03ef..30c5223a28 100644
--- a/src/rubrix/server/tasks/text_classification/service/service.py
+++ b/src/rubrix/server/tasks/text_classification/service/service.py
@@ -240,20 +240,14 @@ def add_labeling_rule(self, dataset: Dataset, rule: LabelingRule) -> None:
 
         rule:
             The rule
-
         """
-        is_multi_label_dataset = self._is_dataset_multi_label(dataset)
-        if is_multi_label_dataset is not None:
-            assert (
-                not is_multi_label_dataset
-            ), "Labeling rules are not supported for multi-label datasets"
         self.__labeling__.add_rule(dataset, rule)
 
     def update_labeling_rule(
         self,
         dataset: Dataset,
         rule_query: str,
-        label: str,
+        labels: List[str],
         description: Optional[str] = None,
     ) -> LabelingRule:
         """
@@ -271,10 +265,10 @@ def update_labeling_rule(
         """
         found_rule = self.__labeling__.find_rule_by_query(dataset, rule_query)
 
-        found_rule.label = label
+        found_rule.labels = labels
+        found_rule.label = labels[0] if len(labels) == 1 else None
         if description is not None:
             found_rule.description = description
-
         self.__labeling__.replace_rule(dataset, found_rule)
         return found_rule
 
@@ -315,7 +309,7 @@ def compute_rule_metrics(
         self,
         dataset: Dataset,
         rule_query: str,
-        label: Optional[str],
+        labels: Optional[List[str]] = None,
     ) -> LabelingRuleMetricsSummary:
         """
         Compute metrics for a given rule. It's not necessary that query rule
@@ -332,28 +326,28 @@ def compute_rule_metrics(
         rule_query:
             The provided rule query. If already created in dataset, the ``label``
             param will be omitted
-        label:
+        labels:
             Label used for the rule metrics. If not provided and no rule was stored with the
             provided query, no precision will be computed.
-            Otherwise, the label from the stored rule will be used to compute the metrics.
+            Otherwise, the labels from the stored rule will be used to compute the metrics.
 
         Returns
         -------
 
-            Metrics summary for rule and label
+            Metrics summary for rule and labels
 
         """
 
         rule_query = rule_query.strip()
 
-        if label is None:
+        if labels is None:
             for rule in self.get_labeling_rules(dataset):
                 if rule.query == rule_query:
-                    label = rule.label
+                    labels = rule.labels
                     break
 
         total, annotated, metrics = self.__labeling__.compute_rule_metrics(
-            dataset, rule_query=rule_query, label=label
+            dataset, rule_query=rule_query, labels=labels
         )
 
         coverage = metrics.covered_records / total if total > 0 else None
diff --git a/tests/labeling/text_classification/test_label_models.py b/tests/labeling/text_classification/test_label_models.py
index 99f2c396a2..ab3eb91474 100644
--- a/tests/labeling/text_classification/test_label_models.py
+++ b/tests/labeling/text_classification/test_label_models.py
@@ -265,7 +265,7 @@ def mock_predict(self, L, return_probs, tie_break_policy):
         assert list(metrics.keys())[:3] == ["negative", "positive", "neutral"]
 
     def test_score_without_annotations(self, weak_labels):
-        weak_labels._annotation_array = np.array([], dtype=np.short)
+        weak_labels._annotation = np.array([], dtype=np.short)
         label_model = Snorkel(weak_labels)
 
         with pytest.raises(MissingAnnotationError, match="need annotated records"):
diff --git a/tests/labeling/text_classification/test_weak_labels.py b/tests/labeling/text_classification/test_weak_labels.py
index fdcb5b294a..8a77e99f89 100644
--- a/tests/labeling/text_classification/test_weak_labels.py
+++ b/tests/labeling/text_classification/test_weak_labels.py
@@ -12,9 +12,8 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from typing import Callable, List, Optional
+from typing import Callable, List, Optional, Union
 
-import httpx
 import numpy as np
 import pandas as pd
 import pytest
@@ -24,6 +23,7 @@
     CreationTextClassificationRecord,
     TextClassificationBulkData,
 )
+from rubrix.labeling.text_classification import WeakLabels, WeakMultiLabels
 from rubrix.labeling.text_classification.rule import Rule
 from rubrix.labeling.text_classification.weak_labels import (
     DuplicatedRuleNameError,
@@ -31,7 +31,7 @@
     MultiLabelError,
     NoRecordsFoundError,
     NoRulesFoundError,
-    WeakLabels,
+    WeakLabelsBase,
 )
 
 
@@ -70,13 +70,13 @@ def log_dataset(mocked_client) -> str:
 
 @pytest.fixture
 def rules(monkeypatch) -> List[Callable]:
-    def first_rule(record: TextClassificationRecord) -> Optional[str]:
+    def first_rule(record: TextClassificationRecord) -> Optional[Union[str, List[str]]]:
         if "negative" in record.inputs["text"]:
             return "negative"
 
-    def rule2(record: TextClassificationRecord) -> Optional[str]:
+    def rule2(record: TextClassificationRecord) -> Optional[Union[str, List[str]]]:
         if "positive" in record.inputs["text"]:
-            return "positive"
+            return ["positive"]
 
     rule2.__name__ = ""
 
@@ -90,352 +90,673 @@ def mock_apply(self, *args, **kwargs):
     return [first_rule, rule2, rubrix_rule]
 
 
-def test_duplicated_rule_name_error():
-    rules = [Rule(query="mock", label="mock"), Rule(query="mock", label="not mock")]
-    with pytest.raises(DuplicatedRuleNameError, match="'mock': 2"):
-        WeakLabels(rules=rules, dataset="mock")
+@pytest.fixture
+def log_multilabel_dataset(mocked_client) -> str:
+    dataset_name = "test_dataset_for_multilabel_applier"
+    mocked_client.delete(f"/api/datasets/{dataset_name}")
+    records = [
+        CreationTextClassificationRecord.parse_obj(
+            {
+                "inputs": {"text": text},
+                "annotation": {
+                    "labels": [{"class": label, "score": 1} for label in labels],
+                    "agent": "test",
+                }
+                if labels is not None
+                else None,
+                "multi_label": True,
+                "id": idx,
+            }
+        )
+        for text, labels, idx in zip(
+            ["negative", "negative and positive", "None"],
+            [["negative"], ["negative", "positive"], None],
+            [1, 2, 3],
+        )
+    ]
+    mocked_client.post(
+        f"/api/datasets/{dataset_name}/TextClassification:bulk",
+        json=TextClassificationBulkData(
+            records=records,
+        ).dict(by_alias=True),
+    )
 
+    return dataset_name
 
-def test_multi_label_error(monkeypatch):
-    def mock_load(*args, **kwargs):
-        return [TextClassificationRecord(inputs="test", multi_label=True)]
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
-    )
+@pytest.fixture
+def multilabel_rules(monkeypatch) -> List[Callable]:
+    def first_rule(record: TextClassificationRecord) -> Optional[Union[str, List[str]]]:
+        if "negative" in record.inputs["text"]:
+            return []
 
-    with pytest.raises(MultiLabelError):
-        WeakLabels(rules=[lambda x: None], dataset="mock")
+    def rule2(record: TextClassificationRecord) -> Optional[Union[str, List[str]]]:
+        if "positive" in record.inputs["text"]:
+            return ["negative", "positive"]
 
+    rule2.__name__ = ""
 
-def test_no_records_found_error(monkeypatch):
-    def mock_load(*args, **kwargs):
-        return []
+    def mock_apply(self, *args, **kwargs):
+        self._matching_ids = {1: None, 2: None}
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
-    )
+    monkeypatch.setattr(Rule, "apply", mock_apply)
 
-    with pytest.raises(
-        NoRecordsFoundError, match="No records found in dataset 'mock'."
-    ):
-        WeakLabels(rules=[lambda x: None], dataset="mock")
-    with pytest.raises(
-        NoRecordsFoundError,
-        match="No records found in dataset 'mock' with query 'mock'.",
-    ):
-        WeakLabels(rules=[lambda x: None], dataset="mock", query="mock")
-    with pytest.raises(
-        NoRecordsFoundError, match="No records found in dataset 'mock' with ids \[-1\]."
-    ):
-        WeakLabels(rules=[lambda x: None], dataset="mock", ids=[-1])
-    with pytest.raises(
-        NoRecordsFoundError,
-        match="No records found in dataset 'mock' with query 'mock' and with ids \[-1\].",
-    ):
-        WeakLabels(rules=[lambda x: None], dataset="mock", query="mock", ids=[-1])
-
-
-@pytest.mark.parametrize(
-    "label2int, expected_label2int, expected_matrix, expected_annotation_array",
-    [
-        (
-            None,
-            {None: -1, "negative": 0, "positive": 1},
-            np.array([[0, -1, 1], [-1, 1, 1], [-1, 1, -1]], dtype=np.short),
-            np.array([0, 1, -1], dtype=np.short),
-        ),
-        (
-            {None: -10, "negative": 50, "positive": 10},
-            {None: -10, "negative": 50, "positive": 10},
-            np.array([[50, -10, 10], [-10, 10, 10], [-10, 10, -10]], dtype=np.short),
-            np.array([50, 10, -10], dtype=np.short),
-        ),
-        ({}, None, None, None),
-        ({None: -1}, None, None, None),
-        ({None: -1, "negative": 0}, None, None, None),
-    ],
-)
-def test_apply(
-    monkeypatch,
-    mocked_client,
-    log_dataset,
-    rules,
-    label2int,
-    expected_label2int,
-    expected_matrix,
-    expected_annotation_array,
-):
-    monkeypatch.setattr(httpx, "get", mocked_client.get)
-    monkeypatch.setattr(httpx, "stream", mocked_client.stream)
-
-    if label2int == {}:
-        with pytest.raises(MissingLabelError) as error:
-            WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
-        assert "required abstention label" in str(error)
-        return
-    elif label2int == {None: -1}:
-        with pytest.raises(MissingLabelError) as error:
-            WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
-        assert "annotation label" in str(error)
-        return
-    elif label2int == {None: -1, "negative": 0}:
-        with pytest.raises(MissingLabelError) as error:
-            WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
-        assert "weak label" in str(error)
-        return
-
-    weak_labels = WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
-
-    # check that all `Rule.apply`s are called
-    assert weak_labels._rules[-1]._matching_ids == {1: None, 2: None}
-
-    assert weak_labels.label2int == expected_label2int
-    assert weak_labels.int2label == {v: k for k, v in expected_label2int.items()}
-    assert (weak_labels.matrix() == expected_matrix).all()
-    assert (weak_labels._annotation_array == expected_annotation_array).all()
-
-
-def test_rules_matrix_records_annotation(monkeypatch):
-    expected_records = [
-        TextClassificationRecord(inputs="test without annot"),
-        TextClassificationRecord(inputs="test with annot", annotation="positive"),
-    ]
+    rubrix_rule = Rule(query="mock", label="positive", name="rubrix_rule")
 
-    def mock_load(*args, **kwargs):
-        return expected_records
+    return [first_rule, rule2, rubrix_rule]
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
-    )
 
-    def mock_apply(self, *args, **kwargs):
-        weak_label_matrix = np.array([[0, 1], [-1, 0]], dtype=np.short)
-        annotation_array = np.array([-1, 0], dtype=np.short)
-        label2int = {None: -1, "negative": 0, "positive": 1}
-        return weak_label_matrix, annotation_array, label2int
-
-    monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
-
-    weak_labels = WeakLabels(rules=[lambda x: "mock"] * 2, dataset="mock")
-
-    # records property
-    assert len(weak_labels.records()) == 2
-    assert weak_labels.records(has_annotation=True) == [expected_records[1]]
-    assert weak_labels.records(has_annotation=False) == [expected_records[0]]
-    assert isinstance(weak_labels.records()[0], TextClassificationRecord)
-
-    # rules property
-    assert len(weak_labels.rules) == 2
-    assert weak_labels._rules_name2index == {"rule_0": 0, "rule_1": 1}
-    assert weak_labels.rules[0](None) == "mock"
-
-    assert (
-        weak_labels.matrix(has_annotation=False) == np.array([[0, 1]], dtype=np.short)
-    ).all()
-    assert (
-        weak_labels.matrix(has_annotation=True) == np.array([[-1, 0]], dtype=np.short)
-    ).all()
-    assert (weak_labels.annotation() == np.array([[0]], dtype=np.short)).all()
-    assert (
-        weak_labels.annotation(include_missing=True)
-        == np.array([[-1, 0]], dtype=np.short)
-    ).all()
-    with pytest.warns(
-        FutureWarning, match="'exclude_missing_annotations' is deprecated"
-    ):
-        weak_labels.annotation(exclude_missing_annotations=True)
+class TestWeakLabelsBase:
+    def test_dataset_type_error(self):
+        with pytest.raises(TypeError, match="must be a string, but you provided"):
+            WeakLabelsBase([1, 2, 3])
 
+    def test_rules_from_dataset(self, monkeypatch, log_dataset):
+        mock_rules = [Rule(query="mock", label="mock")]
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load_rules",
+            lambda x: mock_rules,
+        )
 
-def test_summary(monkeypatch, rules):
-    def mock_load(*args, **kwargs):
-        return [TextClassificationRecord(inputs="test")] * 4
+        wl = WeakLabelsBase(log_dataset)
+        assert wl.rules is mock_rules
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
-    )
+    def test_norulesfounderror(self, monkeypatch):
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load_rules", lambda x: []
+        )
 
-    def mock_apply(self, *args, **kwargs):
-        weak_label_matrix = np.array(
-            [[0, 1, -1], [-1, 0, -1], [-1, -1, -1], [1, 1, -1]], dtype=np.short
+        with pytest.raises(NoRulesFoundError, match="No rules were found"):
+            WeakLabelsBase("mock")
+
+    def test_duplicated_rule_name_error(self):
+        rules = [Rule(query="mock", label="mock"), Rule(query="mock", label="not mock")]
+        with pytest.raises(DuplicatedRuleNameError, match="'mock': 2"):
+            WeakLabelsBase(rules=rules, dataset="mock")
+
+    def test_no_records_found_error(self, monkeypatch):
+        def mock_load(*args, **kwargs):
+            return []
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
         )
-        # weak_label_matrix = np.random.randint(-1, 30, (int(1e5), 50), dtype=np.short)
-        annotation_array = np.array([-1, -1, -1, -1], dtype=np.short)
-        # annotation_array = np.random.randint(-1, 30, int(1e5), dtype=np.short)
-        label2int = {None: -1, "negative": 0, "positive": 1}
-        # label2int = {k: v for k, v in zip(["None"]+list(range(30)), list(range(-1, 30)))}
-        return weak_label_matrix, annotation_array, label2int
-
-    monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
-
-    weak_labels = WeakLabels(rules=rules, dataset="mock")
-
-    summary = weak_labels.summary()
-    expected = pd.DataFrame(
-        {
-            "label": [
-                {"negative", "positive"},
-                {"negative", "positive"},
-                set(),
-                {"negative", "positive"},
-            ],
-            "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
-            "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
-            "conflicts": [1.0 / 4, 1.0 / 4, 0, 1.0 / 4],
-        },
-        index=["first_rule", "rule_1", "rubrix_rule", "total"],
-    )
-    pd.testing.assert_frame_equal(summary, expected)
-
-    summary = weak_labels.summary(normalize_by_coverage=True)
-    expected = pd.DataFrame(
-        {
-            "label": [
-                {"negative", "positive"},
-                {"negative", "positive"},
-                set(),
-                {"negative", "positive"},
-            ],
-            "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
-            "overlaps": [2.0 / 2, 2.0 / 3, 0, 2.0 / 3],
-            "conflicts": [1.0 / 2, 1.0 / 3, 0, 1.0 / 3],
-        },
-        index=["first_rule", "rule_1", "rubrix_rule", "total"],
-    )
-    pd.testing.assert_frame_equal(summary, expected)
-
-    summary = weak_labels.summary(annotation=np.array([1, -1, 0, 1]))
-    expected = pd.DataFrame(
-        {
-            "label": [
-                {"negative", "positive"},
-                {"negative", "positive"},
-                set(),
-                {"negative", "positive"},
-            ],
-            "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
-            "annotated_coverage": [2.0 / 3, 2.0 / 3, 0, 2.0 / 3],
-            "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
-            "conflicts": [1.0 / 4, 1.0 / 4, 0, 1.0 / 4],
-            "correct": [1, 2, 0, 3],
-            "incorrect": [1, 0, 0, 1],
-            "precision": [1.0 / 2, 2 / 2, np.nan, 3.0 / 4],
-        },
-        index=["first_rule", "rule_1", "rubrix_rule", "total"],
-    )
-    pd.testing.assert_frame_equal(summary, expected)
 
+        with pytest.raises(
+            NoRecordsFoundError, match="No records found in dataset 'mock'."
+        ):
+            WeakLabels(rules=[lambda x: None], dataset="mock")
+        with pytest.raises(
+            NoRecordsFoundError,
+            match="No records found in dataset 'mock' with query 'mock'.",
+        ):
+            WeakLabels(rules=[lambda x: None], dataset="mock", query="mock")
+        with pytest.raises(
+            NoRecordsFoundError,
+            match="No records found in dataset 'mock' with ids \[-1\].",
+        ):
+            WeakLabels(rules=[lambda x: None], dataset="mock", ids=[-1])
+        with pytest.raises(
+            NoRecordsFoundError,
+            match="No records found in dataset 'mock' with query 'mock' and with ids \[-1\].",
+        ):
+            WeakLabels(rules=[lambda x: None], dataset="mock", query="mock", ids=[-1])
+
+    def test_rules_records_properties(self, monkeypatch):
+        expected_records = [
+            TextClassificationRecord(inputs="test without annot"),
+            TextClassificationRecord(inputs="test with annot", annotation="positive"),
+        ]
+
+        def mock_load(*args, **kwargs):
+            return expected_records
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-def test_show_records(monkeypatch, rules):
-    def mock_load(*args, **kwargs):
-        return [TextClassificationRecord(inputs="test", id=i) for i in range(5)]
+        weak_labels = WeakLabelsBase(rules=[lambda x: "mock"] * 2, dataset="mock")
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
-    )
+        # records property
+        assert len(weak_labels.records()) == 2
+        assert weak_labels.records(has_annotation=True) == [expected_records[1]]
+        assert weak_labels.records(has_annotation=False) == [expected_records[0]]
+        assert isinstance(weak_labels.records()[0], TextClassificationRecord)
 
-    def mock_apply(self, *args, **kwargs):
-        weak_label_matrix = np.array(
-            [[0, 1, -1], [2, 0, -1], [-1, -1, -1], [1, 1, -1], [-1, 0, 2]],
-            dtype=np.short,
+        # rules property
+        assert len(weak_labels.rules) == 2
+        assert weak_labels._rules_name2index == {"rule_0": 0, "rule_1": 1}
+        assert weak_labels.rules[0](None) == "mock"
+
+    def test_not_implemented_errors(self, monkeypatch):
+        def mock_load(*args, **kwargs):
+            return ["mock"]
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
         )
-        annotation_array = np.array([0, 1, -1, 2, 0], dtype=np.short)
-        label2int = {None: -1, "negative": 0, "positive": 1, "neutral": 2}
-        return weak_label_matrix, annotation_array, label2int
 
-    monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+        weak_labels = WeakLabelsBase(rules=["mock"], dataset="mock")
 
-    weak_labels = WeakLabels(rules=rules, dataset="mock")
+        with pytest.raises(NotImplementedError):
+            weak_labels.matrix()
+        with pytest.raises(NotImplementedError):
+            weak_labels.annotation()
+        with pytest.raises(NotImplementedError):
+            weak_labels.summary()
+        with pytest.raises(NotImplementedError):
+            weak_labels.show_records()
 
-    assert weak_labels.show_records().id.tolist() == [0, 1, 2, 3, 4]
-    assert weak_labels.show_records(labels=["positive"]).id.tolist() == [0, 3]
-    assert weak_labels.show_records(labels=["negative", "neutral"]).id.tolist() == [
-        1,
-        4,
-    ]
-    assert weak_labels.show_records(rules=[0]).id.tolist() == [0, 1, 3]
-    assert weak_labels.show_records(rules=[0, "rule_1"]).id.tolist() == [0, 1, 3]
-    assert weak_labels.show_records(labels=["negative"], rules=[1]).id.tolist() == [
-        0,
-        1,
-        4,
-    ]
-    assert weak_labels.show_records(labels=["positive"], rules=["rubrix_rule"]).empty
 
+class TestWeakLabels:
+    def test_multi_label_error(self, monkeypatch):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="test", multi_label=True)]
 
-def test_change_mapping(monkeypatch, rules):
-    def mock_load(*args, **kwargs):
-        return [TextClassificationRecord(inputs="test", id=i) for i in range(5)]
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        with pytest.raises(MultiLabelError):
+            WeakLabels(rules=[lambda x: None], dataset="mock")
+
+    @pytest.mark.parametrize(
+        "label2int, expected_label2int, expected_matrix, expected_annotation_array",
+        [
+            (
+                None,
+                {None: -1, "negative": 0, "positive": 1},
+                np.array([[0, -1, 1], [-1, 1, 1], [-1, 1, -1]], dtype=np.short),
+                np.array([0, 1, -1], dtype=np.short),
+            ),
+            (
+                {None: -10, "negative": 50, "positive": 10},
+                {None: -10, "negative": 50, "positive": 10},
+                np.array(
+                    [[50, -10, 10], [-10, 10, 10], [-10, 10, -10]], dtype=np.short
+                ),
+                np.array([50, 10, -10], dtype=np.short),
+            ),
+            ({}, None, None, None),
+            ({None: -1}, None, None, None),
+            ({None: -1, "negative": 0}, None, None, None),
+        ],
     )
+    def test_apply(
+        self,
+        monkeypatch,
+        log_dataset,
+        rules,
+        label2int,
+        expected_label2int,
+        expected_matrix,
+        expected_annotation_array,
+    ):
+        if label2int == {}:
+            with pytest.raises(MissingLabelError, match="required abstention label"):
+                WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
+            return
+        elif label2int == {None: -1}:
+            with pytest.raises(MissingLabelError, match="annotation label"):
+                WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
+            return
+        elif label2int == {None: -1, "negative": 0}:
+            with pytest.raises(MissingLabelError, match="weak label"):
+                WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
+            return
+
+        weak_labels = WeakLabels(rules=rules, dataset=log_dataset, label2int=label2int)
+
+        # check that all `Rule.apply`s are called
+        assert weak_labels._rules[-1]._matching_ids == {1: None, 2: None}
+
+        assert weak_labels.label2int == expected_label2int
+        assert weak_labels.int2label == {v: k for k, v in expected_label2int.items()}
+        assert (weak_labels.matrix() == expected_matrix).all()
+        assert (weak_labels._annotation == expected_annotation_array).all()
+
+    def test_apply_MultiLabelError(self, log_dataset):
+        with pytest.raises(
+            MultiLabelError, match="For rules that do not return exactly 1 label"
+        ):
+            WeakLabels(rules=[lambda x: ["a", "b"]], dataset=log_dataset)
+
+    def test_matrix_annotation_properties(self, monkeypatch):
+        expected_records = [
+            TextClassificationRecord(inputs="test without annot"),
+            TextClassificationRecord(inputs="test with annot", annotation="positive"),
+        ]
+
+        def mock_load(*args, **kwargs):
+            return expected_records
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    def mock_apply(self, *args, **kwargs):
-        weak_label_matrix = np.array(
-            [[0, 1, -1], [2, 0, -1], [-1, -1, -1], [1, 1, -1], [-1, 0, 2]],
-            dtype=np.short,
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array([[0, 1], [-1, 0]], dtype=np.short)
+            annotation_array = np.array([-1, 0], dtype=np.short)
+            label2int = {None: -1, "negative": 0, "positive": 1}
+            return weak_label_matrix, annotation_array, label2int
+
+        monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakLabels(rules=[lambda x: "mock"] * 2, dataset="mock")
+
+        assert (
+            weak_labels.matrix(has_annotation=False)
+            == np.array([[0, 1]], dtype=np.short)
+        ).all()
+        assert (
+            weak_labels.matrix(has_annotation=True)
+            == np.array([[-1, 0]], dtype=np.short)
+        ).all()
+        assert (weak_labels.annotation() == np.array([[0]], dtype=np.short)).all()
+        assert (
+            weak_labels.annotation(include_missing=True)
+            == np.array([[-1, 0]], dtype=np.short)
+        ).all()
+        with pytest.warns(
+            FutureWarning, match="'exclude_missing_annotations' is deprecated"
+        ):
+            weak_labels.annotation(exclude_missing_annotations=True)
+
+    def test_summary(self, monkeypatch, rules):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="test")] * 4
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
+
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [[0, 1, -1], [-1, 0, -1], [-1, -1, -1], [1, 1, -1]], dtype=np.short
+            )
+            # weak_label_matrix = np.random.randint(-1, 30, (int(1e5), 50), dtype=np.short)
+            annotation_array = np.array([-1, -1, -1, -1], dtype=np.short)
+            # annotation_array = np.random.randint(-1, 30, int(1e5), dtype=np.short)
+            label2int = {None: -1, "negative": 0, "positive": 1}
+            # label2int = {k: v for k, v in zip(["None"]+list(range(30)), list(range(-1, 30)))}
+            return weak_label_matrix, annotation_array, label2int
+
+        monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakLabels(rules=rules, dataset="mock")
+
+        summary = weak_labels.summary()
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative", "positive"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
+                "conflicts": [1.0 / 4, 1.0 / 4, 0, 1.0 / 4],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
         )
-        annotation_array = np.array([0, 1, -1, 2, 0], dtype=np.short)
-        label2int = {None: -1, "negative": 0, "positive": 1, "neutral": 2}
-        return weak_label_matrix, annotation_array, label2int
+        pd.testing.assert_frame_equal(summary, expected)
 
-    monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+        summary = weak_labels.summary(normalize_by_coverage=True)
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative", "positive"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "overlaps": [2.0 / 2, 2.0 / 3, 0, 2.0 / 3],
+                "conflicts": [1.0 / 2, 1.0 / 3, 0, 1.0 / 3],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
+        )
+        pd.testing.assert_frame_equal(summary, expected)
 
-    weak_labels = WeakLabels(rules=rules, dataset="mock")
+        summary = weak_labels.summary(annotation=np.array([1, -1, 0, 1]))
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative", "positive"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "annotated_coverage": [2.0 / 3, 2.0 / 3, 0, 2.0 / 3],
+                "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
+                "conflicts": [1.0 / 4, 1.0 / 4, 0, 1.0 / 4],
+                "correct": [1, 2, 0, 3],
+                "incorrect": [1, 0, 0, 1],
+                "precision": [1.0 / 2, 2 / 2, np.nan, 3.0 / 4],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
+        )
+        pd.testing.assert_frame_equal(summary, expected)
 
-    with pytest.raises(MissingLabelError):
-        weak_labels.change_mapping({"negative": 2})
+    def test_show_records(self, monkeypatch, rules):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="test", id=i) for i in range(5)]
 
-    new_mapping = {None: -10, "negative": 2, "positive": 10, "neutral": 1}
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    old_wlm = weak_labels.matrix().copy()
-    old_mapping = weak_labels.label2int.copy()
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [[0, 1, -1], [2, 0, -1], [-1, -1, -1], [1, 1, -1], [-1, 0, 2]],
+                dtype=np.short,
+            )
+            annotation_array = np.array([0, 1, -1, 2, 0], dtype=np.short)
+            label2int = {None: -1, "negative": 0, "positive": 1, "neutral": 2}
+            return weak_label_matrix, annotation_array, label2int
+
+        monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakLabels(rules=rules, dataset="mock")
+
+        assert weak_labels.show_records().id.tolist() == [0, 1, 2, 3, 4]
+        assert weak_labels.show_records(labels=["positive"]).id.tolist() == [0, 3]
+        assert weak_labels.show_records(labels=["negative", "neutral"]).id.tolist() == [
+            1,
+            4,
+        ]
+        assert weak_labels.show_records(rules=[0]).id.tolist() == [0, 1, 3]
+        assert weak_labels.show_records(rules=[0, "rule_1"]).id.tolist() == [0, 1, 3]
+        assert weak_labels.show_records(labels=["negative"], rules=[1]).id.tolist() == [
+            0,
+            1,
+            4,
+        ]
+        assert weak_labels.show_records(
+            labels=["positive"], rules=["rubrix_rule"]
+        ).empty
+
+    def test_change_mapping(self, monkeypatch, rules):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="test", id=i) for i in range(5)]
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    weak_labels.change_mapping(new_mapping)
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [[0, 1, -1], [2, 0, -1], [-1, -1, -1], [1, 1, -1], [-1, 0, 2]],
+                dtype=np.short,
+            )
+            annotation_array = np.array([0, 1, -1, 2, 0], dtype=np.short)
+            label2int = {None: -1, "negative": 0, "positive": 1, "neutral": 2}
+            return weak_label_matrix, annotation_array, label2int
+
+        monkeypatch.setattr(WeakLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakLabels(rules=rules, dataset="mock")
+
+        with pytest.raises(MissingLabelError):
+            weak_labels.change_mapping({"negative": 2})
+
+        new_mapping = {None: -10, "negative": 2, "positive": 10, "neutral": 1}
+
+        old_wlm = weak_labels.matrix().copy()
+        old_mapping = weak_labels.label2int.copy()
+
+        weak_labels.change_mapping(new_mapping)
+
+        assert (
+            weak_labels.matrix()
+            == np.array(
+                [
+                    [2, 10, -10],
+                    [1, 2, -10],
+                    [-10, -10, -10],
+                    [10, 10, -10],
+                    [-10, 2, 1],
+                ],
+                dtype=np.short,
+            )
+        ).all()
+        assert (
+            weak_labels.annotation(include_missing=True)
+            == np.array([2, 10, -10, 1, 2], dtype=np.short)
+        ).all()
+        assert weak_labels.label2int == new_mapping
+        assert weak_labels.int2label == {val: key for key, val in new_mapping.items()}
+
+        weak_labels.change_mapping(old_mapping)
+
+        assert (weak_labels.matrix() == old_wlm).all()
+
+
+class TestWeakMultiLabels:
+    def test_apply(
+        self,
+        log_multilabel_dataset,
+        multilabel_rules,
+    ):
+        weak_labels = WeakMultiLabels(
+            rules=multilabel_rules, dataset=log_multilabel_dataset
+        )
 
-    assert (
-        weak_labels.matrix()
-        == np.array(
-            [[2, 10, -10], [1, 2, -10], [-10, -10, -10], [10, 10, -10], [-10, 2, 1]],
-            dtype=np.short,
+        assert weak_labels.labels == ["negative", "positive"]
+
+        # check that all `Rule.apply`s are called
+        assert weak_labels._rules[-1]._matching_ids == {1: None, 2: None}
+
+        assert (
+            weak_labels.matrix()
+            == np.array(
+                [
+                    [[0, 0], [-1, -1], [0, 1]],
+                    [[0, 0], [1, 1], [0, 1]],
+                    [[-1, -1], [-1, -1], [-1, -1]],
+                ],
+                dtype=np.short,
+            )
+        ).all()
+
+        assert (
+            weak_labels._annotation
+            == np.array([[1, 0], [1, 1], [-1, -1]], dtype=np.short)
+        ).all()
+
+    def test_matrix_annotation(self, monkeypatch):
+        expected_records = [
+            TextClassificationRecord(inputs="test without annot", multi_label=True),
+            TextClassificationRecord(
+                inputs="test with annot", annotation="positive", multi_label=True
+            ),
+        ]
+
+        def mock_load(*args, **kwargs):
+            return expected_records
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
         )
-    ).all()
-    assert (
-        weak_labels.annotation(include_missing=True)
-        == np.array([2, 10, -10, 1, 2], dtype=np.short)
-    ).all()
-    assert weak_labels.label2int == new_mapping
-    assert weak_labels.int2label == {val: key for key, val in new_mapping.items()}
 
-    weak_labels.change_mapping(old_mapping)
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [[[1, 0], [0, 1]], [[-1, -1], [1, 0]]], dtype=np.short
+            )
+            annotation_array = np.array([[-1, -1], [1, 0]], dtype=np.short)
+            labels = ["negative", "positive"]
+            return weak_label_matrix, annotation_array, labels
+
+        monkeypatch.setattr(WeakMultiLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakMultiLabels(rules=[lambda x: "mock"] * 2, dataset="mock")
+
+        assert (
+            weak_labels.matrix(has_annotation=False)
+            == np.array([[[1, 0], [0, 1]]], dtype=np.short)
+        ).all()
+        assert (
+            weak_labels.matrix(has_annotation=True)
+            == np.array([[[-1, -1], [1, 0]]], dtype=np.short)
+        ).all()
+        assert (weak_labels.annotation() == np.array([[1, 0]], dtype=np.short)).all()
+        assert (
+            weak_labels.annotation(include_missing=True)
+            == np.array([[[-1, -1], [1, 0]]], dtype=np.short)
+        ).all()
+
+    def test_summary(self, monkeypatch, multilabel_rules):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="test", multi_label=True)] * 4
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    assert (weak_labels.matrix() == old_wlm).all()
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [
+                    [[1, 0], [1, 0], [-1, -1]],
+                    [[-1, -1], [1, 0], [-1, -1]],
+                    [[-1, -1], [-1, -1], [-1, -1]],
+                    [[1, 1], [1, 0], [-1, -1]],
+                ],
+                dtype=np.short,
+            )
+            # weak_label_matrix = np.random.randint(-1, 30, (int(1e5), 50), dtype=np.short)
+            annotation_array = np.array(
+                [[-1, -1], [-1, -1], [-1, -1], [-1, -1]], dtype=np.short
+            )
+            # annotation_array = np.random.randint(-1, 30, int(1e5), dtype=np.short)
+            labels = ["negative", "positive"]
+            # label2int = {k: v for k, v in zip(["None"]+list(range(30)), list(range(-1, 30)))}
+            return weak_label_matrix, annotation_array, labels
+
+        monkeypatch.setattr(WeakMultiLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakMultiLabels(rules=multilabel_rules, dataset="mock")
+
+        summary = weak_labels.summary()
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
+        )
+        pd.testing.assert_frame_equal(summary, expected)
 
+        summary = weak_labels.summary(normalize_by_coverage=True)
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "overlaps": [2.0 / 2, 2.0 / 3, 0, 2.0 / 3],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
+        )
+        pd.testing.assert_frame_equal(summary, expected)
 
-def test_dataset_type_error():
-    with pytest.raises(TypeError, match="must be a string, but you provided"):
-        WeakLabels([1, 2, 3])
+        summary = weak_labels.summary(
+            annotation=np.array([[0, 1], [-1, -1], [0, 1], [1, 1]])
+        )
+        expected = pd.DataFrame(
+            {
+                "label": [
+                    {"negative", "positive"},
+                    {"negative"},
+                    set(),
+                    {"negative", "positive"},
+                ],
+                "coverage": [2.0 / 4, 3.0 / 4, 0, 3.0 / 4],
+                "annotated_coverage": [2.0 / 3, 2.0 / 3, 0, 2.0 / 3],
+                "overlaps": [2.0 / 4, 2.0 / 4, 0, 2.0 / 4],
+                "correct": [2, 1, 0, 3],
+                "incorrect": [1, 1, 0, 2],
+                "precision": [2.0 / 3, 1.0 / 2, np.nan, 3.0 / 5],
+            },
+            index=["first_rule", "rule_1", "rubrix_rule", "total"],
+        )
+        pd.testing.assert_frame_equal(summary, expected)
 
+    def test_compute_correct_incorrect(self, monkeypatch):
+        def mock_load(*args, **kwargs):
+            return [TextClassificationRecord(inputs="mock")]
 
-def test_rules_from_dataset(monkeypatch, mocked_client, log_dataset):
-    monkeypatch.setattr(httpx, "get", mocked_client.get)
-    monkeypatch.setattr(httpx, "stream", mocked_client.stream)
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    mock_rules = [Rule(query="mock", label="mock")]
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load_rules",
-        lambda x: mock_rules,
-    )
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array([[[1, 0, 1, 0], [0, 1, 0, 1]]], dtype=np.short)
+            return weak_label_matrix, None, None
 
-    wl = WeakLabels(log_dataset)
-    assert wl.rules is mock_rules
+        monkeypatch.setattr(WeakMultiLabels, "_apply_rules", mock_apply)
 
+        weak_labels = WeakMultiLabels(rules=[lambda x: "mock"] * 2, dataset="mock")
+        correct, incorrect = weak_labels._compute_correct_incorrect(
+            annotation=np.array([[1, 0, 1, 0]])
+        )
 
-def test_norulesfounderror(monkeypatch):
-    monkeypatch.setattr(
-        "rubrix.labeling.text_classification.weak_labels.load_rules", lambda x: []
-    )
+        assert np.allclose(correct, np.array([2, 0, 2]))
+        assert np.allclose(incorrect, np.array([0, 2, 2]))
+
+    def test_show_records(self, monkeypatch, multilabel_rules):
+        def mock_load(*args, **kwargs):
+            return [
+                TextClassificationRecord(inputs="test", id=i, multi_label=True)
+                for i in range(5)
+            ]
+
+        monkeypatch.setattr(
+            "rubrix.labeling.text_classification.weak_labels.load", mock_load
+        )
 
-    with pytest.raises(NoRulesFoundError, match="No rules were found"):
-        WeakLabels("mock")
+        def mock_apply(self, *args, **kwargs):
+            weak_label_matrix = np.array(
+                [
+                    [[1, 0], [0, 1], [-1, -1]],
+                    [[0, 1], [1, 0], [-1, -1]],
+                    [[-1, -1], [-1, -1], [-1, -1]],
+                    [[0, 1], [0, 1], [-1, -1]],
+                    [[-1, -1], [1, 0], [1, 0]],
+                ],
+                dtype=np.short,
+            )
+            labels = ["negative", "positive"]
+            return weak_label_matrix, None, labels
+
+        monkeypatch.setattr(WeakMultiLabels, "_apply_rules", mock_apply)
+
+        weak_labels = WeakMultiLabels(rules=multilabel_rules, dataset="mock")
+
+        assert weak_labels.show_records().id.tolist() == [0, 1, 2, 3, 4]
+        assert weak_labels.show_records(labels=["positive"]).id.tolist() == [0, 1, 3]
+        assert weak_labels.show_records(labels=["negative"]).id.tolist() == [0, 1, 4]
+        assert weak_labels.show_records(
+            labels=["negative", "positive"]
+        ).id.tolist() == [0, 1]
+
+        assert weak_labels.show_records(rules=[0]).id.tolist() == [0, 1, 3]
+        assert weak_labels.show_records(rules=[0, "rule_1"]).id.tolist() == [0, 1, 3]
+        assert weak_labels.show_records(labels=["negative"], rules=[1]).id.tolist() == [
+            0,
+            1,
+            4,
+        ]
+        assert weak_labels.show_records(
+            labels=["positive"], rules=["rubrix_rule"]
+        ).empty
diff --git a/tests/server/text_classification/test_api_rules.py b/tests/server/text_classification/test_api_rules.py
index 7c0e1bc2a8..3324798e27 100644
--- a/tests/server/text_classification/test_api_rules.py
+++ b/tests/server/text_classification/test_api_rules.py
@@ -1,6 +1,5 @@
 import pytest
 
-from rubrix.server.commons.errors import EntityAlreadyExistsError
 from rubrix.server.tasks.text_classification import (
     CreateLabelingRule,
     DatasetLabelingRulesMetricsSummary,
@@ -75,11 +74,12 @@ def test_dataset_update_rule(mocked_client):
     rules = list(map(LabelingRule.parse_obj, response.json()))
     assert len(rules) == 1
     assert rules[0].label == "NEW Label"
+    assert rules[0].labels == ["NEW Label"]
     assert rules[0].description is None
 
     mocked_client.patch(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules/{query}",
-        json={"label": "NEW Label", "description": "New description"},
+        json={"labels": ["A", "B"], "description": "New description"},
     )
 
     response = mocked_client.get(
@@ -88,25 +88,34 @@ def test_dataset_update_rule(mocked_client):
     rules = list(map(LabelingRule.parse_obj, response.json()))
     assert len(rules) == 1
     assert rules[0].description == "New description"
-
-
-def test_dataset_with_rules(mocked_client):
+    assert rules[0].label is None
+    assert rules[0].labels == ["A", "B"]
+
+
+@pytest.mark.parametrize(
+    "rule",
+    [
+        CreateLabelingRule(query="a query", description="Description", label="LALA"),
+        CreateLabelingRule(
+            query="another query", description="Description", labels=["A", "B", "C"]
+        ),
+    ],
+)
+def test_dataset_with_rules(mocked_client, rule):
     dataset = "test_dataset_with_rules"
     log_some_records(mocked_client, dataset)
 
     response = mocked_client.post(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-        json=CreateLabelingRule(
-            query="a query", description="Description", label="LALA"
-        ).dict(),
+        json=rule.dict(),
     )
     assert response.status_code == 200
 
     created_rule = LabelingRule.parse_obj(response.json())
-    assert created_rule.query == "a query"
-    assert created_rule.label == "LALA"
-    assert created_rule.description == "Description"
-    assert created_rule.author == "rubrix"
+    assert created_rule.query == rule.query
+    assert created_rule.label == rule.label
+    assert created_rule.labels == rule.labels
+    assert created_rule.description == rule.description
 
     response = mocked_client.get(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules"
@@ -117,29 +126,34 @@ def test_dataset_with_rules(mocked_client):
     assert rules[0] == created_rule
 
 
-def test_get_dataset_rule(mocked_client):
+@pytest.mark.parametrize(
+    "rule",
+    [
+        CreateLabelingRule(query="a query", description="Description", label="LALA"),
+        CreateLabelingRule(
+            query="another query", description="Description", labels=["A", "B", "C"]
+        ),
+    ],
+)
+def test_get_dataset_rule(mocked_client, rule):
     dataset = "test_get_dataset_rule"
     log_some_records(mocked_client, dataset)
 
-    rule_query = "a query"
-    rule_label = "TEST"
-    rule_description = "Description"
     response = mocked_client.post(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-        json=CreateLabelingRule(
-            query=rule_query, label=rule_label, description=rule_description
-        ).dict(),
+        json=rule.dict(),
     )
     assert response.status_code == 200
 
     response = mocked_client.get(
-        f"/api/datasets/TextClassification/{dataset}/labeling/rules/{rule_query}"
+        f"/api/datasets/TextClassification/{dataset}/labeling/rules/{rule.query}"
     )
     assert response.status_code == 200
-    rule = LabelingRule.parse_obj(response.json())
-    assert rule.query == rule_query
-    assert rule.label == rule_label
-    assert rule.description == rule_description
+    found_rule = LabelingRule.parse_obj(response.json())
+    assert found_rule.query == rule.query
+    assert found_rule.label == rule.label
+    assert found_rule.labels == rule.labels
+    assert found_rule.description == rule.description
 
 
 def test_delete_dataset_rules(mocked_client):
@@ -170,7 +184,7 @@ def test_duplicated_dataset_rules(mocked_client):
     dataset = "test_duplicated_dataset_rules"
     log_some_records(mocked_client, dataset)
 
-    rule = CreateLabelingRule(query="a query", label="TEST")
+    rule = CreateLabelingRule(query="a query", labels=["TEST"])
     response = mocked_client.post(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules",
         json=rule.dict(),
@@ -194,21 +208,18 @@ def test_rules_with_multi_label_dataset(mocked_client):
     dataset = "test_rules_with_multi_label_dataset"
     log_some_records(mocked_client, dataset, multi_label=True)
 
+    rule = CreateLabelingRule(query="a query", description="Description", label="LALA")
     response = mocked_client.post(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-        json=CreateLabelingRule(
-            query="a query", description="Description", label="LALA"
-        ).dict(),
+        json=rule.dict(),
     )
-    assert response.status_code == 400
-    assert response.json() == {
-        "detail": {
-            "code": "rubrix.api.errors::BadRequestError",
-            "params": {
-                "message": "Labeling rules are not supported for multi-label datasets"
-            },
-        }
-    }
+    assert response.status_code == 200
+    found_rule = LabelingRule.parse_obj(response.json())
+
+    assert found_rule.query == rule.query
+    assert found_rule.label == rule.label
+    assert found_rule.labels == rule.labels
+    assert found_rule.description == rule.description
 
 
 def test_rule_metrics_with_missing_label(mocked_client):
@@ -229,21 +240,96 @@ def test_rule_metrics_with_missing_label(mocked_client):
     }
 
 
-def test_rule_metrics_with_missing_label_for_stored_rule(mocked_client):
+@pytest.mark.parametrize(
+    ("rule", "expected_metrics"),
+    [
+        (
+            CreateLabelingRule(query="ejemplo", label="TEST"),
+            {
+                "annotated_records": 1,
+                "correct": 0.0,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "incorrect": 1.0,
+                "precision": 0.0,
+                "total_records": 1,
+            },
+        ),
+        (
+            CreateLabelingRule(query="ejemplo", label="test.bad"),
+            {
+                "annotated_records": 1,
+                "correct": 0.0,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "incorrect": 1.0,
+                "precision": 0.0,
+                "total_records": 1,
+            },
+        ),
+        (
+            CreateLabelingRule(query="ejemplo", label="o.k."),
+            {
+                "annotated_records": 1,
+                "correct": 1.0,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "incorrect": 0.0,
+                "precision": 1.0,
+                "total_records": 1,
+            },
+        ),
+        (
+            CreateLabelingRule(query="bad query", label="TEST"),
+            {
+                "annotated_records": 1,
+                "correct": 0.0,
+                "coverage": 0.0,
+                "coverage_annotated": 0.0,
+                "incorrect": 0.0,
+                "total_records": 1,
+            },
+        ),
+        (
+            CreateLabelingRule(query="ejemplo", labels=["TEST", "A", "B"]),
+            {
+                "annotated_records": 1,
+                "correct": 0.0,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "incorrect": 3.0,
+                "precision": 0.0,
+                "total_records": 1,
+            },
+        ),
+        (
+            CreateLabelingRule(query="ejemplo", labels=["A", "o.k."]),
+            {
+                "annotated_records": 1,
+                "correct": 1.0,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "incorrect": 1.0,
+                "precision": 0.5,
+                "total_records": 1,
+            },
+        ),
+    ],
+)
+def test_rule_metrics_with_missing_label_for_stored_rule(
+    mocked_client, rule, expected_metrics
+):
     dataset = "test_rule_metrics_with_missing_label_for_stored_rule"
-    log_some_records(mocked_client, dataset, annotation="OK")
-    for query in ["ejemplo", "bad query"]:
-        mocked_client.post(
-            f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-            json=CreateLabelingRule(
-                query=query, label="TEST", description="Description"
-            ).dict(),
-        )
+    log_some_records(mocked_client, dataset, annotation="o.k.")
+    mocked_client.post(
+        f"/api/datasets/TextClassification/{dataset}/labeling/rules", json=rule.dict()
+    )
 
     response = mocked_client.get(
-        f"/api/datasets/TextClassification/{dataset}/labeling/rules/bad query/metrics"
+        f"/api/datasets/TextClassification/{dataset}/labeling/rules/{rule.query}/metrics"
     )
     assert response.status_code == 200
+    assert response.json() == expected_metrics
 
 
 def test_create_rules_and_then_log(mocked_client):
@@ -271,52 +357,67 @@ def test_create_rules_and_then_log(mocked_client):
     assert len(rules) == 2
 
 
-def test_dataset_rules_metrics(mocked_client):
+@pytest.mark.parametrize(
+    ("rules", "expected_metrics", "annotation"),
+    [
+        (
+            [
+                CreateLabelingRule(query="ejemplo", label="TEST"),
+                CreateLabelingRule(query="bad request", label="TEST"),
+                CreateLabelingRule(query="other", labels=["A", "B"]),
+            ],
+            {
+                "annotated_records": 1,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "total_records": 1,
+            },
+            "OK",
+        ),
+        (
+            [
+                CreateLabelingRule(query="ejemplo", label="TEST"),
+                CreateLabelingRule(query="bad request", label="TEST"),
+                CreateLabelingRule(query="other", labels=["A", "B"]),
+            ],
+            {
+                "annotated_records": 0,
+                "coverage": 1.0,
+                "total_records": 1,
+            },
+            None,
+        ),
+        (
+            [
+                CreateLabelingRule(query="ejemplo", label="TEST"),
+                CreateLabelingRule(query="bad request", label="bad.label"),
+                CreateLabelingRule(query="other", labels=["A", "B", "good.label"]),
+            ],
+            {
+                "annotated_records": 1,
+                "coverage": 1.0,
+                "coverage_annotated": 1.0,
+                "total_records": 1,
+            },
+            "good.label",
+        ),
+    ],
+)
+def test_dataset_rules_metrics(mocked_client, rules, expected_metrics, annotation):
     dataset = "test_dataset_rules_metrics"
-    log_some_records(mocked_client, dataset, annotation="OK")
-
-    for query in ["ejemplo", "bad query"]:
-        mocked_client.post(
-            f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-            json=CreateLabelingRule(
-                query=query, label="TEST", description="Description"
-            ).dict(),
-        )
-
-    response = mocked_client.get(
-        f"/api/datasets/TextClassification/{dataset}/labeling/rules/metrics"
-    )
-    assert response.status_code == 200, response.json()
-
-    metrics = DatasetLabelingRulesMetricsSummary.parse_obj(response.json())
-    assert metrics.coverage == 1
-    assert metrics.coverage_annotated == 1
-    assert metrics.total_records == 1
-    assert metrics.annotated_records == 1
-
+    log_some_records(mocked_client, dataset, annotation=annotation)
 
-def test_dataset_rules_metrics_without_annotation(mocked_client):
-    dataset = "test_dataset_rules_metrics_without_annotation"
-    log_some_records(mocked_client, dataset)
-
-    for query in ["ejemplo", "bad query"]:
+    for rule in rules:
         mocked_client.post(
             f"/api/datasets/TextClassification/{dataset}/labeling/rules",
-            json=CreateLabelingRule(
-                query=query, label="TEST", description="Description"
-            ).dict(),
+            json=rule.dict(),
         )
 
     response = mocked_client.get(
         f"/api/datasets/TextClassification/{dataset}/labeling/rules/metrics"
     )
     assert response.status_code == 200, response.json()
-
-    metrics = DatasetLabelingRulesMetricsSummary.parse_obj(response.json())
-    assert metrics.coverage == 1
-    assert metrics.total_records == 1
-    assert metrics.annotated_records == 0
-    assert metrics.coverage_annotated is None
+    assert response.json() == expected_metrics
 
 
 def test_rule_metric(mocked_client):
@@ -371,6 +472,25 @@ def test_rule_metric(mocked_client):
     assert metrics.precision is None
 
 
+def test_rule_metric_with_multiple_labels(mocked_client):
+    dataset = "test_rule_metric"
+    log_some_records(mocked_client, dataset, annotation="OK")
+
+    response = mocked_client.get(
+        f"/api/datasets/TextClassification/{dataset}/labeling/rules/ejemplo/metrics?label=A&label=OK"
+    )
+    assert response.status_code == 200
+    assert response.json() == {
+        "annotated_records": 1,
+        "correct": 1.0,
+        "coverage": 1.0,
+        "coverage_annotated": 1.0,
+        "incorrect": 1.0,
+        "precision": 0.5,
+        "total_records": 1,
+    }
+
+
 def test_search_records_with_uncovered_rules(mocked_client):
     dataset = "test_search_records_with_uncovered_rules"
     log_some_records(mocked_client, dataset, annotation="OK")