Add authors content

.github/workflows/build.yml

@@ -0,0 +1,38 @@
+name: build
+
+on:
+  workflow_dispatch:
+  push:
+    branches: main
+
+jobs:
+  build-deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+        with:
+          tinytex: true
+          version: pre-release
+
+      - name: Install Computo extension for Quarto
+        run: |
+            quarto add --no-prompt computorg/computo-quarto-extension
+      - name: Install Python and Dependencies
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - run: pip install -r requirements.txt
+
+      - name: Render and Publish
+        uses: quarto-dev/quarto-actions/publish@v2
+        with:
+          target: gh-pages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,18 @@
+*.swp
+_build
+content.html
+
+/.quarto/
+
+_extensions/
+
+content_files/
+
+simus/
+
+.jupyter_cache/
+
+datasets/*/**
+!datasets/*/*.py
+!datasets/*/*.json
+/.luarc.json

_quarto.yml

@@ -0,0 +1,5 @@
+project:
+  title: "peerannot-library"
+
+
+

benchmark_crowdsourcing/.gitignore

@@ -0,0 +1,25 @@
+# Cache directories
+.pytest_cache
+__pycache__
+__cache__
+*.egg-info
+.coverage
+**/outputs
+joblib/
+
+# IDE specific folders
+.vscode
+
+# Config files
+benchopt.ini
+
+.DS_Store
+coverage.xml
+
+*.jpg
+
+data/
+
+tmp/
+
+simulation_data/

benchmark_crowdsourcing/README.rst

@@ -0,0 +1,37 @@
+
+Benchmark Crowdsourcing
+=====================
+|Build Status| |Python 3.6+|
+
+The label aggregation for crowdsourced classification datasets consists in presenting a set of $n_{task}$ training tasks to classify to a crowd.
+The label given for task $i$ by worker $j$ is denoted $y_i^{(j)}$.
+Given an aggregation strategy $\\texttt{agg}$ (like majority voting or Dawid and Skene's model), we look at the recovery of the underlying ground truth labels $y_i^*$:
+
+$$ \\mathrm{AccTrain} = \\frac{1}{n_{task}} \\sum_{i=1}^{n_{task}} 1\\!\\!1(\\hat{y}_i^{\\texttt{agg}}=y_i^*).$$
+
+Other objectives as the F1 score can also be considered.
+
+Install
+--------
+
+This benchmark can be run using the following commands:
+
+.. code-block::
+
+   $ pip install -U benchopt
+   $ git clone https://github.com/benchopt/benchmark_crowdsourcing
+   $ benchopt run benchmark_crowdsourcing
+
+Apart from the problem, options can be passed to ``benchopt run``, to restrict the benchmarks to some solvers or datasets, e.g.:
+
+.. code-block::
+
+	$ benchopt run benchmark_crowdsourcing -s solver1 -d dataset2 --max-runs 10 --n-repetitions 10
+
+
+Use ``benchopt run -h`` for more details about these options, or visit https://benchopt.github.io/api.html.
+
+.. |Build Status| image:: https://github.com/benchopt/benchmark_crowdsourcing/workflows/Tests/badge.svg
+   :target: https://github.com/benchopt/benchmark_crowdsourcing/actions
+.. |Python 3.6+| image:: https://img.shields.io/badge/python-3.6%2B-blue
+   :target: https://www.python.org/downloads/release/python-360/

benchmark_crowdsourcing/datasets/adult2.py

@@ -0,0 +1,79 @@
+from benchopt import BaseDataset
+from benchopt import safe_import_context
+
+with safe_import_context() as import_ctx:
+    import pooch
+    import numpy as np
+    import pandas as pd
+
+
+class Dataset(BaseDataset):
+
+    name = "AdultContent2"
+    install_cmd = "conda"
+    requirements = ["pip:pooch", "numpy", "pandas"]
+    classification_type = "text"
+
+    def prepare_data(self):
+        """
+        AdultContent2 dataset:
+            - partial ground truth in gold.txt
+            - votes in labels.txt formatted as:
+                AMTid url classcode
+        """
+        train_truth = pooch.retrieve(
+            "https://raw.githubusercontent.com/ipeirotis/"
+            "Get-Another-Label/master/data/AdultContent2/gold.txt",
+            known_hash=None,
+            path=pooch.os_cache(f"./data/{self.name}"),
+        )
+        votes = pooch.retrieve(
+            "https://raw.githubusercontent.com/ipeirotis/Get-Another-Label/"
+            "master/data/AdultContent2/labels.txt",
+            known_hash=None,
+            path=pooch.os_cache(f"./data/{self.name}"),
+        )
+
+        labels = {
+            key: code for (code, key) in enumerate(["G", "P", "R", "X", "B"])
+        }
+        train_truth = pd.read_csv(train_truth, header=None, sep="\t")
+        df = pd.read_csv(votes, header=None, sep="\t")
+        df[2] = df[2].replace(labels)
+        # handling tasks and partial ground truth
+        n_task = df[1].nunique()
+        self.task_converter = {
+            taskid: taskrank
+            for taskid, taskrank in zip(df[1].unique(), range(n_task))
+        }
+        gold_truth = train_truth[
+            train_truth.set_index([0]).index.isin(df.set_index([1]).index)
+        ].copy()
+        gold_truth[0].replace(self.task_converter, inplace=True)
+        gold_truth[1].replace(labels, inplace=True)
+        self.ground_truth = -np.ones(n_task)
+        for _, (i, k) in gold_truth.iterrows():
+            self.ground_truth[i] = k
+        # handling workers votes
+        workers = df[0].unique()
+        self.worker_converter = {
+            workerid: workerrank
+            for workerid, workerrank in zip(workers, range(len(workers)))
+        }
+        votes = {task: {} for task in self.task_converter.values()}
+        for idx, (worker, task, label) in df.iterrows():
+            votes[self.task_converter[task]][
+                self.worker_converter[worker]
+            ] = label
+        self.votes = votes
+
+    def get_data(self):
+        self.prepare_data()
+        data = dict(
+            votes=self.votes,
+            ground_truth=self.ground_truth,
+            n_worker=len(self.worker_converter),
+            n_task=len(self.votes),
+            n_classes=5,
+        )
+        return data

benchmark_crowdsourcing/datasets/bluebirds.py

@@ -0,0 +1,68 @@
+from benchopt import BaseDataset
+from benchopt import safe_import_context
+
+
+with safe_import_context() as import_ctx:
+    import pooch
+    import numpy as np
+    import yaml
+
+
+class Dataset(BaseDataset):
+
+    name = "bluebirds"
+    install_cmd = "conda"
+    requirements = ["pip:pooch", "numpy"]
+    classification_type = "image"
+
+    def prepare_data(self):
+        """
+        BlueBirds dataset:
+            - ground truth in gt.yaml
+            - votes in labels.yaml formatted as:
+                {worker: {task: vote}}
+        """
+        odie = pooch.create(
+            path=pooch.os_cache(f"./data/{self.name}"),
+            base_url="https://raw.githubusercontent.com/welinder/"
+            "cubam/public/demo/bluebirds/",
+            registry={
+                "gt.yaml": None,
+                "labels.yaml": None,
+            },
+        )
+        train_truth = odie.fetch("gt.yaml")
+        votes = odie.fetch("labels.yaml")
+        with open(train_truth, "r") as f:
+            ground_truth = yaml.safe_load(f)
+        self.task_converter = {
+            taskid: taskrank
+            for taskid, taskrank in zip(
+                ground_truth.keys(), range(len(ground_truth))
+            )
+        }
+        self.ground_truth = np.array(list(ground_truth.values()))
+        with open(votes, "r") as f:
+            labels = yaml.safe_load(f)
+        self.worker_converter = {
+            workerid: workerrank
+            for workerid, workerrank in zip(labels.keys(), range(len(labels)))
+        }
+        votes = {task: {} for task in self.task_converter.values()}
+        for worker, values in labels.items():
+            for task, label in values.items():
+                votes[self.task_converter[task]][
+                    self.worker_converter[worker]
+                ] = int(label)
+        self.votes = votes
+
+    def get_data(self):
+        self.prepare_data()
+        data = dict(
+            votes=self.votes,
+            ground_truth=self.ground_truth,
+            n_worker=len(self.worker_converter),
+            n_task=len(self.votes),
+            n_classes=2,
+        )
+        return data

benchmark_crowdsourcing/datasets/labelme.py

@@ -0,0 +1,39 @@
+from benchopt import BaseDataset, safe_import_context
+
+with safe_import_context() as import_ctx:
+    import numpy as np
+    import json
+    from pathlib import Path
+
+
+class Dataset(BaseDataset):
+
+    name = "LabelMe"
+    requirements = ["pip:json", "numpy"]
+    install_cmd = "conda"
+
+    def votes(self):
+        current_path = Path(__file__).parent
+        with open(
+            current_path
+            / ".."
+            / ".."
+            / "datasets"
+            / "labelme"
+            / "answers.json",
+            "r",
+        ) as f:
+            votes = json.load(f)
+        true_labels = np.load(current_path / "truth_labelme.npy")
+        self.answers = votes
+        self.ground_truth = true_labels
+
+    def get_data(self):
+        self.votes()
+        return dict(
+            votes=self.answers,
+            ground_truth=self.ground_truth,
+            n_worker=77,
+            n_task=1000,
+            n_classes=8,
+        )

benchmark_crowdsourcing/datasets/simulated.py

@@ -0,0 +1,59 @@
+from benchopt import BaseDataset, safe_import_context
+
+with safe_import_context() as import_ctx:
+    import numpy as np
+
+
+class Dataset(BaseDataset):
+
+    name = "Simulated"
+    install_cmd = "conda"
+
+    # List of parameters to generate the datasets. The benchmark will consider
+    # the cross product for each key in the dictionary.
+    parameters = {
+        "n_task": [200],
+        "n_worker": [50],
+        "n_classes": [3],
+        "ratio": [0.5],  # probability to generate a spammer
+    }
+
+    def votes(self):
+        rng = np.random.default_rng(42)
+        true_labels = rng.choice(
+            self.n_classes, size=self.n_task, replace=True
+        )
+        votes = {}
+        type_worker = rng.choice(
+            ["hammer", "spammer"],
+            size=self.n_worker,
+            p=[1 - self.ratio, self.ratio],
+        )
+        for i in range(self.n_task):
+            votes[i] = {}
+            for j in range(self.n_worker):
+                if type_worker[j] == "hammer":
+                    votes[i][j] = true_labels[i]
+                else:
+                    votes[i][j] = rng.choice(range(1, self.n_classes))
+        self.answers = votes
+        self.ground_truth = true_labels
+
+    def get_data(self):
+        self.votes()
+        return dict(
+            votes=self.answers,
+            ground_truth=self.ground_truth,
+            n_worker=self.n_worker,
+            n_task=self.n_task,
+            n_classes=self.n_classes,
+        )
+
+        rng = np.random.RandomState(self.random_state)
+        X = rng.randn(self.n_samples, self.n_features)
+        y = rng.randn(self.n_samples)
+
+        # `data` holds the keyword arguments for the `set_data` method of the
+        # objective.
+        # They are customizable.
+        return dict(X=X, y=y)

benchmark_crowdsourcing/example_config.yml

@@ -0,0 +1,19 @@
+objective:
+  - Crowdsourcing
+
+dataset: # dataset to run benchmark on
+  - bluebirds
+  - adultcontent2
+  - rte
+solver: # list of example solvers to do benchmark
+  - peerannot[strategy=MV]
+  - peerannot[strategy=DS]
+  - peerannot[strategy=GLAD]
+  - peerannot[strategy=WDS]
+  - crowd-kit
+  - FDS
+  - DS-mcmc
+# note: use more repetitions to have quantiles on the plot
+n-repetitions: 1
+max-runs: 5
+timeout: 15m