Skip to content

Commit

Permalink
Merge pull request #78 from Natalyl3/crowd-kit/docs
Browse files Browse the repository at this point in the history 
updated docs for Image Segmentation section
  • Loading branch information
@dustalov
dustalov committed Apr 27, 2023
2 parents b57aeb8 + a6ba08f commit 908dbd8
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 76 deletions.
2 changes: 1 addition & 1 deletion crowdkit/aggregation/classification/majority_vote.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'MajorityVote':
data (DataFrame): The training dataset of workers' labeling results
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `label` columns.
skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker
skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker`
and has the corresponding worker skill.
Returns:
Expand Down
82 changes: 52 additions & 30 deletions crowdkit/aggregation/image_segmentation/segmentation_em.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,28 @@

@attr.s
class SegmentationEM(BaseImageSegmentationAggregator):
"""The EM algorithm for the image segmentation task.
r"""The **Segmentation EM-algorithm** performs a categorical
aggregation task for each pixel: should it be included in the resulting aggregate or not.
This task is solved by the single-coin Dawid-Skene algorithm.
Each worker has a latent parameter `skill` that shows the probability of this worker to answer correctly.
This method performs a categorical aggregation task for each pixel: should
it be included to the resulting aggregate or no. This task is solved by
the single coin Dawid-Skene algorithm. Each worker has a latent parameter
"skill" that shows the probability of this worker to answer correctly.
Skills and true pixels' labels are optimized by the Expectation-Maximization
algorithm.
Skills and true pixel labels are optimized by the Expectation-Maximization algorithm:
1. **E-step**. Estimates the posterior probabilities using the specified workers' segmentations, the prior probabilities for each pixel,
and the workers' error probability vector.
2. **M-step**. Estimates the probability of a worker answering correctly using the specified workers' segmentations and the posterior probabilities for each pixel.
Doris Jung-Lin Lee. 2018.
Quality Evaluation Methods for Crowdsourced Image Segmentation
<https://ilpubs.stanford.edu:8090/1161/1/main.pdf>
D. Jung-Lin Lee, A. Das Sarma and A. Parameswaran. Aggregating Crowdsourced Image Segmentations.
*CEUR Workshop Proceedings. Vol. 2173*, (2018), 1-44.
<https://ceur-ws.org/Vol-2173/paper10.pdf>
Args:
n_iter: A number of EM iterations.
n_iter: The maximum number of EM iterations.
tol: The tolerance stopping criterion for iterative methods with a variable number of steps.
The algorithm converges when the loss change is less than the `tol` parameter.
eps: The convergence threshold.
Examples:
>>> import numpy as np
Expand All @@ -44,15 +50,34 @@ class SegmentationEM(BaseImageSegmentationAggregator):
>>> result = SegmentationEM().fit_predict(df)
Attributes:
segmentations_ (Series): Tasks' segmentations.
A pandas.Series indexed by `task` such that `labels.loc[task]`
is the tasks's aggregated segmentation.
segmentations_ (Series): The task segmentations.
The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]`
is the task aggregated segmentation.
segmentation_region_size_ (int): Segmentation region size.
segmentations_sizes_ (npt.NDArray[Any]): Sizes of image segmentations.
priors_ (Union[float, npt.NDArray[Any]]): The prior probabilities for each pixel to be included in the resulting aggregate.
Each probability is in the range from 0 to 1, all probabilities must sum up to 1.
posteriors_ (npt.NDArray[Any]): The posterior probabilities for each pixel to be included in the resulting aggregate.
Each probability is in the range from 0 to 1, all probabilities must sum up to 1.
errors_ (npt.NDArray[Any]): The workers' error probability vector.
loss_history_ (List[float]): A list of loss values during training.
"""

n_iter: int = attr.ib(default=10)
tol: float = attr.ib(default=1e-5)
eps: float = 1e-15
# segmentation_region_size_
# segmentations_sizes_
# segmentations_
# errors_
# priors_
# posteriors_
loss_history_: List[float] = attr.ib(init=False)

@staticmethod
Expand All @@ -62,9 +87,9 @@ def _e_step(
priors: Union[float, npt.NDArray[Any]],
) -> npt.NDArray[Any]:
"""
Perform E-step of algorithm.
Given workers' segmentations and error vector and priors
for each pixel calculates posteriori probabilities.
Performs E-step of the algorithm.
Estimates the posterior probabilities using the specified workers' segmentations, the prior probabilities for each pixel,
and the workers' error probability vector.
"""

weighted_seg = np.multiply(errors, segmentations.T.astype(float)).T + \
Expand All @@ -86,9 +111,8 @@ def _e_step(
def _m_step(segmentations: pd.Series, posteriors: npt.NDArray[Any],
segmentation_region_size: int, segmentations_sizes: npt.NDArray[Any]) -> npt.NDArray[Any]:
"""
Perform M-step of algorithm.
Given a priori probabilities for each pixel and the segmentation of the workers,
it estimates worker's errors probabilities vector.
Performs M-step of the algorithm.
Estimates the probability of a worker answering correctly using the specified workers' segmentations and the posterior probabilities for each pixel.
"""

mean_errors_expectation: npt.NDArray[Any] = (segmentations_sizes + posteriors.sum() -
Expand All @@ -115,7 +139,7 @@ def _evidence_lower_bound(self, segmentations: pd.Series,

def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[np.bool_]:
"""
Performs an expectation maximization algorithm for a single image.
Performs the Expectation-Maximization algorithm for a single image.
"""
priors = sum(segmentations) / len(segmentations)
segmentations = np.stack(segmentations.values)
Expand Down Expand Up @@ -144,11 +168,11 @@ def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[np.bool_]:
return cast(npt.NDArray[np.bool_], priors > 0.5)

def fit(self, data: pd.DataFrame) -> 'SegmentationEM':
"""Fit the model.
"""Fits the model to the training data with the EM algorithm.
Args:
data (DataFrame): Workers' segmentations.
A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'.
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
Returns:
SegmentationEM: self.
Expand All @@ -162,16 +186,14 @@ def fit(self, data: pd.DataFrame) -> 'SegmentationEM':
return self

def fit_predict(self, data: pd.DataFrame) -> pd.Series:
"""Fit the model and return the aggregated segmentations.
"""Fits the model to the training data and returns the aggregated segmentations.
Args:
data (DataFrame): Workers' segmentations.
A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'.
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
Returns:
Series: Tasks' segmentations.
A pandas.Series indexed by `task` such that `labels.loc[task]`
is the tasks's aggregated segmentation.
Series: Task segmentations. The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]` is the task aggregated segmentation.
"""

return self.fit(data).segmentations_
68 changes: 48 additions & 20 deletions crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,23 @@

@attr.s
class SegmentationMajorityVote(BaseImageSegmentationAggregator):
"""Segmentation Majority Vote - chooses a pixel if more than half of workers voted.
r"""The **Segmentation Majority Vote** algorithm chooses a pixel if and only if the pixel has "yes" votes
from at least half of all workers.
This method implements a straightforward approach to the image segmentations aggregation:
it assumes that if pixel is not inside in the worker's segmentation, this vote counts
as 0, otherwise, as 1. Next, the `SegmentationEM` aggregates these categorical values
for each pixel by the Majority Vote.
This method implements a straightforward approach to the image segmentation aggregation:
it assumes that if a pixel is not inside the worker's segmentation, this vote is considered to be equal to 0.
Otherwise, it is equal to 1. Then the `SegmentationEM` algorithm aggregates these categorical values
for each pixel by the Majority Vote algorithm.
The method also supports weighted majority voting if `skills` were provided to `fit` method.
The method also supports the weighted majority voting if the `skills` parameter is provided for the `fit` method.
Doris Jung-Lin Lee. 2018.
Quality Evaluation Methods for Crowdsourced Image Segmentation
<https://ilpubs.stanford.edu:8090/1161/1/main.pdf>
D. Jung-Lin Lee, A. Das Sarma and A. Parameswaran. Aggregating Crowdsourced Image Segmentations.
*CEUR Workshop Proceedings. Vol. 2173*, (2018), 1-44.
<https://ceur-ws.org/Vol-2173/paper10.pdf>
Args:
default_skill: A default skill value for missing skills.
default_skill: Default worker weight value.
Examples:
>>> import numpy as np
Expand All @@ -43,26 +45,41 @@ class SegmentationMajorityVote(BaseImageSegmentationAggregator):
>>> result = SegmentationMajorityVote().fit_predict(df)
Attributes:
segmentations_ (Series): Tasks' segmentations.
A pandas.Series indexed by `task` such that `labels.loc[task]`
is the tasks's aggregated segmentation.
segmentations_ (Series): The task segmentations.
The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]`
is the task aggregated segmentation.
skills_ (Series): The workers' skills. The `pandas.Series` data is indexed by `worker`
and has the corresponding worker skill.
on_missing_skill (str): A value which specifies how to handle assignments performed by workers with an unknown skill.
on_missing_skill (str): How to handle assignments done by workers with unknown skill.
Possible values:
* "error" — raise an exception if there is at least one assignment done by user with unknown skill;
* "ignore" — drop assignments with unknown skill values during prediction. Raise an exception if there is no
assignments with known skill for any task;
* value — default value will be used if skill is missing.
* "error" — raises an exception if there is at least one assignment performed by a worker with an unknown skill;
* "ignore" — drops assignments performed by workers with an unknown skill during prediction. Raises an exception if there are no
assignments with a known skill for any task;
* value — the default value will be used if a skill is missing.
"""

# segmentations_
# skills_

on_missing_skill: str = attr.ib(default='error')
default_skill: Optional[float] = attr.ib(default=None)

def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'SegmentationMajorityVote':
"""
Fit the model.
Fits the model to the training data.
Args:
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker`
and has the corresponding worker skill.
Returns:
SegmentationMajorityVote: self.
"""

data = data[['task', 'worker', 'segmentation']]
Expand All @@ -80,7 +97,18 @@ def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'SegmentationMajo

def fit_predict(self, data: pd.DataFrame, skills: Optional[pd.Series] = None) -> pd.Series:
"""
Fit the model and return the aggregated segmentations.
Fits the model to the training data and returns the aggregated segmentations.
Args:
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker`
and has the corresponding worker skill.
Returns:
Series: Task segmentations. The `pandas.Series` data is indexed by `task`
so that `segmentations.loc[task]` is the task aggregated segmentation.
"""

return self.fit(data, skills).segmentations_
61 changes: 36 additions & 25 deletions crowdkit/aggregation/image_segmentation/segmentation_rasa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,26 @@

@attr.s
class SegmentationRASA(BaseImageSegmentationAggregator):
"""Segmentation RASA - chooses a pixel if sum of weighted votes of each workers' more than 0.5.
r"""The **Segmentation RASA** (Reliability Aware Sequence Aggregation) algorithm chooses a pixel if the sum of the weighted votes of each worker is more than 0.5.
Algorithm works iteratively, at each step, the workers are reweighted in proportion to their distances
to the current answer estimation. The distance is considered as $1 - IOU$. Modification of the RASA method
for texts.
The Segmentation RASA algorithm consists of three steps:
1. Performs the weighted Majority Vote algorithm.
2. Calculates weights for each worker from the current Majority Vote estimation.
3. Performs the Segmentation RASA algorithm for a single image.
The algorithm works iteratively. At each step, the workers are reweighted in proportion to their distances
from the current answer estimation. The distance is calculated as $1 - IOU$, where `IOU` (Intersection over Union) is an extent of overlap of two boxes.
This algorithm is a modification of the RASA method for texts.
J. Li, F. Fukumoto. A Dataset of Crowdsourced Word Sequences: Collections and Answer Aggregation for Ground Truth Creation.
*Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP*, (2019), 24-28.
Jiyi Li.
A Dataset of Crowdsourced Word Sequences: Collections and Answer Aggregation for Ground Truth Creation.
*Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP*,
pages 24–28 Hong Kong, China, November 3, 2019.
<https://doi.org/10.18653/v1/D19-5904>
Args:
n_iter: A number of iterations.
n_iter: The maximum number of iterations.
tol: The tolerance stopping criterion for iterative methods with a variable number of steps.
The algorithm converges when the loss change is less than the `tol` parameter.
Examples:
>>> import numpy as np
Expand All @@ -44,9 +50,15 @@ class SegmentationRASA(BaseImageSegmentationAggregator):
>>> result = SegmentationRASA().fit_predict(df)
Attributes:
segmentations_ (Series): Tasks' segmentations.
A pandas.Series indexed by `task` such that `labels.loc[task]`
is the tasks's aggregated segmentation.
segmentations_ (Series): The task segmentations.
The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]`
is the task aggregated segmentation.
weights_ (npt.NDArray[Any]): A list of workers' weights.
mv_ (npt.NDArray[Any]): The weighted task segmentations calculated with the Majority Vote algorithm.
loss_history_ (List[float]): A list of loss values during training.
"""

n_iter: int = attr.ib(default=10)
Expand All @@ -57,18 +69,18 @@ class SegmentationRASA(BaseImageSegmentationAggregator):
@staticmethod
def _segmentation_weighted(segmentations: pd.Series, weights: npt.NDArray[Any]) -> npt.NDArray[Any]:
"""
Performs weighted majority vote algorithm.
Performs the weighted Majority Vote algorithm.
From the weights of all workers and their segmentation, performs a
weighted majority vote for the inclusion of each pixel in the answer.
From the weights of all workers and their segmentation, performs the
weighted Majority Vote for the inclusion of each pixel in the answer.
"""
weighted_segmentations = (weights * segmentations.T).T
return cast(npt.NDArray[Any], weighted_segmentations.sum(axis=0))

@staticmethod
def _calculate_weights(segmentations: pd.Series, mv: npt.NDArray[Any]) -> npt.NDArray[Any]:
"""
Calculates weights of each workers, from current majority vote estimation.
Calculates weights for each worker from the current Majority Vote estimation.
"""
intersection = (segmentations & mv).astype(float)
union = (segmentations | mv).astype(float)
Expand Down Expand Up @@ -109,11 +121,11 @@ def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[Any]:
return mv

def fit(self, data: pd.DataFrame) -> 'SegmentationRASA':
"""Fit the model.
"""Fits the model to the training data.
Args:
data (DataFrame): Workers' segmentations.
A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'.
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
Returns:
SegmentationRASA: self.
Expand All @@ -131,16 +143,15 @@ def fit(self, data: pd.DataFrame) -> 'SegmentationRASA':
return self

def fit_predict(self, data: pd.DataFrame) -> pd.Series:
"""Fit the model and return the aggregated segmentations.
"""Fits the model to the training data and returns the aggregated segmentations.
Args:
data (DataFrame): Workers' segmentations.
A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'.
data (DataFrame): The training dataset of workers' segmentations
which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns.
Returns:
Series: Tasks' segmentations.
A pandas.Series indexed by `task` such that `labels.loc[task]`
is the tasks's aggregated segmentation.
Series: Task segmentations. The `pandas.Series` data is indexed by `task`
so that `segmentations.loc[task]` is the task aggregated segmentation.
"""

return self.fit(data).segmentations_

0 comments on commit 908dbd8

Please sign in to comment.