diff --git a/crowdkit/aggregation/classification/majority_vote.py b/crowdkit/aggregation/classification/majority_vote.py index 78cf55cc..05095b61 100644 --- a/crowdkit/aggregation/classification/majority_vote.py +++ b/crowdkit/aggregation/classification/majority_vote.py @@ -85,7 +85,7 @@ def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'MajorityVote': data (DataFrame): The training dataset of workers' labeling results which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `label` columns. - skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker + skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker` and has the corresponding worker skill. Returns: diff --git a/crowdkit/aggregation/image_segmentation/segmentation_em.py b/crowdkit/aggregation/image_segmentation/segmentation_em.py index ec70599c..4f395977 100644 --- a/crowdkit/aggregation/image_segmentation/segmentation_em.py +++ b/crowdkit/aggregation/image_segmentation/segmentation_em.py @@ -12,22 +12,28 @@ @attr.s class SegmentationEM(BaseImageSegmentationAggregator): - """The EM algorithm for the image segmentation task. + r"""The **Segmentation EM-algorithm** performs a categorical + aggregation task for each pixel: should it be included in the resulting aggregate or not. + This task is solved by the single-coin Dawid-Skene algorithm. + Each worker has a latent parameter `skill` that shows the probability of this worker to answer correctly. - This method performs a categorical aggregation task for each pixel: should - it be included to the resulting aggregate or no. This task is solved by - the single coin Dawid-Skene algorithm. Each worker has a latent parameter - "skill" that shows the probability of this worker to answer correctly. - Skills and true pixels' labels are optimized by the Expectation-Maximization - algorithm. + Skills and true pixel labels are optimized by the Expectation-Maximization algorithm: + 1. **E-step**. Estimates the posterior probabilities using the specified workers' segmentations, the prior probabilities for each pixel, + and the workers' error probability vector. + 2. **M-step**. Estimates the probability of a worker answering correctly using the specified workers' segmentations and the posterior probabilities for each pixel. - Doris Jung-Lin Lee. 2018. - Quality Evaluation Methods for Crowdsourced Image Segmentation - + D. Jung-Lin Lee, A. Das Sarma and A. Parameswaran. Aggregating Crowdsourced Image Segmentations. + *CEUR Workshop Proceedings. Vol. 2173*, (2018), 1-44. + + + Args: - n_iter: A number of EM iterations. + n_iter: The maximum number of EM iterations. + tol: The tolerance stopping criterion for iterative methods with a variable number of steps. + The algorithm converges when the loss change is less than the `tol` parameter. + eps: The convergence threshold. Examples: >>> import numpy as np @@ -44,15 +50,34 @@ class SegmentationEM(BaseImageSegmentationAggregator): >>> result = SegmentationEM().fit_predict(df) Attributes: - segmentations_ (Series): Tasks' segmentations. - A pandas.Series indexed by `task` such that `labels.loc[task]` - is the tasks's aggregated segmentation. + segmentations_ (Series): The task segmentations. + The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]` + is the task aggregated segmentation. + + segmentation_region_size_ (int): Segmentation region size. + + segmentations_sizes_ (npt.NDArray[Any]): Sizes of image segmentations. + + priors_ (Union[float, npt.NDArray[Any]]): The prior probabilities for each pixel to be included in the resulting aggregate. + Each probability is in the range from 0 to 1, all probabilities must sum up to 1. + + posteriors_ (npt.NDArray[Any]): The posterior probabilities for each pixel to be included in the resulting aggregate. + Each probability is in the range from 0 to 1, all probabilities must sum up to 1. + + errors_ (npt.NDArray[Any]): The workers' error probability vector. + + loss_history_ (List[float]): A list of loss values during training. """ n_iter: int = attr.ib(default=10) tol: float = attr.ib(default=1e-5) eps: float = 1e-15 + # segmentation_region_size_ + # segmentations_sizes_ # segmentations_ + # errors_ + # priors_ + # posteriors_ loss_history_: List[float] = attr.ib(init=False) @staticmethod @@ -62,9 +87,9 @@ def _e_step( priors: Union[float, npt.NDArray[Any]], ) -> npt.NDArray[Any]: """ - Perform E-step of algorithm. - Given workers' segmentations and error vector and priors - for each pixel calculates posteriori probabilities. + Performs E-step of the algorithm. + Estimates the posterior probabilities using the specified workers' segmentations, the prior probabilities for each pixel, + and the workers' error probability vector. """ weighted_seg = np.multiply(errors, segmentations.T.astype(float)).T + \ @@ -86,9 +111,8 @@ def _e_step( def _m_step(segmentations: pd.Series, posteriors: npt.NDArray[Any], segmentation_region_size: int, segmentations_sizes: npt.NDArray[Any]) -> npt.NDArray[Any]: """ - Perform M-step of algorithm. - Given a priori probabilities for each pixel and the segmentation of the workers, - it estimates worker's errors probabilities vector. + Performs M-step of the algorithm. + Estimates the probability of a worker answering correctly using the specified workers' segmentations and the posterior probabilities for each pixel. """ mean_errors_expectation: npt.NDArray[Any] = (segmentations_sizes + posteriors.sum() - @@ -115,7 +139,7 @@ def _evidence_lower_bound(self, segmentations: pd.Series, def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[np.bool_]: """ - Performs an expectation maximization algorithm for a single image. + Performs the Expectation-Maximization algorithm for a single image. """ priors = sum(segmentations) / len(segmentations) segmentations = np.stack(segmentations.values) @@ -144,11 +168,11 @@ def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[np.bool_]: return cast(npt.NDArray[np.bool_], priors > 0.5) def fit(self, data: pd.DataFrame) -> 'SegmentationEM': - """Fit the model. + """Fits the model to the training data with the EM algorithm. Args: - data (DataFrame): Workers' segmentations. - A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'. + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. Returns: SegmentationEM: self. @@ -162,16 +186,14 @@ def fit(self, data: pd.DataFrame) -> 'SegmentationEM': return self def fit_predict(self, data: pd.DataFrame) -> pd.Series: - """Fit the model and return the aggregated segmentations. + """Fits the model to the training data and returns the aggregated segmentations. Args: - data (DataFrame): Workers' segmentations. - A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'. + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. Returns: - Series: Tasks' segmentations. - A pandas.Series indexed by `task` such that `labels.loc[task]` - is the tasks's aggregated segmentation. + Series: Task segmentations. The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]` is the task aggregated segmentation. """ return self.fit(data).segmentations_ diff --git a/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py b/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py index 8cc4e5c6..40694fd0 100644 --- a/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py +++ b/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py @@ -12,21 +12,23 @@ @attr.s class SegmentationMajorityVote(BaseImageSegmentationAggregator): - """Segmentation Majority Vote - chooses a pixel if more than half of workers voted. + r"""The **Segmentation Majority Vote** algorithm chooses a pixel if and only if the pixel has "yes" votes + from at least half of all workers. - This method implements a straightforward approach to the image segmentations aggregation: - it assumes that if pixel is not inside in the worker's segmentation, this vote counts - as 0, otherwise, as 1. Next, the `SegmentationEM` aggregates these categorical values - for each pixel by the Majority Vote. + This method implements a straightforward approach to the image segmentation aggregation: + it assumes that if a pixel is not inside the worker's segmentation, this vote is considered to be equal to 0. + Otherwise, it is equal to 1. Then the `SegmentationEM` algorithm aggregates these categorical values + for each pixel by the Majority Vote algorithm. - The method also supports weighted majority voting if `skills` were provided to `fit` method. + The method also supports the weighted majority voting if the `skills` parameter is provided for the `fit` method. - Doris Jung-Lin Lee. 2018. - Quality Evaluation Methods for Crowdsourced Image Segmentation - + D. Jung-Lin Lee, A. Das Sarma and A. Parameswaran. Aggregating Crowdsourced Image Segmentations. + *CEUR Workshop Proceedings. Vol. 2173*, (2018), 1-44. + + Args: - default_skill: A default skill value for missing skills. + default_skill: Default worker weight value. Examples: >>> import numpy as np @@ -43,26 +45,41 @@ class SegmentationMajorityVote(BaseImageSegmentationAggregator): >>> result = SegmentationMajorityVote().fit_predict(df) Attributes: - segmentations_ (Series): Tasks' segmentations. - A pandas.Series indexed by `task` such that `labels.loc[task]` - is the tasks's aggregated segmentation. + segmentations_ (Series): The task segmentations. + The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]` + is the task aggregated segmentation. + + skills_ (Series): The workers' skills. The `pandas.Series` data is indexed by `worker` + and has the corresponding worker skill. + + on_missing_skill (str): A value which specifies how to handle assignments performed by workers with an unknown skill. - on_missing_skill (str): How to handle assignments done by workers with unknown skill. Possible values: - * "error" — raise an exception if there is at least one assignment done by user with unknown skill; - * "ignore" — drop assignments with unknown skill values during prediction. Raise an exception if there is no - assignments with known skill for any task; - * value — default value will be used if skill is missing. + * "error" — raises an exception if there is at least one assignment performed by a worker with an unknown skill; + * "ignore" — drops assignments performed by workers with an unknown skill during prediction. Raises an exception if there are no + assignments with a known skill for any task; + * value — the default value will be used if a skill is missing. """ # segmentations_ + # skills_ on_missing_skill: str = attr.ib(default='error') default_skill: Optional[float] = attr.ib(default=None) def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'SegmentationMajorityVote': """ - Fit the model. + Fits the model to the training data. + + Args: + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. + + skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker` + and has the corresponding worker skill. + + Returns: + SegmentationMajorityVote: self. """ data = data[['task', 'worker', 'segmentation']] @@ -80,7 +97,18 @@ def fit(self, data: pd.DataFrame, skills: pd.Series = None) -> 'SegmentationMajo def fit_predict(self, data: pd.DataFrame, skills: Optional[pd.Series] = None) -> pd.Series: """ - Fit the model and return the aggregated segmentations. + Fits the model to the training data and returns the aggregated segmentations. + + Args: + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. + + skills (Series): The workers' skills. The `pandas.Series` data is indexed by `worker` + and has the corresponding worker skill. + + Returns: + Series: Task segmentations. The `pandas.Series` data is indexed by `task` + so that `segmentations.loc[task]` is the task aggregated segmentation. """ return self.fit(data, skills).segmentations_ diff --git a/crowdkit/aggregation/image_segmentation/segmentation_rasa.py b/crowdkit/aggregation/image_segmentation/segmentation_rasa.py index 397fffe8..687a6f45 100644 --- a/crowdkit/aggregation/image_segmentation/segmentation_rasa.py +++ b/crowdkit/aggregation/image_segmentation/segmentation_rasa.py @@ -14,20 +14,26 @@ @attr.s class SegmentationRASA(BaseImageSegmentationAggregator): - """Segmentation RASA - chooses a pixel if sum of weighted votes of each workers' more than 0.5. + r"""The **Segmentation RASA** (Reliability Aware Sequence Aggregation) algorithm chooses a pixel if the sum of the weighted votes of each worker is more than 0.5. - Algorithm works iteratively, at each step, the workers are reweighted in proportion to their distances - to the current answer estimation. The distance is considered as $1 - IOU$. Modification of the RASA method - for texts. + The Segmentation RASA algorithm consists of three steps: + 1. Performs the weighted Majority Vote algorithm. + 2. Calculates weights for each worker from the current Majority Vote estimation. + 3. Performs the Segmentation RASA algorithm for a single image. + + The algorithm works iteratively. At each step, the workers are reweighted in proportion to their distances + from the current answer estimation. The distance is calculated as $1 - IOU$, where `IOU` (Intersection over Union) is an extent of overlap of two boxes. + This algorithm is a modification of the RASA method for texts. + + J. Li, F. Fukumoto. A Dataset of Crowdsourced Word Sequences: Collections and Answer Aggregation for Ground Truth Creation. + *Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP*, (2019), 24-28. - Jiyi Li. - A Dataset of Crowdsourced Word Sequences: Collections and Answer Aggregation for Ground Truth Creation. - *Proceedings of the First Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP*, - pages 24–28 Hong Kong, China, November 3, 2019. Args: - n_iter: A number of iterations. + n_iter: The maximum number of iterations. + tol: The tolerance stopping criterion for iterative methods with a variable number of steps. + The algorithm converges when the loss change is less than the `tol` parameter. Examples: >>> import numpy as np @@ -44,9 +50,15 @@ class SegmentationRASA(BaseImageSegmentationAggregator): >>> result = SegmentationRASA().fit_predict(df) Attributes: - segmentations_ (Series): Tasks' segmentations. - A pandas.Series indexed by `task` such that `labels.loc[task]` - is the tasks's aggregated segmentation. + segmentations_ (Series): The task segmentations. + The `pandas.Series` data is indexed by `task` so that `segmentations.loc[task]` + is the task aggregated segmentation. + + weights_ (npt.NDArray[Any]): A list of workers' weights. + + mv_ (npt.NDArray[Any]): The weighted task segmentations calculated with the Majority Vote algorithm. + + loss_history_ (List[float]): A list of loss values during training. """ n_iter: int = attr.ib(default=10) @@ -57,10 +69,10 @@ class SegmentationRASA(BaseImageSegmentationAggregator): @staticmethod def _segmentation_weighted(segmentations: pd.Series, weights: npt.NDArray[Any]) -> npt.NDArray[Any]: """ - Performs weighted majority vote algorithm. + Performs the weighted Majority Vote algorithm. - From the weights of all workers and their segmentation, performs a - weighted majority vote for the inclusion of each pixel in the answer. + From the weights of all workers and their segmentation, performs the + weighted Majority Vote for the inclusion of each pixel in the answer. """ weighted_segmentations = (weights * segmentations.T).T return cast(npt.NDArray[Any], weighted_segmentations.sum(axis=0)) @@ -68,7 +80,7 @@ def _segmentation_weighted(segmentations: pd.Series, weights: npt.NDArray[Any]) @staticmethod def _calculate_weights(segmentations: pd.Series, mv: npt.NDArray[Any]) -> npt.NDArray[Any]: """ - Calculates weights of each workers, from current majority vote estimation. + Calculates weights for each worker from the current Majority Vote estimation. """ intersection = (segmentations & mv).astype(float) union = (segmentations | mv).astype(float) @@ -109,11 +121,11 @@ def _aggregate_one(self, segmentations: pd.Series) -> npt.NDArray[Any]: return mv def fit(self, data: pd.DataFrame) -> 'SegmentationRASA': - """Fit the model. + """Fits the model to the training data. Args: - data (DataFrame): Workers' segmentations. - A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'. + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. Returns: SegmentationRASA: self. @@ -131,16 +143,15 @@ def fit(self, data: pd.DataFrame) -> 'SegmentationRASA': return self def fit_predict(self, data: pd.DataFrame) -> pd.Series: - """Fit the model and return the aggregated segmentations. + """Fits the model to the training data and returns the aggregated segmentations. Args: - data (DataFrame): Workers' segmentations. - A pandas.DataFrame containing `worker`, `task` and `segmentation` columns'. + data (DataFrame): The training dataset of workers' segmentations + which is represented as the `pandas.DataFrame` data containing `task`, `worker`, and `segmentation` columns. Returns: - Series: Tasks' segmentations. - A pandas.Series indexed by `task` such that `labels.loc[task]` - is the tasks's aggregated segmentation. + Series: Task segmentations. The `pandas.Series` data is indexed by `task` + so that `segmentations.loc[task]` is the task aggregated segmentation. """ return self.fit(data).segmentations_