From 20e64edb7068f8b957cf01412d85db01c6d1e25b Mon Sep 17 00:00:00 2001 From: Jakub Date: Thu, 21 Jun 2018 17:57:35 +0200 Subject: [PATCH] Dev lgbm (#147) (#152) * initial restructure * thresholds on unet output * added gmean tta, experimented with thresholding (#125) * feature exractor and lightgbm * pipeline is running ok * tmp commit * lgbm ready for tests * tmp * faster nms and feature extraction * small fix * cleaning * Dev repo cleanup (#138) * initial restructure * clean structure (#126) * clean structure * correct readme * further cleaning * Dev apply transformer (#131) * clean structure * correct readme * further cleaning * resizer docstring * couple docstrings * make apply transformer, memory cache * fixes * postprocessing docstrings * fixes in PR * Dev repo cleanup (#132) * cleanup * remove src. * Dev clean tta (#134) * added resize padding, refactored inference pipelines * refactored piepliens * added color shift augmentation * reduced caching to just mask_resize * updated config * Dev-repo_cleanup models and losses docstrings (#135) * models and losses docstrings * small fixes in docstrings * resolve conflicts in with TTA PR (#137) * refactor in stream mode (#139) * hot fix of mask_postprocessing in tta with new make transformer * finishing merge * finishing merge v2 * finishing merge v3 * finishing merge v4 * tmp commit * lgbm train and evaluate pipelines run correctly * something is not yes * fix * working lgbm training with ugly train_mode=True * back to pipelines.py * small fix * preparing PR * preparing PR v2 * preparing PR v2 * fix * fix_2 * fix_3 * fix_4 --- neptune.yaml | 22 ++++- src/callbacks.py | 4 +- src/loaders.py | 6 +- src/models.py | 97 +++++++++++++++++- src/pipeline_config.py | 24 ++++- src/pipeline_manager.py | 59 ++++++++--- src/pipelines.py | 98 ++++++++++++++++++- src/postprocessing.py | 189 +++++++++++++++++++++++++++++++++++- src/preparation.py | 4 +- src/steps/base.py | 1 + src/steps/sklearn/models.py | 2 +- src/utils.py | 30 ++++-- 12 files changed, 485 insertions(+), 51 deletions(-) diff --git a/neptune.yaml b/neptune.yaml index f3b5940..38bd19e 100644 --- a/neptune.yaml +++ b/neptune.yaml @@ -1,7 +1,7 @@ project: YOUR_PROJECT_NAME name: mapping_challenge_open_solution -tags: [solution_1] +tags: [solution_5] metric: channel: 'Final Validation Score' @@ -41,7 +41,6 @@ parameters: loader_mode: resize stream_mode: 0 - # General parameters image_h: 256 image_w: 256 @@ -86,11 +85,10 @@ parameters: # Postprocessing threshold: 0.5 - min_nuclei_size: 20 - erosion_percentages: '[10,20,30]' erode_selem_size: 0 dilate_selem_size: 2 tta_aggregation_method: gmean + nms__iou_threshold: 0.5 # Inference padding crop_image_h: 300 @@ -100,4 +98,18 @@ parameters: pad_method: 'replicate' #Neptune monitor - unet_outputs_to_plot: '["multichannel_map",]' \ No newline at end of file + unet_outputs_to_plot: '["multichannel_map",]' + +#Scoring model + scoring_model: 'lgbm' + scoring_model__num_training_examples: 10000 + +#LightGBM + lgbm__learning_rate: 0.001 + lgbm__num_leaves: 10 + lgbm__min_data: 50 + lgbm__max_depth: 10 + lgbm__number_of_trees: 100 + lgbm__early_stopping: 5 + lgbm__train_size: 0.7 + lgbm__target: 'iou' diff --git a/src/callbacks.py b/src/callbacks.py index c274f5d..6717d0a 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -13,7 +13,7 @@ from .steps.utils import get_logger from .steps.pytorch.callbacks import NeptuneMonitor, ValidationMonitor from .utils import softmax, coco_evaluation, create_annotations, make_apply_transformer -from .pipeline_config import CATEGORY_IDS, Y_COLUMNS_SCORING +from .pipeline_config import CATEGORY_IDS, Y_COLUMNS_SCORING, CATEGORY_LAYERS logger = get_logger() @@ -200,7 +200,7 @@ def _generate_prediction(self, cache_dirpath, outputs): output = pipeline.transform(data) y_pred = output['y_pred'] - prediction = create_annotations(self.meta_valid, y_pred, logger, CATEGORY_IDS) + prediction = create_annotations(self.meta_valid, y_pred, logger, CATEGORY_IDS, CATEGORY_LAYERS) return prediction diff --git a/src/loaders.py b/src/loaders.py index b88150e..3064989 100644 --- a/src/loaders.py +++ b/src/loaders.py @@ -436,9 +436,9 @@ def _get_tta_data(self, i, row): class TestTimeAugmentationAggregator(BaseTransformer): - def __init__(self, method, nthreads): + def __init__(self, method, num_threads): self.method = method - self.nthreads = nthreads + self.num_threads = num_threads @property def agg_method(self): @@ -456,7 +456,7 @@ def transform(self, images, tta_params, img_ids, **kwargs): img_ids=img_ids, agg_method=self.agg_method) unique_img_ids = set(img_ids) - threads = min(self.nthreads, len(unique_img_ids)) + threads = min(self.num_threads, len(unique_img_ids)) with mp.pool.ThreadPool(threads) as executor: averages_images = executor.map(_aggregate_augmentations, unique_img_ids) return {'aggregated_prediction': averages_images} diff --git a/src/models.py b/src/models.py index 86196a7..935c8eb 100644 --- a/src/models.py +++ b/src/models.py @@ -4,6 +4,10 @@ import torch.nn as nn from torch.autograd import Variable from torch import optim +import pandas as pd +from sklearn.model_selection import train_test_split +from sklearn.externals import joblib +from sklearn.ensemble import RandomForestRegressor from .callbacks import NeptuneMonitorSegmentation, ValidationMonitorSegmentation from .steps.pytorch.architectures.unet import UNet @@ -11,6 +15,7 @@ ExperimentTiming, ExponentialLRScheduler, EarlyStopping from .steps.pytorch.models import Model from .steps.pytorch.validation import multiclass_segmentation_loss, DiceLoss +from .steps.sklearn.models import LightGBM, make_transformer, SklearnRegressor from .utils import softmax from .unet_models import AlbuNet, UNet11, UNetVGG16, UNetResNet @@ -159,9 +164,12 @@ def __init__(self, architecture_config, training_config, callbacks_config): class PyTorchUNetWeightedStream(BasePyTorchUNet): def __init__(self, architecture_config, training_config, callbacks_config): super().__init__(architecture_config, training_config, callbacks_config) - weighted_loss = partial(multiclass_weighted_cross_entropy, - **get_loss_variables(**architecture_config['weighted_cross_entropy'])) - loss = partial(mixed_dice_cross_entropy_loss, dice_weight=architecture_config['loss_weights']['dice_mask'], + weights_function = partial(get_weights, **architecture_config['weighted_cross_entropy']) + weighted_loss = partial(multiclass_weighted_cross_entropy, weights_function=weights_function) + dice_loss = partial(multiclass_dice_loss, excluded_classes=[0]) + loss = partial(mixed_dice_cross_entropy_loss, + dice_loss=dice_loss, + dice_weight=architecture_config['loss_weights']['dice_mask'], cross_entropy_weight=architecture_config['loss_weights']['bce_mask'], cross_entropy_loss=weighted_loss, **architecture_config['dice']) @@ -201,6 +209,81 @@ def _transform(self, datagen, validation_datagen=None): self.model.train() +class ScoringLightGBM(LightGBM): + def __init__(self, model_params, training_params, train_size, target): + self.train_size = train_size + self.target = target + self.feature_names = [] + self.estimator = None + super().__init__(model_params, training_params) + + def fit(self, features, **kwargs): + df_features = _convert_features_to_df(features) + train_data, val_data = train_test_split(df_features, train_size=self.train_size) + self.feature_names = list(df_features.columns.drop(self.target)) + super().fit(X=train_data[self.feature_names], + y=train_data[self.target], + X_valid=val_data[self.feature_names], + y_valid=val_data[self.target], + feature_names=self.feature_names, + categorical_features=[]) + return self + + def transform(self, features, **kwargs): + scores = [] + for image_features in features: + image_scores = [] + for layer_features in image_features: + if len(layer_features) > 0: + layer_scores = super().transform(layer_features[self.feature_names]) + image_scores.append(list(layer_scores['prediction'])) + else: + image_scores.append([]) + scores.append(image_scores) + return {'scores': scores} + + def save(self, filepath): + joblib.dump((self.estimator, self.feature_names), filepath) + + def load(self, filepath): + self.estimator, self.feature_names = joblib.load(filepath) + + +class ScoringRandomForest(SklearnRegressor): + def __init__(self, train_size, target, **kwargs): + self.train_size = train_size + self.target = target + self.feature_names = [] + self.estimator = RandomForestRegressor() + + def fit(self, features, **kwargs): + df_features = _convert_features_to_df(features) + train_data, val_data = train_test_split(df_features, train_size=self.train_size) + self.feature_names = list(df_features.columns.drop(self.target)) + super().fit(X=train_data[self.feature_names], + y=train_data[self.target]) + return self + + def transform(self, features, **kwargs): + scores = [] + for image_features in features: + image_scores = [] + for layer_features in image_features: + if len(layer_features) > 0: + layer_scores = super().transform(layer_features[self.feature_names]) + image_scores.append(list(layer_scores['prediction'])) + else: + image_scores.append([]) + scores.append(image_scores) + return {'scores': scores} + + def save(self, filepath): + joblib.dump((self.estimator, self.feature_names), filepath) + + def load(self, filepath): + self.estimator, self.feature_names = joblib.load(filepath) + + def weight_regularization_unet(model, regularize, weight_decay_conv2d): if regularize: parameter_list = [{'params': model.parameters(), 'weight_decay': weight_decay_conv2d}] @@ -369,3 +452,11 @@ def multiclass_dice_loss(output, target, smooth=0, activation='softmax', exclude class_target.data = class_target.data.float() loss += dice(output[:, class_nr, :, :], class_target) return loss + + +def _convert_features_to_df(features): + df_features = [] + for image_features in features: + for layer_features in image_features[1:]: + df_features.append(layer_features) + return pd.concat(df_features) \ No newline at end of file diff --git a/src/pipeline_config.py b/src/pipeline_config.py index 5bd91a4..6a0e97f 100644 --- a/src/pipeline_config.py +++ b/src/pipeline_config.py @@ -12,8 +12,9 @@ X_COLUMNS = ['file_path_image'] Y_COLUMNS = ['file_path_mask_eroded_0_dilated_0'] Y_COLUMNS_SCORING = ['ImageId'] -CATEGORY_IDS = [None, 100] SEED = 1234 +CATEGORY_IDS = [None, 100] +CATEGORY_LAYERS = [1, 19] MEAN = [0.485, 0.456, 0.406] STD = [0.229, 0.224, 0.225] @@ -121,9 +122,8 @@ 'rotation': True, 'color_shift_runs': False}, 'tta_aggregator': {'method': params.tta_aggregation_method, - 'nthreads': params.num_threads + 'num_threads': params.num_threads }, - 'dropper': {'min_size': params.min_nuclei_size}, 'postprocessor': {'mask_dilation': {'dilate_selem_size': params.dilate_selem_size }, 'mask_erosion': {'erode_selem_size': params.erode_selem_size @@ -131,5 +131,23 @@ 'prediction_crop': {'h_crop': params.crop_image_h, 'w_crop': params.crop_image_w }, + 'scoring_model': params.scoring_model, + 'lightGBM': {'model_params': {'learning_rate': params.lgbm__learning_rate, + 'boosting_type': 'gbdt', + 'objective': 'regression', + 'metric': 'regression_l2', + 'sub_feature': 1.0, + 'num_leaves': params.lgbm__num_leaves, + 'min_data': params.lgbm__min_data, + 'max_depth': params.lgbm__max_depth}, + 'training_params': {'number_boosting_rounds': params.lgbm__number_of_trees, + 'early_stopping_rounds': params.lgbm__early_stopping}, + 'train_size': params.lgbm__train_size, + 'target': params.lgbm__target + }, + 'random_forest': {'train_size': params.lgbm__train_size, + 'target': params.lgbm__target}, + 'nms': {'iou_threshold': params.nms__iou_threshold, + 'num_threads': params.num_threads}, } }) diff --git a/src/pipeline_manager.py b/src/pipeline_manager.py index a433431..19cd610 100644 --- a/src/pipeline_manager.py +++ b/src/pipeline_manager.py @@ -5,8 +5,9 @@ from deepsense import neptune import crowdai import json +from pycocotools.coco import COCO -from .pipeline_config import SOLUTION_CONFIG, Y_COLUMNS_SCORING, CATEGORY_IDS, SEED +from .pipeline_config import SOLUTION_CONFIG, Y_COLUMNS_SCORING, CATEGORY_IDS, SEED, CATEGORY_LAYERS from .pipelines import PIPELINES from .preparation import overlay_masks from .utils import init_logger, read_params, generate_metadata, set_seed, coco_evaluation, \ @@ -55,7 +56,7 @@ def prepare_masks(dev_mode, logger, params): erode=params.erode_selem_size, dilate=params.dilate_selem_size, is_small=dev_mode, - nthreads=params.num_threads, + num_threads=params.num_threads, border_width=params.border_width, small_annotations_size=params.small_annotations_size) @@ -87,15 +88,25 @@ def train(pipeline_name, dev_mode, logger, params, seed): meta_train = meta[meta['is_train'] == 1] meta_valid = meta[meta['is_valid'] == 1] + train_mode = True + meta_valid = meta_valid.sample(int(params.evaluation_data_sample), random_state=seed) if dev_mode: meta_train = meta_train.sample(20, random_state=seed) meta_valid = meta_valid.sample(10, random_state=seed) + if pipeline_name=='scoring_model': + train_mode = False + meta_train, annotations = _get_scoring_model_data(params.data_dir, meta_train, params.scoring_model__num_training_examples, seed) + else: + annotations = None + data = {'input': {'meta': meta_train, - 'target_sizes': [(300, 300)] * len(meta_train)}, - 'specs': {'train_mode': True}, + 'target_sizes': [(300, 300)] * len(meta_train), + 'annotations': annotations}, + 'specs': {'train_mode': train_mode, + 'num_threads': params.num_threads}, 'callback_input': {'meta_valid': meta_valid} } @@ -117,7 +128,8 @@ def evaluate(pipeline_name, dev_mode, chunk_size, logger, params, seed, ctx): meta_valid = meta_valid.sample(30, random_state=seed) pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG) - prediction = generate_prediction(meta_valid, pipeline, logger, CATEGORY_IDS, chunk_size) + prediction = generate_prediction(meta_valid, pipeline, logger, CATEGORY_IDS, chunk_size, params.num_threads) + prediction_filepath = os.path.join(params.experiment_dir, 'prediction.json') with open(prediction_filepath, "w") as fp: fp.write(json.dumps(prediction)) @@ -146,7 +158,7 @@ def predict(pipeline_name, dev_mode, submit_predictions, chunk_size, logger, par meta_test = meta_test.sample(2, random_state=seed) pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG) - prediction = generate_prediction(meta_test, pipeline, logger, CATEGORY_IDS, chunk_size) + prediction = generate_prediction(meta_test, pipeline, logger, CATEGORY_IDS, chunk_size, params.num_threads) submission = prediction submission_filepath = os.path.join(params.experiment_dir, 'submission.json') @@ -167,18 +179,19 @@ def make_submission(submission_filepath, logger, params): challenge.submit(submission_filepath) -def generate_prediction(meta_data, pipeline, logger, category_ids, chunk_size): +def generate_prediction(meta_data, pipeline, logger, category_ids, chunk_size, num_threads=1): if chunk_size is not None: - return _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size) + return _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size, num_threads) else: - return _generate_prediction(meta_data, pipeline, logger, category_ids) + return _generate_prediction(meta_data, pipeline, logger, category_ids, num_threads) -def _generate_prediction(meta_data, pipeline, logger, category_ids): +def _generate_prediction(meta_data, pipeline, logger, category_ids, num_threads=1): data = {'input': {'meta': meta_data, 'target_sizes': [(300, 300)] * len(meta_data), }, - 'specs': {'train_mode': False}, + 'specs': {'train_mode': False, + 'num_threads': num_threads}, 'callback_input': {'meta_valid': None} } @@ -187,17 +200,18 @@ def _generate_prediction(meta_data, pipeline, logger, category_ids): pipeline.clean_cache() y_pred = output['y_pred'] - prediction = create_annotations(meta_data, y_pred, logger, category_ids) + prediction = create_annotations(meta_data, y_pred, logger, category_ids, CATEGORY_LAYERS) return prediction -def _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size): +def _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size, num_threads=1): prediction = [] for meta_chunk in generate_data_frame_chunks(meta_data, chunk_size): data = {'input': {'meta': meta_chunk, 'target_sizes': [(300, 300)] * len(meta_chunk) }, - 'specs': {'train_mode': False}, + 'specs': {'train_mode': False, + 'num_threads': num_threads}, 'callback_input': {'meta_valid': None} } @@ -206,7 +220,22 @@ def _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, ch pipeline.clean_cache() y_pred = output['y_pred'] - prediction_chunk = create_annotations(meta_chunk, y_pred, logger, category_ids) + prediction_chunk = create_annotations(meta_chunk, y_pred, logger, category_ids, CATEGORY_LAYERS) prediction.extend(prediction_chunk) return prediction + + +def _get_scoring_model_data(data_dir, meta, num_training_examples, random_seed): + annotation_file_path = os.path.join(data_dir, 'train', "annotation.json") + coco = COCO(annotation_file_path) + meta = meta.sample(num_training_examples, random_state=random_seed) + annotations = [] + for image_id in meta['ImageId'].values: + image_annotations = {} + for category_id in CATEGORY_IDS: + annotation_ids = coco.getAnnIds(imgIds=image_id, catIds=category_id) + category_annotations = coco.loadAnns(annotation_ids) + image_annotations[category_id] = category_annotations + annotations.append(image_annotations) + return meta, annotations \ No newline at end of file diff --git a/src/pipelines.py b/src/pipelines.py index 7f3944c..af62c6d 100644 --- a/src/pipelines.py +++ b/src/pipelines.py @@ -4,7 +4,8 @@ from .steps.base import Step, Dummy from .steps.preprocessing.misc import XYSplit from .utils import squeeze_inputs, make_apply_transformer, make_apply_transformer_stream -from .models import PyTorchUNet, PyTorchUNetWeighted, PyTorchUNetStream, PyTorchUNetWeightedStream +from .models import PyTorchUNet, PyTorchUNetWeighted, PyTorchUNetStream, PyTorchUNetWeightedStream, ScoringLightGBM, \ + ScoringRandomForest from . import postprocessing as post @@ -21,8 +22,9 @@ def unet(config, train_mode): input_data=['callback_input'], input_steps=[loader], cache_dirpath=config.env.cache_dirpath, + save_output=save_output, is_trainable=True, - save_output=save_output, load_saved_output=load_saved_output) + load_saved_output=load_saved_output) mask_postprocessed = mask_postprocessing(unet, config, make_apply_transformer_, save_output=save_output) @@ -257,7 +259,7 @@ def mask_postprocessing(model, config, make_transformer, **kwargs): cache_output=not config.execution.stream_mode, **kwargs) category_mapper = Step(name='category_mapper', - transformer=make_transformer(post.categorize_image, + transformer=make_transformer(post.categorize_multilayer_image, output_name='categorized_images'), input_steps=[mask_resize], adapter={'images': ([('mask_resize', 'resized_images')]), @@ -273,7 +275,7 @@ def mask_postprocessing(model, config, make_transformer, **kwargs): cache_dirpath=config.env.cache_dirpath, **kwargs) labeler = Step(name='labeler', - transformer=make_transformer(post.label_multiclass_image, + transformer=make_transformer(post.label_multilayer_image, output_name='labeled_images'), input_steps=[mask_erosion], adapter={'images': ([(mask_erosion.name, 'eroded_images')]), @@ -301,6 +303,90 @@ def mask_postprocessing(model, config, make_transformer, **kwargs): return score_builder +def scoring_model_train(config): + save_output = False + config['execution']['stream_mode'] = True + + unet_pipeline = unet(config, train_mode=False) + + mask_dilation = unet_pipeline.get_step('mask_dilation') + mask_resize = unet_pipeline.get_step('mask_resize') + + feature_extractor = Step(name='feature_extractor', + transformer=post.FeatureExtractor(), + input_steps=[mask_dilation, mask_resize], + input_data=['input'], + adapter={'images': ([(mask_dilation.name, 'dilated_images')]), + 'probabilities': ([(mask_resize.name, 'resized_images')]), + 'annotations': ([('input', 'annotations')]), + }, + cache_dirpath=config.env.cache_dirpath, + save_output=True) + + scoring_model = Step(name='scoring_model', + transformer=ScoringLightGBM(**config.postprocessor.lightGBM) + if config.postprocessor.scoring_model == 'lgbm' else + ScoringRandomForest(**config.postprocessor.random_forest), + input_steps=[feature_extractor], + cache_dirpath=config.env.cache_dirpath, + save_output=save_output, + is_trainable=True, + force_fitting=False + ) + + return scoring_model + + +def scoring_model_inference(config, input_pipeline): + save_output = False + + mask_dilation = input_pipeline(config).get_step('mask_dilation') + mask_resize = input_pipeline(config).get_step('mask_resize') + + feature_extractor = Step(name='feature_extractor', + transformer=post.FeatureExtractor(), + input_steps=[mask_dilation, mask_resize], + input_data=['input'], + adapter={'images': ([(mask_dilation.name, 'dilated_images')]), + 'probabilities': ([(mask_resize.name, 'resized_images')]) + }, + cache_dirpath=config.env.cache_dirpath, + save_output=save_output) + + scoring_model = Step(name='scoring_model', + transformer=ScoringLightGBM(**config.postprocessor.lightGBM) + if config.postprocessor.scoring_model == 'lgbm' else + ScoringRandomForest(**config.postprocessor.random_forest), + input_steps=[feature_extractor], + cache_dirpath=config.env.cache_dirpath, + is_trainable=True, + save_output=save_output) + + score_builder = Step(name='score_builder', + transformer=post.ScoreImageJoiner(), + input_steps=[scoring_model, mask_dilation], + adapter={'images': ([(mask_dilation.name, 'dilated_images')]), + 'scores': ([(scoring_model.name, 'scores')])}, + cache_dirpath=config.env.cache_dirpath, + save_output=save_output) + + nms = Step(name='nms', + transformer=post.NonMaximumSupression(**config.postprocessor.nms), + input_steps=[score_builder], + cache_dirpath=config.env.cache_dirpath, + save_output=save_output) + + output = Step(name='output', + transformer=Dummy(), + input_steps=[nms], + adapter={'y_pred': ([(nms.name, 'images_with_scores')]), + }, + cache_dirpath=config.env.cache_dirpath, + save_output=save_output, + load_saved_output=False) + return output + + PIPELINES = {'unet': {'train': partial(unet, train_mode=True), 'inference': partial(unet, train_mode=False), }, @@ -311,5 +397,9 @@ def mask_postprocessing(model, config, make_transformer, **kwargs): }, 'unet_padded': {'inference': unet_padded, }, + 'scoring_model': {'train': scoring_model_train}, + 'unet_scoring_model': {'inference': partial(scoring_model_inference, input_pipeline=partial(unet, train_mode=False))}, + 'unet_padded_scoring_model': {'inference': partial(scoring_model_inference, input_pipeline=unet_padded)}, + 'unet_tta_scoring_model': {'inference': partial(scoring_model_inference, input_pipeline=unet_tta)}, } diff --git a/src/postprocessing.py b/src/postprocessing.py index 350b6b2..7b11698 100644 --- a/src/postprocessing.py +++ b/src/postprocessing.py @@ -1,11 +1,49 @@ +import multiprocessing as mp + import numpy as np from skimage.transform import resize from skimage.morphology import erosion, dilation, rectangle +from tqdm import tqdm from pydensecrf.densecrf import DenseCRF2D from pydensecrf.utils import unary_from_softmax +from pycocotools import mask as cocomask +import pandas as pd +import cv2 + +from .steps.base import BaseTransformer +from .utils import denormalize_img, add_dropped_objects, label, rle_from_binary +from .pipeline_config import MEAN, STD, CATEGORY_LAYERS, CATEGORY_IDS + + +class FeatureExtractor(BaseTransformer): + + def transform(self, images, probabilities, annotations=None): + if annotations is None: + annotations = [{}] * len(images) + all_features = [] + for image, im_probabilities, im_annotations in zip(images, probabilities, annotations): + all_features.append(get_features_for_image(image, im_probabilities, im_annotations)) + return {'features': all_features} + + +class ScoreImageJoiner(BaseTransformer): + def transform(self, images, scores): + images_with_scores = [] + for image, score in tqdm(zip(images, scores)): + images_with_scores.append((image, score)) + return {'images_with_scores': images_with_scores} + -from .utils import denormalize_img, add_dropped_objects, label -from .pipeline_config import MEAN, STD +class NonMaximumSupression(BaseTransformer): + def __init__(self, iou_threshold, num_threads=1): + self.iou_threshold = iou_threshold + self.num_threads = num_threads + + def transform(self, images_with_scores): + with mp.pool.ThreadPool(self.num_threads) as executor: + cleaned_images_with_scores = executor.map( + lambda p: remove_overlapping_masks(*p, iou_threshold=self.iou_threshold), images_with_scores) + return {'images_with_scores': cleaned_images_with_scores} def resize_image(image, target_size): @@ -37,6 +75,16 @@ def categorize_image(image): return np.argmax(image, axis=0) +def categorize_multilayer_image(image): + categorized_image = [] + for category_id, category_output in enumerate(image): + threshold_step = 1. / (CATEGORY_LAYERS[category_id] + 1) + thresholds = np.arange(threshold_step, 1, threshold_step) + for threshold in thresholds: + categorized_image.append(category_output > threshold) + return np.stack(categorized_image) + + def label_multiclass_image(mask): """Label separate class instances on a mask. @@ -77,6 +125,14 @@ def label_multiclass_image(mask): return labeled_image +def label_multilayer_image(mask): + labeled_channels = [] + for channel in mask: + labeled_channels.append(label(channel)) + labeled_image = np.stack(labeled_channels) + return labeled_image + + def erode_image(mask, erode_selem_size): """Erode mask. @@ -200,4 +256,131 @@ def crop_image_center_per_class(image, h_crop, w_crop): cropped_prediction = class_prediction[h_start:-h_start, w_start:-w_start] cropped_per_class_prediction.append(cropped_prediction) cropped_per_class_prediction = np.stack(cropped_per_class_prediction) - return cropped_per_class_prediction \ No newline at end of file + return cropped_per_class_prediction + + +def get_features_for_image(image, probabilities, annotations): + image_features = [] + category_layers_inds = np.cumsum(CATEGORY_LAYERS) + thresholds = get_thresholds() + for category_ind, category_instances in enumerate(image): + layer_features = [] + threshold = round(thresholds[category_ind], 2) + for mask, iou, category_probabilities in get_mask_with_iou(category_ind, category_instances, category_layers_inds, annotations, probabilities): + layer_features.append(get_features_for_mask(mask, iou, threshold, category_probabilities)) + image_features.append(pd.DataFrame(layer_features)) + return image_features + + +def get_mask_with_iou(category_ind, category_instances, category_layers_inds, annotations, probabilities): + category_nr = np.searchsorted(category_layers_inds, category_ind, side='right') + category_annotations = annotations.get(CATEGORY_IDS[category_nr], []) + iou_matrix = get_iou_matrix(category_instances, category_annotations) + category_probabilities = probabilities[category_nr] + for label_nr in range(1, category_instances.max() + 1): + mask = category_instances == label_nr + iou = get_iou(iou_matrix, label_nr) + yield mask, iou, category_probabilities + + +def get_features_for_mask(mask, iou, threshold, category_probabilities): + mask_probabilities = np.where(mask, category_probabilities, 0) + area = np.count_nonzero(mask) + mean_prob = mask_probabilities.sum() / area + max_prob = mask_probabilities.max() + bbox = get_bbox(mask) + bbox_height = bbox[1] - bbox[0] + bbox_width = bbox[3] - bbox[2] + bbox_aspect_ratio = bbox_height / bbox_width + bbox_area = bbox_width * bbox_height + bbox_fill = area / bbox_area + min_dist_to_border, max_dist_to_border = get_min_max_distance_to_border(bbox, mask.shape) + contour_length = get_contour_length(mask) + mask_features = {'iou': iou, 'threshold': threshold, 'area': area, 'mean_prob': mean_prob, + 'max_prob': max_prob, 'bbox_ar': bbox_aspect_ratio, + 'bbox_area': bbox_area, 'bbox_fill': bbox_fill, 'min_dist_to_border': min_dist_to_border, + 'max_dist_to_border': max_dist_to_border, 'contour_length': contour_length} + return mask_features + + +def get_iou_matrix(labels, annotations): + mask_anns = [] + if annotations is None or annotations == []: + return None + else: + for annotation in annotations: + if not isinstance(annotation['segmentation'], dict): + annotation['segmentation'] = \ + cocomask.frPyObjects(annotation['segmentation'], labels.shape[0], labels.shape[1])[0] + annotations = [annotation['segmentation'] for annotation in annotations] + for label_nr in range(1, labels.max() + 1): + mask = labels == label_nr + mask_ann = rle_from_binary(mask.astype('uint8')) + mask_anns.append(mask_ann) + iou_matrix = cocomask.iou(mask_anns, annotations, [0, ] * len(annotations)) + return iou_matrix + + +def get_iou(iou_matrix, label_nr): + if iou_matrix is not None: + return iou_matrix[label_nr - 1].max() + else: + return None + + +def get_thresholds(): + thresholds = [] + for n_thresholds in CATEGORY_LAYERS: + threshold_step = 1. / (n_thresholds + 1) + category_thresholds = np.arange(threshold_step, 1, threshold_step) + thresholds.extend(category_thresholds) + return thresholds + + +def get_bbox(mask): + '''taken from https://stackoverflow.com/questions/31400769/bounding-box-of-numpy-array and + modified to prevent bbox of zero area''' + rows = np.any(mask, axis=1) + cols = np.any(mask, axis=0) + rmin, rmax = np.where(rows)[0][[0, -1]] + cmin, cmax = np.where(cols)[0][[0, -1]] + return rmin, rmax + 1, cmin, cmax + 1 + + +def get_min_max_distance_to_border(bbox, im_size): + min_distance = min(bbox[0], im_size[0] - bbox[1], bbox[2], im_size[1] - bbox[3]) + max_distance = max(bbox[0], im_size[0] - bbox[1], bbox[2], im_size[1] - bbox[3]) + return min_distance, max_distance + + +def get_contour(mask): + mask_contour = np.zeros_like(mask).astype(np.uint8) + _, contours, hierarchy = cv2.findContours(mask.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) + cv2.drawContours(mask_contour, contours, -1, (255, 255, 255), 1) + return mask_contour + + +def get_contour_length(mask): + return np.count_nonzero(get_contour(mask)) + + +def remove_overlapping_masks(image, scores, iou_threshold=0.5): + scores_with_labels = [] + for layer_nr, layer_scores in enumerate(scores): + scores_with_labels.extend([(score, layer_nr, label_nr + 1) for label_nr, score in enumerate(layer_scores)]) + scores_with_labels.sort(key=lambda x: x[0], reverse=True) + for i, (score_i, layer_nr_i, label_nr_i) in enumerate(scores_with_labels): + base_mask = image[layer_nr_i] == label_nr_i + for score_j, layer_nr_j, label_nr_j in scores_with_labels[i + 1:]: + mask_to_check = image[layer_nr_j] == label_nr_j + iou = get_iou_for_mask_pair(base_mask, mask_to_check) + if iou > iou_threshold: + scores_with_labels.remove((score_j, layer_nr_j, label_nr_j)) + scores[layer_nr_j][label_nr_j - 1] = 0 + return image, scores + + +def get_iou_for_mask_pair(mask1, mask2): + intersection = np.count_nonzero(mask1 * mask2) + union = np.count_nonzero(mask1 + mask2) + return intersection / union diff --git a/src/preparation.py b/src/preparation.py index aae45da..6099960 100644 --- a/src/preparation.py +++ b/src/preparation.py @@ -17,7 +17,7 @@ logger = get_logger() -def overlay_masks(data_dir, dataset, target_dir, category_ids, erode=0, dilate=0, is_small=False, nthreads=1, +def overlay_masks(data_dir, dataset, target_dir, category_ids, erode=0, dilate=0, is_small=False, num_threads=1, border_width=0, small_annotations_size=14): if is_small: suffix = "-small" @@ -38,7 +38,7 @@ def overlay_masks(data_dir, dataset, target_dir, category_ids, erode=0, dilate=0 border_width=border_width, small_annotations_size=small_annotations_size) - process_nr = min(nthreads, len(image_ids)) + process_nr = min(num_threads, len(image_ids)) with mp.pool.ThreadPool(process_nr) as executor: executor.map(_overlay_mask_one_image, image_ids) diff --git a/src/steps/base.py b/src/steps/base.py index 9300c0d..5d701bb 100644 --- a/src/steps/base.py +++ b/src/steps/base.py @@ -325,3 +325,4 @@ def average_inputs(inputs): def exp_transform(inputs): return np.exp(inputs[0]) + diff --git a/src/steps/sklearn/models.py b/src/steps/sklearn/models.py index 62cb7e7..61bbf19 100644 --- a/src/steps/sklearn/models.py +++ b/src/steps/sklearn/models.py @@ -2,7 +2,7 @@ import numpy as np import sklearn.linear_model as lr from attrdict import AttrDict -from catboost import CatBoostClassifier +#from catboost import CatBoostClassifier from sklearn import ensemble from sklearn import svm from sklearn.externals import joblib diff --git a/src/utils.py b/src/utils.py index 4cca3aa..fe2b34a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -68,19 +68,29 @@ def decompose(labeled): return masks -def create_annotations(meta, predictions, logger, category_ids, save=False, experiment_dir='./'): - ''' - :param meta: pd.DataFrame with metadata - :param predictions: list of labeled masks or numpy array of size [n_images, im_height, im_width] - :param logger: - :param save: True, if one want to save submission, False if one want to return it - :param experiment_dir: path to save submission - :return: submission if save==False else True - ''' +def create_annotations(meta, predictions, logger, category_ids, category_layers, save=False, experiment_dir='./'): + """ + + Args: + meta: pd.DataFrame with metadata + predictions: list of labeled masks or numpy array of size [n_images, im_height, im_width] + logger: logging object + category_ids: list with ids of categories, + e.g. [None, 100] means, that no annotations will be created from category 0 data, and annotations + from category 1 will be created with category_id=100 + category_layers: + save: True, if one want to save submission, False if one want to return it + experiment_dir: directory of experiment to save annotations, relevant if save==True + + Returns: submission if save==False else True + + """ annotations = [] logger.info('Creating annotations') + category_layers_inds = np.cumsum(category_layers) for image_id, (prediction, image_scores) in zip(meta["ImageId"].values, predictions): - for category_nr, (category_instances, category_scores) in enumerate(zip(prediction, image_scores)): + for category_ind, (category_instances, category_scores) in enumerate(zip(prediction, image_scores)): + category_nr = np.searchsorted(category_layers_inds, category_ind, side='right') if category_ids[category_nr] != None: masks = decompose(category_instances) for mask_nr, (mask, score) in enumerate(zip(masks, category_scores)):