From 578fa8e47ce8897e574d2bb37e3d42d6dc6624ff Mon Sep 17 00:00:00 2001 From: Labib Asari <94868003+labeeb-7z@users.noreply.github.com> Date: Wed, 22 Feb 2023 08:03:14 +0530 Subject: [PATCH] added evaluation script for PPHumanSeg model (#130) * added evaluation script for PPHumanSeg * added quantized model, renamed dataset * minor spacing changes * moved _all variables outside loop and updated accuracy * removed printing for class accuracy and IoU * added 2 transforms * evaluation done on same size tensor as input size with mIoU 0.9085 * final changes * added mIoU and reference --- .../human_segmentation_pphumanseg/README.md | 12 ++ .../pphumanseg.py | 17 +- tools/eval/README.md | 21 ++ tools/eval/datasets/__init__.py | 4 +- tools/eval/datasets/minisupervisely.py | 202 ++++++++++++++++++ tools/eval/eval.py | 11 + 6 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 tools/eval/datasets/minisupervisely.py diff --git a/models/human_segmentation_pphumanseg/README.md b/models/human_segmentation_pphumanseg/README.md index c59f253a..018bb56f 100644 --- a/models/human_segmentation_pphumanseg/README.md +++ b/models/human_segmentation_pphumanseg/README.md @@ -22,6 +22,18 @@ python demo.py --help ![messi](./examples/messi.jpg) +--- +Results of accuracy evaluation with [tools/eval](../../tools/eval). + +| Models | Accuracy | mIoU | +| ------------------ | -------------- | ------------- | +| PPHumanSeg | 0.9581 | 0.8996 | +| PPHumanSeg quant | 0.4365 | 0.2788 | + + +\*: 'quant' stands for 'quantized'. + +--- ## License All files in this directory are licensed under [Apache 2.0 License](./LICENSE). diff --git a/models/human_segmentation_pphumanseg/pphumanseg.py b/models/human_segmentation_pphumanseg/pphumanseg.py index 2f20c7d3..2f38159e 100644 --- a/models/human_segmentation_pphumanseg/pphumanseg.py +++ b/models/human_segmentation_pphumanseg/pphumanseg.py @@ -19,6 +19,7 @@ def __init__(self, modelPath, backendId=0, targetId=0): self._inputNames = '' self._outputNames = ['save_infer_model/scale_0.tmp_1'] + self._currentInputSize = None self._inputSize = [192, 192] self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :] self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :] @@ -36,21 +37,25 @@ def setTarget(self, target_id): self._model.setPreferableTarget(self._targetId) def _preprocess(self, image): + + image = cv.cvtColor(image, cv.COLOR_BGR2RGB) + + self._currentInputSize = image.shape + image = cv.resize(image, (192, 192)) + image = image.astype(np.float32, copy=False) / 255.0 image -= self._mean image /= self._std return cv.dnn.blobFromImage(image) def infer(self, image): - assert image.shape[0] == self._inputSize[1], '{} (height of input image) != {} (preset height)'.format(image.shape[0], self._inputSize[1]) - assert image.shape[1] == self._inputSize[0], '{} (width of input image) != {} (preset width)'.format(image.shape[1], self._inputSize[0]) # Preprocess inputBlob = self._preprocess(image) # Forward self._model.setInput(inputBlob, self._inputNames) - outputBlob = self._model.forward(self._outputNames) + outputBlob = self._model.forward() # Postprocess results = self._postprocess(outputBlob) @@ -58,6 +63,10 @@ def infer(self, image): return results def _postprocess(self, outputBlob): - result = np.argmax(outputBlob[0], axis=1).astype(np.uint8) + + outputBlob = outputBlob[0] + outputBlob = cv.resize(outputBlob.transpose(1,2,0), (self._currentInputSize[1], self._currentInputSize[0]), interpolation=cv.INTER_LINEAR).transpose(2,0,1)[np.newaxis, ...] + + result = np.argmax(outputBlob, axis=1).astype(np.uint8) return result diff --git a/tools/eval/README.md b/tools/eval/README.md index 650e4d50..6c715637 100644 --- a/tools/eval/README.md +++ b/tools/eval/README.md @@ -21,6 +21,7 @@ Supported datasets: - [LFW](#lfw) - [ICDAR](#ICDAR2003) - [IIIT5K](#iiit5k) +- [Mini Supervisely](#mini_supervisely) ## ImageNet @@ -190,4 +191,24 @@ Run evaluation with the following command: ```shell python eval.py -m crnn -d iiit5k -dr /path/to/iiit5k +``` + + +## Mini Supervisely + +### Prepare data +Please download the mini_supervisely data from [here](https://paddleseg.bj.bcebos.com/humanseg/data/mini_supervisely.zip) which includes the validation dataset and unzip it. + +### Evaluation + +Run evaluation with the following command : + +```shell +python eval.py -m pphumanseg -d mini_supervisely -dr /path/to/pphumanseg +``` + +Run evaluation on quantized model with the following command : + +```shell +python eval.py -m pphumanseg_q -d mini_supervisely -dr /path/to/pphumanseg ``` \ No newline at end of file diff --git a/tools/eval/datasets/__init__.py b/tools/eval/datasets/__init__.py index 84388375..5ed59faa 100644 --- a/tools/eval/datasets/__init__.py +++ b/tools/eval/datasets/__init__.py @@ -3,6 +3,7 @@ from .lfw import LFW from .icdar import ICDAR from .iiit5k import IIIT5K +from .minisupervisely import MiniSupervisely class Registery: def __init__(self, name): @@ -20,4 +21,5 @@ def register(self, item): DATASETS.register(WIDERFace) DATASETS.register(LFW) DATASETS.register(ICDAR) -DATASETS.register(IIIT5K) \ No newline at end of file +DATASETS.register(IIIT5K) +DATASETS.register(MiniSupervisely) diff --git a/tools/eval/datasets/minisupervisely.py b/tools/eval/datasets/minisupervisely.py new file mode 100644 index 00000000..63008dab --- /dev/null +++ b/tools/eval/datasets/minisupervisely.py @@ -0,0 +1,202 @@ +import os +import cv2 as cv +import numpy as np +from tqdm import tqdm + + +class MiniSupervisely : + + ''' + Refer to https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.7/paddleseg/core/val.py + for official evaluation implementation. + ''' + + def __init__(self, root) : + self.root = root + self.val_path = os.path.join(root, 'val.txt') + self.image_set = self.load_data(self.val_path) + self.num_classes = 2 + self.miou = -1 + self.class_miou = -1 + self.acc = -1 + self.class_acc = -1 + + + @property + def name(self): + return self.__class__.__name__ + + + def load_data(self, val_path) : + """ + Load validation image set from val.txt file + Args : + val_path (str) : path to val.txt file + Returns : + image_set (list) : list of image path of input and expected image + """ + + image_set = [] + with open(val_path, 'r') as f : + for line in f.readlines() : + image_set.append(line.strip().split()) + + return image_set + + + def eval(self, model) : + """ + Evaluate model on validation set + Args : + model (object) : PP_HumanSeg model object + """ + + intersect_area_all = np.zeros([1], dtype=np.int64) + pred_area_all = np.zeros([1], dtype=np.int64) + label_area_all = np.zeros([1], dtype=np.int64) + + pbar = tqdm(self.image_set) + + pbar.set_description( + "Evaluating {} with {} val set".format(model.name, self.name)) + + for input_image, expected_image in pbar : + + input_image = cv.imread(os.path.join(self.root, input_image)).astype('float32') + + expected_image = cv.imread(os.path.join(self.root, expected_image), cv.IMREAD_GRAYSCALE)[np.newaxis, :, :] + + output_image = model.infer(input_image) + + intersect_area, pred_area, label_area = self.calculate_area( + output_image.astype('uint32'), + expected_image.astype('uint32'), + self.num_classes) + + intersect_area_all = intersect_area_all + intersect_area + pred_area_all = pred_area_all + pred_area + label_area_all = label_area_all + label_area + + self.class_iou, self.miou = self.mean_iou(intersect_area_all, pred_area_all, + label_area_all) + self.class_acc, self.acc = self.accuracy(intersect_area_all, pred_area_all) + + + def get_results(self) : + """ + Get evaluation results + Returns : + miou (float) : mean iou + class_miou (list) : iou on all classes + acc (float) : mean accuracy + class_acc (list) : accuracy on all classes + """ + return self.miou, self.class_miou, self.acc, self.class_acc + + + def print_result(self) : + """ + Print evaluation results + """ + print("Mean IoU : ", self.miou) + print("Mean Accuracy : ", self.acc) + print("Class IoU : ", self.class_iou) + print("Class Accuracy : ", self.class_acc) + + + def calculate_area(self,pred, label, num_classes, ignore_index=255): + """ + Calculate intersect, prediction and label area + Args: + pred (Tensor): The prediction by model. + label (Tensor): The ground truth of image. + num_classes (int): The unique number of target classes. + ignore_index (int): Specifies a target value that is ignored. Default: 255. + Returns: + Tensor: The intersection area of prediction and the ground on all class. + Tensor: The prediction area on all class. + Tensor: The ground truth area on all class + """ + + + if len(pred.shape) == 4: + pred = np.squeeze(pred, axis=1) + if len(label.shape) == 4: + label = np.squeeze(label, axis=1) + if not pred.shape == label.shape: + raise ValueError('Shape of `pred` and `label should be equal, ' + 'but there are {} and {}.'.format(pred.shape, + label.shape)) + + mask = label != ignore_index + pred_area = [] + label_area = [] + intersect_area = [] + + #iterate over all classes and calculate their respective areas + for i in range(num_classes): + pred_i = np.logical_and(pred == i, mask) + label_i = label == i + intersect_i = np.logical_and(pred_i, label_i) + pred_area.append(np.sum(pred_i.astype('int32'))) + label_area.append(np.sum(label_i.astype('int32'))) + intersect_area.append(np.sum(intersect_i.astype('int32'))) + + return intersect_area, pred_area, label_area + + + def mean_iou(self,intersect_area, pred_area, label_area): + """ + Calculate iou. + Args: + intersect_area (Tensor): The intersection area of prediction and ground truth on all classes. + pred_area (Tensor): The prediction area on all classes. + label_area (Tensor): The ground truth area on all classes. + Returns: + np.ndarray: iou on all classes. + float: mean iou of all classes. + """ + intersect_area = np.array(intersect_area) + pred_area = np.array(pred_area) + label_area = np.array(label_area) + + union = pred_area + label_area - intersect_area + + class_iou = [] + for i in range(len(intersect_area)): + if union[i] == 0: + iou = 0 + else: + iou = intersect_area[i] / union[i] + class_iou.append(iou) + + miou = np.mean(class_iou) + + return np.array(class_iou), miou + + + def accuracy(self,intersect_area, pred_area): + """ + Calculate accuracy + Args: + intersect_area (Tensor): The intersection area of prediction and ground truth on all classes.. + pred_area (Tensor): The prediction area on all classes. + Returns: + np.ndarray: accuracy on all classes. + float: mean accuracy. + """ + + intersect_area = np.array(intersect_area) + pred_area = np.array(pred_area) + + class_acc = [] + for i in range(len(intersect_area)): + if pred_area[i] == 0: + acc = 0 + else: + acc = intersect_area[i] / pred_area[i] + class_acc.append(acc) + + macc = np.sum(intersect_area) / np.sum(pred_area) + + return np.array(class_acc), macc diff --git a/tools/eval/eval.py b/tools/eval/eval.py index 9c25f5ed..9125045d 100644 --- a/tools/eval/eval.py +++ b/tools/eval/eval.py @@ -77,6 +77,14 @@ name="CRNN", topic="text_recognition", modelPath=os.path.join(root_dir, "models/text_recognition_crnn/text_recognition_CRNN_EN_2021sep.onnx")), + pphumanseg=dict( + name="PPHumanSeg", + topic="human_segmentation", + modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx")), + pphumanseg_q=dict( + name="PPHumanSeg", + topic="human_segmentation", + modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct-act_int8-wt_int8-quantized.onnx")), ) datasets = dict( @@ -97,6 +105,9 @@ iiit5k=dict( name="IIIT5K", topic="text_recognition"), + mini_supervisely=dict( + name="MiniSupervisely", + topic="human_segmentation"), ) def main(args):