From 79202cf6604c282be4c9f377f95387b75ac7a159 Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Thu, 24 Jun 2021 09:31:04 +0000 Subject: [PATCH 01/30] Stable version for lines and polygons combined --- backend/webserver/util/coco_util.py | 25 ++++++++++++++++--------- docker-compose.dev.yml | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) mode change 100644 => 100755 backend/webserver/util/coco_util.py diff --git a/backend/webserver/util/coco_util.py b/backend/webserver/util/coco_util.py old mode 100644 new mode 100755 index 5540deb3..12de05df --- a/backend/webserver/util/coco_util.py +++ b/backend/webserver/util/coco_util.py @@ -2,6 +2,9 @@ import numpy as np import shapely from shapely.geometry import LineString, Point +import logging +logger = logging.getLogger('gunicorn.error') + from database import ( fix_ids, @@ -27,9 +30,9 @@ def paperjs_to_coco(image_width, image_height, paperjs): # Compute segmentation # paperjs points are relative to the center, so we must shift them relative to the top left. - segments = [] + segments_with_area = [] + all_segments = [] center = [image_width/2, image_height/2] - if paperjs[0] == "Path": compound_path = {"children": [paperjs]} else: @@ -62,6 +65,7 @@ def paperjs_to_coco(image_width, image_height, paperjs): # len 4 means this is a line with no width; it contributes # no area to the mask, and if we include it, coco will treat # it instead as a bbox (and throw an error) + all_segments.append(segments_to_add) continue num_widths = segments_to_add.count(image_width) @@ -69,15 +73,18 @@ def paperjs_to_coco(image_width, image_height, paperjs): if num_widths + num_heights == len(segments_to_add): continue - segments.append(segments_to_add) + segments_with_area.append(segments_to_add) + all_segments.append(segments_to_add) - if len(segments) < 1: + if len(all_segments) < 1: return [], 0, [0, 0, 0, 0] - - area, bbox = get_segmentation_area_and_bbox( - segments, image_height, image_width) - - return segments, area, bbox + elif len(segments_with_area) < 1: + return all_segments, 0, [0, 0, 0, 0] + else : + area, bbox = get_segmentation_area_and_bbox( + segments_with_area, image_height, image_width) + logger.info(f"all segments: {all_segments}") + return all_segments, area, bbox def paperjs_to_coco_cliptobounds(image_width, image_height, paperjs): # todo: there's lots of edge cases to this. It needs a different solution or many many if statements :P diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7855863f..c929e39d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3" volumes: mongodb_data: external: false From 657e6ec4c1ff7e453e66ea43c799f8efbaa76c96 Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Mon, 5 Jul 2021 13:12:20 +0000 Subject: [PATCH 02/30] add an inputNumber panel for simplification degree --- backend/webserver/util/coco_util.py | 31 ++++++------------ .../annotator/panels/PolygonPanel.vue | 7 ++++ .../components/annotator/tools/BBoxTool.vue | 2 +- .../components/annotator/tools/BrushTool.vue | 2 +- .../annotator/tools/PolygonTool.vue | 32 +++++++++++++++++-- 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/backend/webserver/util/coco_util.py b/backend/webserver/util/coco_util.py index 12de05df..ff45aa73 100755 --- a/backend/webserver/util/coco_util.py +++ b/backend/webserver/util/coco_util.py @@ -2,8 +2,6 @@ import numpy as np import shapely from shapely.geometry import LineString, Point -import logging -logger = logging.getLogger('gunicorn.error') from database import ( @@ -14,11 +12,9 @@ AnnotationModel ) - def paperjs_to_coco(image_width, image_height, paperjs): """ Given a paperjs CompoundPath, converts path into coco segmentation format based on children paths - :param image_width: Width of Image :param image_height: Height of Image :param paperjs: paperjs CompoundPath in dict format @@ -30,16 +26,15 @@ def paperjs_to_coco(image_width, image_height, paperjs): # Compute segmentation # paperjs points are relative to the center, so we must shift them relative to the top left. - segments_with_area = [] - all_segments = [] + segments = [] center = [image_width/2, image_height/2] + if paperjs[0] == "Path": compound_path = {"children": [paperjs]} else: compound_path = paperjs[1] children = compound_path.get('children', []) - for child in children: child_segments = child[1].get('segments', []) @@ -48,7 +43,7 @@ def paperjs_to_coco(image_width, image_height, paperjs): for point in child_segments: # Cruve - if len(point) == 4: + if len(point) == 4 or len(point) == 3: point = point[0] # Point @@ -65,7 +60,6 @@ def paperjs_to_coco(image_width, image_height, paperjs): # len 4 means this is a line with no width; it contributes # no area to the mask, and if we include it, coco will treat # it instead as a bbox (and throw an error) - all_segments.append(segments_to_add) continue num_widths = segments_to_add.count(image_width) @@ -73,18 +67,14 @@ def paperjs_to_coco(image_width, image_height, paperjs): if num_widths + num_heights == len(segments_to_add): continue - segments_with_area.append(segments_to_add) - all_segments.append(segments_to_add) - - if len(all_segments) < 1: + segments.append(segments_to_add) + if len(segments) < 1: return [], 0, [0, 0, 0, 0] - elif len(segments_with_area) < 1: - return all_segments, 0, [0, 0, 0, 0] - else : - area, bbox = get_segmentation_area_and_bbox( - segments_with_area, image_height, image_width) - logger.info(f"all segments: {all_segments}") - return all_segments, area, bbox + + area, bbox = get_segmentation_area_and_bbox( + segments, image_height, image_width) + + return segments, area, bbox def paperjs_to_coco_cliptobounds(image_width, image_height, paperjs): # todo: there's lots of edge cases to this. It needs a different solution or many many if statements :P @@ -252,7 +242,6 @@ def get_image_coco(image_id): category_annotations = fix_ids(category_annotations) for annotation in category_annotations: - has_segmentation = len(annotation.get('segmentation', [])) > 0 has_keypoints = len(annotation.get('keypoints', [])) > 0 diff --git a/client/src/components/annotator/panels/PolygonPanel.vue b/client/src/components/annotator/panels/PolygonPanel.vue index 951d8761..47b1a3e2 100755 --- a/client/src/components/annotator/panels/PolygonPanel.vue +++ b/client/src/components/annotator/panels/PolygonPanel.vue @@ -27,6 +27,13 @@ step="2" v-model="polygon.polygon.minDistance" /> + diff --git a/client/src/components/annotator/tools/BBoxTool.vue b/client/src/components/annotator/tools/BBoxTool.vue index 116b14d2..2e00fae3 100755 --- a/client/src/components/annotator/tools/BBoxTool.vue +++ b/client/src/components/annotator/tools/BBoxTool.vue @@ -152,7 +152,7 @@ export default { this.polygon.path.fillColor = "black"; this.polygon.path.closePath(); - this.$parent.uniteCurrentAnnotation(this.polygon.path, true, true, true); + this.$parent.uniteCurrentAnnotation(this.polygon.path, false, true, true); this.polygon.path.remove(); this.polygon.path = null; diff --git a/client/src/components/annotator/tools/BrushTool.vue b/client/src/components/annotator/tools/BrushTool.vue index 5270e1c4..c122b85d 100755 --- a/client/src/components/annotator/tools/BrushTool.vue +++ b/client/src/components/annotator/tools/BrushTool.vue @@ -92,7 +92,7 @@ export default { * Unites current selection with selected annotation */ merge() { - this.$parent.uniteCurrentAnnotation(this.selection); + this.$parent.uniteCurrentAnnotation(this.selection, false); }, decreaseRadius() { if (!this.isActive) return; diff --git a/client/src/components/annotator/tools/PolygonTool.vue b/client/src/components/annotator/tools/PolygonTool.vue index 15cf7e6d..917dff99 100755 --- a/client/src/components/annotator/tools/PolygonTool.vue +++ b/client/src/components/annotator/tools/PolygonTool.vue @@ -1,5 +1,6 @@ diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index fbc2e6b8..b0de7957 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -80,7 +80,10 @@
- + @@ -245,6 +248,7 @@ import DEXTRTool from "@/components/annotator/tools/DEXTRTool"; import CopyAnnotationsButton from "@/components/annotator/tools/CopyAnnotationsButton"; import CenterButton from "@/components/annotator/tools/CenterButton"; import DownloadButton from "@/components/annotator/tools/DownloadButton"; +import DownloadBinMaskButton from "@/components/annotator/tools/DownloadBinMaskButton"; import SaveButton from "@/components/annotator/tools/SaveButton"; import SettingsButton from "@/components/annotator/tools/SettingsButton"; import ModeButton from "@/components/annotator/tools/ModeButton"; @@ -282,6 +286,7 @@ export default { BrushTool, KeypointTool, DownloadButton, + DownloadBinMaskButton, SaveButton, SettingsButton, DeleteButton, @@ -673,11 +678,9 @@ export default { if (this.currentCategory == null) return; this.currentAnnotation.subtract(compound, simplify, undoable); }, - selectLastEditorTool() { this.activeTool = localStorage.getItem("editorTool") || "Select"; }, - setCursor(newCursor) { this.cursor = newCursor; }, @@ -1026,7 +1029,12 @@ export default { }, user() { return this.$store.getters["user/user"]; - } + },currentAnnotationId() { + if (this.currentAnnotation == null) { + return -1; + } + return this.currentAnnotation.annotation.id; + }, }, sockets: { annotating(data) { From 0dba42b283d68b649217093aff87d8c6aed6dc3e Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Wed, 4 Aug 2021 12:43:52 +0000 Subject: [PATCH 05/30] 1st stable version, polygon segmentation format --- README.md | 3 +- backend/database/annotations.py | 12 ------ backend/webserver/api/annotations.py | 0 backend/webserver/api/annotator.py | 3 -- backend/webserver/api/datasets.py | 0 backend/webserver/api/images.py | 10 ++--- backend/webserver/util/coco_util.py | 5 --- backend/webserver/util/mask_rcnn.py | 0 backend/workers/tasks/data.py | 1 - client/public/worker.js | 42 +++++++++++++++++++ .../src/components/annotator/Annotation.vue | 33 ++++++++------- .../components/annotator/tools/BrushTool.vue | 2 +- .../annotator/tools/DownloadBinMaskButton.vue | 5 ++- .../components/annotator/tools/EraserTool.vue | 4 +- .../annotator/tools/PolygonTool.vue | 1 - docker-compose.dev.yml | 23 +++++----- 16 files changed, 86 insertions(+), 58 deletions(-) mode change 100644 => 100755 README.md mode change 100644 => 100755 backend/database/annotations.py mode change 100644 => 100755 backend/webserver/api/annotations.py mode change 100644 => 100755 backend/webserver/api/annotator.py mode change 100644 => 100755 backend/webserver/api/datasets.py mode change 100644 => 100755 backend/webserver/util/mask_rcnn.py mode change 100644 => 100755 backend/workers/tasks/data.py create mode 100755 client/public/worker.js mode change 100644 => 100755 docker-compose.dev.yml diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 4c43c887..6e7b8e33 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ -

+ COCO Annotator is a web-based image annotation tool designed for versatility and efficiently label images to create training data for image localization and object detection. It provides many distinct features including the ability to label an image segment (or part of a segment), track object instances, labeling objects with disconnected visible parts, efficiently storing and export annotations in the well-known [COCO format](http://cocodataset.org/#format-data). The annotation process is delivered through an intuitive and customizable interface and provides many tools for creating accurate datasets. diff --git a/backend/database/annotations.py b/backend/database/annotations.py old mode 100644 new mode 100755 index 1b9d3737..d506bd53 --- a/backend/database/annotations.py +++ b/backend/database/annotations.py @@ -1,8 +1,6 @@ import imantics as im import json - from mongoengine import * - from .datasets import DatasetModel from .categories import CategoryModel from .events import Event @@ -59,7 +57,6 @@ def __init__(self, image_id=None, **data): super(AnnotationModel, self).__init__(**data) def save(self, copy=False, *args, **kwargs): - if self.dataset_id and not copy: dataset = DatasetModel.objects(id=self.dataset_id).first() @@ -79,15 +76,6 @@ def save(self, copy=False, *args, **kwargs): def is_empty(self): return len(self.segmentation) == 0 or self.area == 0 - def mask(self): - """ Returns binary mask of annotation """ - mask = np.zeros((self.height, self.width)) - pts = [ - np.array(anno).reshape(-1, 2).round().astype(int) - for anno in self.segmentation - ] - mask = cv2.fillPoly(mask, pts, 1) - return mask def clone(self): """ Creates a clone """ diff --git a/backend/webserver/api/annotations.py b/backend/webserver/api/annotations.py old mode 100644 new mode 100755 diff --git a/backend/webserver/api/annotator.py b/backend/webserver/api/annotator.py old mode 100644 new mode 100755 index b173f3ef..7214edbf --- a/backend/webserver/api/annotator.py +++ b/backend/webserver/api/annotator.py @@ -108,7 +108,6 @@ def post(self): ) paperjs_object = annotation.get('compoundPath', []) - # Update paperjs if it exists if len(paperjs_object) == 2: @@ -126,7 +125,6 @@ def post(self): set__bbox=bbox, set__paper_object=paperjs_object, ) - if area > 0: counted = True @@ -142,7 +140,6 @@ def post(self): ) thumbnails.generate_thumbnail(image_model) - return {"success": True} diff --git a/backend/webserver/api/datasets.py b/backend/webserver/api/datasets.py old mode 100644 new mode 100755 diff --git a/backend/webserver/api/images.py b/backend/webserver/api/images.py index 93f85159..68345348 100755 --- a/backend/webserver/api/images.py +++ b/backend/webserver/api/images.py @@ -14,8 +14,6 @@ import datetime import os import io -import logging -logger = logging.getLogger('gunicorn.error') api = Namespace('image', description='Image related operations') @@ -203,7 +201,7 @@ class ImageBinaryMask(Resource): @api.expect(image_download) @login_required def get(self, image_id, annotation_id): - """ Returns binary mask by annotatio's ID """ + """ Returns binary mask by annotation's ID """ args = image_download.parse_args() as_attachment = args.get('asAttachment') @@ -215,7 +213,8 @@ def get(self, image_id, annotation_id): #image dimensions width = image.width height = image.height - + + # Get the binary mask png as attachement try: annotation = AnnotationModel.objects.get(id= annotation_id) except: @@ -223,13 +222,12 @@ def get(self, image_id, annotation_id): if (len(list(annotation.segmentation)) == 0): return {'message': 'annotation is empty'}, 400 - + bin_mask = coco_util.get_bin_mask(list(annotation.segmentation), height, width) img = Image.fromarray((255*bin_mask).astype('uint8')) image_io = io.BytesIO() img.save(image_io, "PNG", quality=95) image_io.seek(0) - return send_file(image_io, attachment_filename=image.file_name, as_attachment=as_attachment) diff --git a/backend/webserver/util/coco_util.py b/backend/webserver/util/coco_util.py index c81ab4cd..faf32003 100755 --- a/backend/webserver/util/coco_util.py +++ b/backend/webserver/util/coco_util.py @@ -1,9 +1,6 @@ import pycocotools.mask as mask import numpy as np -import shapely from shapely.geometry import LineString, Point -import logging -logger = logging.getLogger('gunicorn.error') from database import ( fix_ids, @@ -25,7 +22,6 @@ def paperjs_to_coco(image_width, image_height, paperjs): assert image_width > 0 assert image_height > 0 assert len(paperjs) == 2 - #logger.info(f"compounfPAth: {paperjs}") # Compute segmentation # paperjs points are relative to the center, so we must shift them relative to the top left. segments = [] @@ -191,7 +187,6 @@ def get_segmentation_area_and_bbox(segmentation, image_height, image_width): # Convert into rle rles = mask.frPyObjects(segmentation, image_height, image_width) rle = mask.merge(rles) - bin_mask = mask.decode(rle) return mask.area(rle), mask.toBbox(rle) def get_bin_mask(segmentation, image_height, image_width): diff --git a/backend/webserver/util/mask_rcnn.py b/backend/webserver/util/mask_rcnn.py old mode 100644 new mode 100755 diff --git a/backend/workers/tasks/data.py b/backend/workers/tasks/data.py old mode 100644 new mode 100755 index 0df11efe..9d5ab83c --- a/backend/workers/tasks/data.py +++ b/backend/workers/tasks/data.py @@ -19,7 +19,6 @@ from ..socket import create_socket from mongoengine import Q - @shared_task def export_annotations(task_id, dataset_id, categories): diff --git a/client/public/worker.js b/client/public/worker.js new file mode 100755 index 00000000..38ebb038 --- /dev/null +++ b/client/public/worker.js @@ -0,0 +1,42 @@ +//Loading a JavaScript library from a CDN +importScripts("https://cdnjs.cloudflare.com/ajax/libs/paper.js/0.12.15/paper-full.js"); + + +onmessage = function(e) { + console.log('Worker: Message received from main script'); + + // Get received data + let x = e.data[1]; + let y = e.data[2]; + let height = e.data[3]; + let width = e.data[4]; + + console.log("height", height, "width", width) + + // Create a scope to avoid the error due to 'project' is null + let scope = new paper.PaperScope(); + scope.setup(new paper.Size(width, height)); + + //recreate the paperjs object + let path = new paper.CompoundPath(); + path.importJSON(e.data[0]); + + //inintiate a binary mask full of zeros + let mask = Array.from(Array(height), () => new Array(width).fill(0)); + + //register the pixels who belong to the current polygon path + for(var i = 0; i < height; i++) { + for(var j = 0; j < width; j++) { + if (path.contains( new paper.Point(i + x, j + y))) { + mask[i][j] = 1; + } + } + } + + console.log('Worker: Posting message back to main script'); + this.postMessage(mask); + } + + onerror = event => { + console.error(event.message) + } diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index c7c6da42..57d3818a 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -517,13 +517,8 @@ export default { }); this.addUndo(action); }, - - simplifyPath(path = null) { - - //Eliminate segments' handles - //ToDo: find a way to represent Bézier curves and get their binary masks in python - path.flatten(1); - + simplifyPath(path = this.compoundPath) { + let simplify = this.simplify; if (path.hasOwnProperty('simplifyDegree')) simplify = path['simplifyDegree']; @@ -535,18 +530,24 @@ export default { } ); - //Simplify its points + // Simplify its points points = simplifyjs(points, simplify, true); - //Update current path with simplified contour + // Update current path with simplified contour path.segments = points; } - if (path.hasOwnProperty('segmentsType') && path['segmentsType'] == 'pixel'){ - //added this because it seems that the function simplifies the path - //a little bit even with tolerance = 0 ! - if (simplify != 0) path.simplify(simplify); + if (path.hasOwnProperty('segmentsType') && path['segmentsType'] == 'pixel') { + // added this because it seems that the function simplifies the path + // a little bit even with tolerance = 0 ! + if (simplify != 0) { + path.simplify(simplify); + } } + // Eliminate segments' handles + // ToDo: find a way to represent Bézier curves and get their binary masks in python + path.flatten(1); + return path; }, undoCompound() { @@ -675,6 +676,9 @@ export default { * @param {undoable} undoable add an undo action */ subtract(compound, simplify = true, undoable = true) { + + if (simplify) compound = this.simplifyPath(compound); + if (this.compoundPath == null) this.createCompoundPath(); let newCompound = this.compoundPath.subtract(compound); @@ -684,8 +688,7 @@ export default { this.compoundPath.remove(); this.compoundPath = newCompound; this.keypoints.bringToFront(); - - if (simplify) this.simplifyPath(); + }, setColor() { if (this.compoundPath == null) return; diff --git a/client/src/components/annotator/tools/BrushTool.vue b/client/src/components/annotator/tools/BrushTool.vue index 3cf85881..e84be074 100755 --- a/client/src/components/annotator/tools/BrushTool.vue +++ b/client/src/components/annotator/tools/BrushTool.vue @@ -19,7 +19,7 @@ export default { scaleFactor: 3, brush: { path: null, - simplify: 0.001, + simplify: 0, pathOptions: { strokeColor: "white", strokeWidth: 1, diff --git a/client/src/components/annotator/tools/DownloadBinMaskButton.vue b/client/src/components/annotator/tools/DownloadBinMaskButton.vue index 3a594a72..3b8b3cd8 100755 --- a/client/src/components/annotator/tools/DownloadBinMaskButton.vue +++ b/client/src/components/annotator/tools/DownloadBinMaskButton.vue @@ -22,7 +22,10 @@ export default { }, methods: { download() { - if (this.annotation_id == -1) alert("Please select an annotation !"); + if (this.annotation_id == -1) { + alert("Please select an annotation !"); + return ; + } let uri = "/api/image/binmask/" + this.image.id + "/" + this.annotation_id + "?asAttachment=true"; //download URI let link = document.createElement("a"); diff --git a/client/src/components/annotator/tools/EraserTool.vue b/client/src/components/annotator/tools/EraserTool.vue index 8be6591b..18828526 100755 --- a/client/src/components/annotator/tools/EraserTool.vue +++ b/client/src/components/annotator/tools/EraserTool.vue @@ -63,12 +63,12 @@ export default { this.erase(); }, onMouseUp() { - this.$parent.currentAnnotation.simplifyPath(); + //this.$parent.currentAnnotation.simplifyPath(); }, erase() { // Undo action, will be handled on mouse down // Simplify, will be handled on mouse up - this.$parent.currentAnnotation.subtract(this.eraser.brush, false, false); + this.$parent.currentAnnotation.subtract(this.eraser.brush, true, false); }, decreaseRadius() { if (!this.isActive) return; diff --git a/client/src/components/annotator/tools/PolygonTool.vue b/client/src/components/annotator/tools/PolygonTool.vue index aeeee5f6..7497afea 100755 --- a/client/src/components/annotator/tools/PolygonTool.vue +++ b/client/src/components/annotator/tools/PolygonTool.vue @@ -1,6 +1,5 @@ diff --git a/client/src/components/annotator/tools/BBoxTool.vue b/client/src/components/annotator/tools/BBoxTool.vue index 2e00fae3..116b14d2 100755 --- a/client/src/components/annotator/tools/BBoxTool.vue +++ b/client/src/components/annotator/tools/BBoxTool.vue @@ -152,7 +152,7 @@ export default { this.polygon.path.fillColor = "black"; this.polygon.path.closePath(); - this.$parent.uniteCurrentAnnotation(this.polygon.path, false, true, true); + this.$parent.uniteCurrentAnnotation(this.polygon.path, true, true, true); this.polygon.path.remove(); this.polygon.path = null; diff --git a/client/src/components/annotator/tools/BrushTool.vue b/client/src/components/annotator/tools/BrushTool.vue index e84be074..93624c47 100755 --- a/client/src/components/annotator/tools/BrushTool.vue +++ b/client/src/components/annotator/tools/BrushTool.vue @@ -3,7 +3,7 @@ import paper from "paper"; import tool from "@/mixins/toolBar/tool"; export default { - name: "EraserTool", + name: "BrushTool", mixins: [tool], props: { scale: { @@ -19,7 +19,7 @@ export default { scaleFactor: 3, brush: { path: null, - simplify: 0, + simplify: 1, pathOptions: { strokeColor: "white", strokeWidth: 1, diff --git a/client/src/components/annotator/tools/EraserTool.vue b/client/src/components/annotator/tools/EraserTool.vue index 18828526..e16ef175 100755 --- a/client/src/components/annotator/tools/EraserTool.vue +++ b/client/src/components/annotator/tools/EraserTool.vue @@ -25,7 +25,8 @@ export default { strokeWidth: 1, radius: 30 } - } + }, + selection: null }; }, methods: { @@ -35,15 +36,19 @@ export default { this.eraser.brush = null; } }, + removeSelection() { + if (this.selection != null) { + this.selection.remove(); + this.selection = null; + } + }, moveBrush(point) { if (this.eraser.brush == null) this.createBrush(); - this.eraser.brush.bringToFront(); this.eraser.brush.position = point; }, createBrush(center) { center = center || new paper.Point(0, 0); - this.eraser.brush = new paper.Path.Circle({ strokeColor: this.eraser.pathOptions.strokeColor, strokeWidth: this.eraser.pathOptions.strokeWidth, @@ -51,6 +56,12 @@ export default { center: center }); }, + createSelection() { + this.selection = new paper.Path({ + strokeColor: this.eraser.pathOptions.strokeColor, + strokeWidth: this.eraser.pathOptions.strokeWidth + }); + }, onMouseMove(event) { this.moveBrush(event.point); }, @@ -59,16 +70,20 @@ export default { this.erase(); }, onMouseDown() { - this.$parent.currentAnnotation.createUndoAction("Subtract"); + this.createSelection(); this.erase(); }, onMouseUp() { - //this.$parent.currentAnnotation.simplifyPath(); + this.$parent.currentAnnotation.createUndoAction("Subtract"); + this.$parent.currentAnnotation.subtract(this.selection, true, false); + this.removeSelection(); }, erase() { - // Undo action, will be handled on mouse down - // Simplify, will be handled on mouse up - this.$parent.currentAnnotation.subtract(this.eraser.brush, true, false); + // Undo action, will be handled on mouse up + let newSelection = this.selection.unite(this.eraser.brush); + + this.selection.remove(); + this.selection = newSelection; }, decreaseRadius() { if (!this.isActive) return; @@ -99,21 +114,18 @@ export default { watch: { "eraser.pathOptions.radius"() { if (this.eraser.brush == null) return; - let position = this.eraser.brush.position; this.eraser.brush.remove(); this.createBrush(position); }, "eraser.pathOptions.strokeColor"(newColor) { if (this.eraser.brush == null) return; - this.eraser.brush.strokeColor = newColor; }, isActive(active) { if (this.eraser.brush != null) { this.eraser.brush.visible = active; } - if (active) { this.tool.activate(); localStorage.setItem("editorTool", this.name); @@ -130,4 +142,4 @@ export default { }, mounted() {} }; - + \ No newline at end of file diff --git a/client/src/components/annotator/tools/PolygonTool.vue b/client/src/components/annotator/tools/PolygonTool.vue index 7497afea..0f493e55 100755 --- a/client/src/components/annotator/tools/PolygonTool.vue +++ b/client/src/components/annotator/tools/PolygonTool.vue @@ -211,7 +211,7 @@ export default { } this.removeUndos(this.actionTypes.ADD_POINTS); - this.$parent.save(); + //this.$parent.save(); return true; }, removeLastPoint() { From 31230232354333e143ff94a095f34856e076b9e9 Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Fri, 20 Aug 2021 08:16:31 +0000 Subject: [PATCH 10/30] Use pixelMode --- backend/webserver/api/images.py | 90 +++++++++++++++---- .../src/components/annotator/Annotation.vue | 67 +++++++------- 2 files changed, 112 insertions(+), 45 deletions(-) diff --git a/backend/webserver/api/images.py b/backend/webserver/api/images.py index 6ada0b4d..1d5176e0 100755 --- a/backend/webserver/api/images.py +++ b/backend/webserver/api/images.py @@ -5,15 +5,19 @@ import pycocotools.mask as mask from ..util import query_util, coco_util from database import ( + fix_ids, ImageModel, + CategoryModel, DatasetModel, AnnotationModel ) - +import numpy as np from PIL import Image import datetime import os import io +import logging +logger = logging.getLogger('gunicorn.error') api = Namespace('image', description='Image related operations') @@ -206,16 +210,76 @@ def get(self, image_id, annotation_id): args = image_download.parse_args() as_attachment = args.get('asAttachment') - image = current_user.images.filter(id=image_id, deleted=False).first() + image_0 = current_user.images.filter(id=image_id, deleted=False).first() - if image is None: + if image_0 is None: return {'success': False}, 400 #image dimensions - width = image.width - height = image.height - + width = image_0.width + height = image_0.height + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + image = ImageModel.objects(id=image_id)\ + .only(*ImageModel.COCO_PROPERTIES) + + image = fix_ids(image)[0] + dataset = DatasetModel.objects(id=image.get('dataset_id')).first() + + bulk_categories = CategoryModel.objects(id__in=dataset.categories, deleted=False) \ + .only(*CategoryModel.COCO_PROPERTIES) + + db_annotations = AnnotationModel.objects(deleted=False, image_id=image_id) + final_image_array = np.zeros((height, width)) + category_count = 1 + label_colors = [(0, 0, 0)] + for category in fix_ids(bulk_categories): + label_colors.append( tuple(np.random.random(size=3) * 255)) + category_annotations = db_annotations\ + .filter(category_id=category.get('id'))\ + .only(*AnnotationModel.COCO_PROPERTIES) + + if category_annotations.count() == 0: + continue + + category_annotations = fix_ids(category_annotations) + logger.info(f"category name {category.get('name')}") + logger.info(f"length annotations {len(category_annotations)}") + for annotation in category_annotations: + + has_segmentation = len(annotation.get('segmentation', [])) > 0 + has_rle_segmentation = annotation.get('rle', {}) != {} + + if has_rle_segmentation: + # Convert uncompressed RLE to encoded RLE mask + rles = mask.frPyObjects(dict(annotation.get('rle', {})), height, width) + rle = mask.merge([rles]) + # Extract the binary mask + bin_mask = mask.decode(rle) + idx = bin_mask == 1 + final_image_array[idx] = category_count + elif has_segmentation: + bin_mask = coco_util.get_bin_mask(list(annotation.get('segmentation')), height, width) + idx = bin_mask == 1 + final_image_array[idx] = category_count + category_count += 1 + #label_colors = np.array([(0, 0, 0), (128, 0, 0), (0, 128, 0), (0, 64, 128)]) + logger.info(f"label colors {label_colors}") + r = np.zeros_like(final_image_array).astype(np.uint8) + g = np.zeros_like(final_image_array).astype(np.uint8) + b = np.zeros_like(final_image_array).astype(np.uint8) + for l in range(0, category_count): + idx = final_image_array == l + x, y, z = label_colors[l] + r[idx] = x + g[idx] = y + b[idx] = z + + rgb = np.stack([r, g, b], axis=2) + + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # Get the binary mask png as attachement + ''' try: annotation = AnnotationModel.objects.get(id= annotation_id) except: @@ -223,14 +287,9 @@ def get(self, image_id, annotation_id): if (len(list(annotation.segmentation)) == 0 and not dict(annotation.rle)): return {'message': 'annotation is empty'}, 400 - - # Convert uncompressed RLE to encoded RLE mask - '''segmentation = { - "counts": list(annotation.binaryMask), - "size": [height, width] - } - ''' + if dict(annotation.rle) : + # Convert uncompressed RLE to encoded RLE mask rles = mask.frPyObjects(dict(annotation.rle), height, width) rle = mask.merge([rles]) # Extract the binary mask @@ -239,8 +298,9 @@ def get(self, image_id, annotation_id): bin_mask = coco_util.get_bin_mask(list(annotation.segmentation), height, width) img = Image.fromarray((255*bin_mask).astype('uint8')) + ''' + img = Image.fromarray(rgb.astype('uint8'), 'RGB') image_io = io.BytesIO() img.save(image_io, "PNG", quality=95) image_io.seek(0) - return send_file(image_io, attachment_filename=image.file_name, as_attachment=as_attachment) - + return send_file(image_io, attachment_filename=image_0.file_name, as_attachment=as_attachment) \ No newline at end of file diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 585c4c1f..611985ab 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -312,6 +312,7 @@ export default { showKeypoints: false, color: this.annotation.color, compoundPath: null, + pixelMode: false, binaryMask: [], keypoints: null, metadata: [], @@ -352,12 +353,12 @@ export default { this.compoundPath.remove(); this.compoundPath = null; } - this.initBinaryMask(); this.createCompoundPath( this.annotation.paper_object, this.annotation.segmentation ); + if(JSON.stringify(this.annotation.rle) != "{}") this.pixelMode = true; }, initBinaryMask() { this.binaryMask = Array.from(Array(this.annotation.height), () => @@ -539,7 +540,7 @@ export default { let simplify = this.simplify; if (path.hasOwnProperty("simplifyDegree")) simplify = path["simplifyDegree"]; - + //polygon format if ( path.hasOwnProperty("segmentsType") && path["segmentsType"] == "polygon" @@ -556,21 +557,22 @@ export default { // Update current path with simplified contour path.segments = points; } - - if ( + // rle format + else if ( path.hasOwnProperty("segmentsType") && path["segmentsType"] == "pixel" ) { + this.pixelMode = true; // added this because it seems that the function simplifies the path // a little bit even with tolerance = 0 ! if (simplify != 0) { path.simplify(simplify); } - //path.smooth(); + } + // Eraser: flatten its path only if we are in polygon format + else if (! this.pixelMode) { + path.flatten(1); } - // Eliminate segments' handles - // ToDo: find a way to represent Bézier curves and get their binary masks in python - //path.flatten(1); return path; }, @@ -664,22 +666,6 @@ export default { deleteKeypoint(keypoint) { this.keypoints.deleteKeypoint(keypoint); }, - updateBinaryMask(subMask, x, y, height, width, eraser = false) { - x = Math.round(x + this.annotation.width / 2); - y = Math.round(y + this.annotation.height / 2); - - for (let i = 0; i < width; i++) { - for (let j = 0; j < height; j++) { - if (!eraser) { - this.binaryMask[y + j][x + i] = - this.binaryMask[y + j][x + i] || subMask[j][i]; - } else if (this.binaryMask[y + j][x + i] && subMask[j][i]) { - this.binaryMask[y + j][x + i] = 0; - } - } - } - console.log("finished updating bin mask"); - }, /** * Extract current annotation's children who are points or lines */ @@ -705,6 +691,7 @@ export default { * @param {isBBox} isBBox mark annotation as bbox. */ unite(compound, simplify = true, undoable = true, isBBox = false) { + console.log("pixelMode from unite", this.pixelMode) if (this.compoundPath == null) this.createCompoundPath(); if (undoable) this.createUndoAction("Unite"); @@ -769,25 +756,36 @@ export default { this.compoundPath.remove(); this.compoundPath = newCompound; + this.compoundPath.data.annotationId = this.index; + this.compoundPath.data.categoryId = this.categoryIndex; this.keypoints.bringToFront(); }, getRLE() { + + this.initBinaryMask(); + let path, height, width, x, y, x_0, y_0; + let eraser = false; let children = this.compoundPath.children; + for (let index in children) { + path = children[index]; + eraser = (path.fillColor == null) ; height = path.bounds.height; width = path.bounds.width; x = path.bounds.x; y = path.bounds.y; x_0 = Math.round(x + this.annotation.width / 2); y_0 = Math.round(y + this.annotation.height / 2); - + // Register the pixels who belong to the current compoundPath for (var i = 0; i < height; i++) { for (var j = 0; j < width; j++) { if (path.contains(new paper.Point(x + j, y + i))) { - this.binaryMask[i + y_0][j + x_0] = 1; + + if (eraser) this.binaryMask[i + y_0][j + x_0] = 0; + else this.binaryMask[i + y_0][j + x_0] = 1; } } } @@ -878,10 +876,19 @@ export default { //export binary mask //TODO: export rle only if brushTool is used in this annotation - annotationData.rle = { - size: [this.annotation.height, this.annotation.width], - counts: this.getRLE(), - }; + if (this.pixelMode) { + annotationData.rle = { + size: [this.annotation.height, this.annotation.width], + counts: this.getRLE(), + }; + annotationData.area = Math.round(this.compoundPath.area); + annotationData.bbox = [ + Math.round(this.compoundPath.bounds.x + this.annotation.width / 2), + Math.round(this.compoundPath.bounds.y + this.annotation.height / 2), + Math.round(this.compoundPath.bounds.height), + Math.round(this.compoundPath.bounds.width), + ]; + } return annotationData; }, emitModify() { From 7e2714491e563e4e1f4e27aa4af4c61989fa8036 Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Tue, 31 Aug 2021 13:33:20 +0000 Subject: [PATCH 11/30] Add a task for Generating a dataset's semantic segmentation --- backend/workers/tasks/__init__.py | 1 + backend/workers/tasks/data.py | 13 +- .../workers/tasks/semantic_segmentation.py | 137 ++++++++++++++++++ 3 files changed, 147 insertions(+), 4 deletions(-) mode change 100644 => 100755 backend/workers/tasks/__init__.py create mode 100755 backend/workers/tasks/semantic_segmentation.py diff --git a/backend/workers/tasks/__init__.py b/backend/workers/tasks/__init__.py old mode 100644 new mode 100755 index b00d7307..2660ac22 --- a/backend/workers/tasks/__init__.py +++ b/backend/workers/tasks/__init__.py @@ -1,5 +1,6 @@ from .data import * +from .semantic_segmentation import * from .test import * from .scan import * from .thumbnails import * \ No newline at end of file diff --git a/backend/workers/tasks/data.py b/backend/workers/tasks/data.py index 9d5ab83c..89cac73e 100755 --- a/backend/workers/tasks/data.py +++ b/backend/workers/tasks/data.py @@ -21,7 +21,6 @@ @shared_task def export_annotations(task_id, dataset_id, categories): - task = TaskModel.objects.get(id=task_id) dataset = DatasetModel.objects.get(id=dataset_id) @@ -51,6 +50,7 @@ def export_annotations(task_id, dataset_id, categories): # iterate though all categoires and upsert category_names = [] + for category in fix_ids(db_categories): if len(category.get('keypoint_labels', [])) > 0: @@ -86,12 +86,12 @@ def export_annotations(task_id, dataset_id, categories): num_annotations = 0 for annotation in annotations: - + has_keypoints = len(annotation.get('keypoints', [])) > 0 has_segmentation = len(annotation.get('segmentation', [])) > 0 + has_rle_segmentation = annotation.get('rle', {}) != {} - if has_keypoints or has_segmentation: - + if has_keypoints or has_segmentation or has_rle_segmentation: if not has_keypoints: if 'keypoints' in annotation: del annotation['keypoints'] @@ -100,8 +100,13 @@ def export_annotations(task_id, dataset_id, categories): arr = arr[2::3] annotation['num_keypoints'] = len(arr[arr > 0]) + if has_rle_segmentation: + annotation['segmentation'] = annotation.get('rle') + num_annotations += 1 + annotation.pop('rle') coco.get('annotations').append(annotation) + task.info( f"Exporting {num_annotations} annotations for image {image.get('id')}") diff --git a/backend/workers/tasks/semantic_segmentation.py b/backend/workers/tasks/semantic_segmentation.py new file mode 100755 index 00000000..917203da --- /dev/null +++ b/backend/workers/tasks/semantic_segmentation.py @@ -0,0 +1,137 @@ +from database import ( + fix_ids, + ImageModel, + CategoryModel, + AnnotationModel, + DatasetModel, + TaskModel, + ExportModel +) +import io +import os +import time +import numpy as np +import pycocotools.mask as mask + +import zipfile +from PIL import Image +from celery import shared_task +from ..socket import create_socket + +@shared_task +def export_semantic_segmentation(task_id, dataset_id, categories): + # Initiate a task and its socket + task = TaskModel.objects.get(id=task_id) + task.info(f"Beginning Export Semantic Segmentation") + task.update(status="PROGRESS") + socket = create_socket() + + # Get the needed items from the database + dataset = DatasetModel.objects.get(id=dataset_id) + db_categories = CategoryModel.objects(id__in=categories, deleted=False) \ + .only(*CategoryModel.COCO_PROPERTIES) + db_images = ImageModel.objects( + deleted=False, dataset_id=dataset.id).only( + *ImageModel.COCO_PROPERTIES) + db_annotations = AnnotationModel.objects( + deleted=False, category_id__in=categories) + + # Iterate through all categories to pick a color for each one + category_names = [] + label_colors = [(0, 0, 0)] + for category in fix_ids(db_categories): + category_names.append(category.get('name')) + label_colors.append( tuple(np.random.random(size=3) * 255)) + + # Get the path + # Generate a unique name for the zip + timestamp = time.time() + directory = f"{dataset.directory}.exports/" + zip_path = f"{directory}SemanticSeg-{timestamp}.zip" + + if not os.path.exists(directory): + os.makedirs(directory) + + # Initiate progress counter + progress = 0 + total_images = len(db_images) + + with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED) as zip_file: + # Iterate through each image and + # save its corresponding semantic segmentation + for image in db_images: + image = fix_ids(image) + width = image.get('width') + height = image.get('height') + + img_annotations = db_annotations.filter(image_id=image.get('id'))\ + .only(*AnnotationModel.COCO_PROPERTIES) + + final_image_array = np.zeros((height, width)) + category_index = 1 + found_categories = [] + + for category in fix_ids(db_categories): + category_annotations = img_annotations\ + .filter(category_id=category.get('id'))\ + .only(*AnnotationModel.COCO_PROPERTIES) + + if category_annotations.count() == 0: + category_index += 1 + + continue + found_categories.append(category_index) + category_annotations = fix_ids(category_annotations) + + for annotation in category_annotations: + has_segmentation = len(annotation.get('segmentation', [])) > 0 + has_rle_segmentation = annotation.get('rle', {}) != {} + + if has_rle_segmentation: + # Convert uncompressed RLE to encoded RLE mask + rles = mask.frPyObjects(dict(annotation.get('rle', {})), height, width) + rle = mask.merge([rles]) + # Extract the binary mask + bin_mask = mask.decode(rle) + idx = bin_mask == 1 + final_image_array[idx] = category_index + elif has_segmentation: + # Convert into rle + rles = mask.frPyObjects(list(annotation.get('segmentation')), height, width) + rle = mask.merge(rles) + # Extract the binary mask + bin_mask = mask.decode(rle) + idx = bin_mask == 1 + final_image_array[idx] = category_index + category_index += 1 + # Generate a RGB image to be saved in the zip + r = np.zeros_like(final_image_array).astype(np.uint8) + g = np.zeros_like(final_image_array).astype(np.uint8) + b = np.zeros_like(final_image_array).astype(np.uint8) + for l in found_categories: + idx = final_image_array == l + x, y, z = label_colors[l] + r[idx] = x + g[idx] = y + b[idx] = z + rgb = np.stack([r, g, b], axis=2) + image_io = io.BytesIO() + Image.fromarray(rgb.astype('uint8')).save(image_io, "PNG", quality=95) + + # Write the image to the zip + task.info(f"Writing image {image.get('id')} to the zipfile") + zip_file.writestr(image.get('file_name'), image_io.getvalue()) + + # Update progress + progress+=1 + task.set_progress((progress / total_images) * 100, socket=socket) + + zip_file.close() + + task.info("Finished Generating Image segmentation... Sending the zipfile") + + export = ExportModel(dataset_id=dataset.id, path=zip_path, + tags=["SemanticSeg", *category_names]) + export.save() + +__all__ = ["export_semantic_segmentation"] From 13516351ab56d5d9eb48906d9f2842cf5587f6ba Mon Sep 17 00:00:00 2001 From: chaimaa-louahabi Date: Tue, 31 Aug 2021 13:57:47 +0000 Subject: [PATCH 12/30] Add export semantic segmentation button and its handlers --- client/src/views/Dataset.vue | 469 ++++++++++++++++++++++++----------- 1 file changed, 323 insertions(+), 146 deletions(-) diff --git a/client/src/views/Dataset.vue b/client/src/views/Dataset.vue index e4d11db7..e0c2c8e7 100755 --- a/client/src/views/Dataset.vue +++ b/client/src/views/Dataset.vue @@ -1,32 +1,54 @@