diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bbc45348ba..52efb211e13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.6.1] - 2020-03-21 +### Changed +- VOC task export now does not use official label map by default, but takes one + from the source task to avoid primary-class and class part name + clashing ([#1275](https://github.com/opencv/cvat/issues/1275)) + +### Fixed +- File names in LabelMe format export are no longer truncated ([#1259](https://github.com/opencv/cvat/issues/1259)) +- `occluded` and `z_order` annotation attributes are now correctly passed to Datumaro ([#1271](https://github.com/opencv/cvat/pull/1271)) +- Annotation-less tasks now can be exported as empty datasets in COCO ([#1277](https://github.com/opencv/cvat/issues/1277)) +- Frame name matching for video annotations import - + allowed `frame_XXXXXX[.ext]` format ([#1274](https://github.com/opencv/cvat/pull/1274)) + +### Security +- Bump acorn from 6.3.0 to 6.4.1 in /cvat-ui ([#1270](https://github.com/opencv/cvat/pull/1270)) + ## [0.6.0] - 2020-03-15 ### Added - Server only support for projects. Extend REST API v1 (/api/v1/projects*) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0ff41aca79..40faceb7642 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Next steps should work on clear Ubuntu 18.04. - Install necessary dependencies: ```sh -$ sudo apt update && apt install -y nodejs npm curl redis-server python3-dev python3-pip python3-venv libldap2-dev libsasl2-dev +$ sudo apt-get update && sudo apt-get --no-install-recommends install -y ffmpeg build-essential nodejs npm curl redis-server python3-dev python3-pip python3-venv libldap2-dev libsasl2-dev ``` - Install [Visual Studio Code](https://code.visualstudio.com/docs/setup/linux#_debian-and-ubuntu-based-distributions) @@ -28,7 +28,7 @@ git clone https://github.com/opencv/cvat cd cvat && mkdir logs keys python3 -m venv .env . .env/bin/activate -pip install -U pip wheel +pip install -U pip wheel setuptools pip install -r cvat/requirements/development.txt pip install -r datumaro/requirements.txt python manage.py migrate diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json index 46c12b74665..022e4582b77 100644 --- a/cvat-canvas/package-lock.json +++ b/cvat-canvas/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "0.1.0", + "version": "0.5.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1203,9 +1203,9 @@ } }, "acorn": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.2.1.tgz", - "integrity": "sha512-JD0xT5FCRDNyjDda3Lrg/IxFscp9q4tiYtxE1/nOzlKCk7hIRuYjhq1kCNkbPjMRMZuFq20HNQn1I9k8Oj0E+Q==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", "dev": true }, "acorn-jsx": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 828118cfaaf..f2453993069 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1421,9 +1421,9 @@ } }, "acorn": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.3.0.tgz", - "integrity": "sha512-/czfa8BwS88b9gWQVhc8eknunSA2DoJpJyTQkhheIf5E48u1N0R4q/YxxsAeqRrmK9TQ/uYfgLDfZo91UlANIA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.1.tgz", + "integrity": "sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==", "dev": true }, "acorn-jsx": { @@ -4293,14 +4293,6 @@ "acorn": "^7.1.0", "acorn-jsx": "^5.1.0", "eslint-visitor-keys": "^1.1.0" - }, - "dependencies": { - "acorn": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", - "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==", - "dev": true - } } }, "esprima": { @@ -12354,6 +12346,12 @@ "webpack-sources": "^1.4.1" }, "dependencies": { + "acorn": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "dev": true + }, "schema-utils": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", diff --git a/cvat/__init__.py b/cvat/__init__.py index a30e5d43896..5da4433250d 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (0, 6, 0, 'final', 0) +VERSION = (0, 6, 1, 'final', 0) __version__ = get_version(VERSION) diff --git a/cvat/apps/annotation/labelme.py b/cvat/apps/annotation/labelme.py index 0128ca73922..baacb388ef0 100644 --- a/cvat/apps/annotation/labelme.py +++ b/cvat/apps/annotation/labelme.py @@ -107,17 +107,17 @@ def dump_frame_anno(frame_annotation): return ET.tostring(root_elem, encoding='unicode', pretty_print=True) def dump_as_labelme_annotation(file_object, annotations): + import os.path as osp from zipfile import ZipFile, ZIP_DEFLATED with ZipFile(file_object, 'w', compression=ZIP_DEFLATED) as output_zip: for frame_annotation in annotations.group_by_frame(): xml_data = dump_frame_anno(frame_annotation) - filename = frame_annotation.name - filename = filename[ : filename.rfind('.')] + '.xml' + filename = osp.splitext(frame_annotation.name)[0] + '.xml' output_zip.writestr(filename, xml_data) def parse_xml_annotations(xml_data, annotations, input_zip): - from cvat.apps.annotation.coco import mask_to_polygon + from datumaro.util.mask_tools import mask_to_polygons from io import BytesIO from lxml import etree as ET import numpy as np @@ -229,7 +229,7 @@ def parse_attributes(attributes_string): mask = input_zip.read(osp.join(_MASKS_DIR, mask_file)) mask = np.asarray(Image.open(BytesIO(mask)).convert('L')) mask = (mask != 0) - polygons = mask_to_polygon(mask) + polygons = mask_to_polygons(mask) for polygon in polygons: ann_items.append(annotations.LabeledShape( diff --git a/cvat/apps/annotation/pascal_voc.py b/cvat/apps/annotation/pascal_voc.py index 2dd0aa48f51..b6bcfaa33ee 100644 --- a/cvat/apps/annotation/pascal_voc.py +++ b/cvat/apps/annotation/pascal_voc.py @@ -74,7 +74,7 @@ def dump(file_object, annotations): extractor = CvatAnnotationsExtractor('', annotations) extractor = extractor.transform(id_from_image) extractor = Dataset.from_extractors(extractor) # apply lazy transforms - converter = env.make_converter('voc') + converter = env.make_converter('voc', label_map='source') with TemporaryDirectory() as temp_dir: converter(extractor, save_dir=temp_dir) make_zip_archive(temp_dir, file_object) \ No newline at end of file diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index da37a3048e6..d1d98af279a 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -91,7 +91,9 @@ def categories(self): @staticmethod def _load_categories(cvat_anno): categories = {} - label_categories = datumaro.LabelCategories() + + label_categories = datumaro.LabelCategories( + attributes=['occluded', 'z_order']) for _, label in cvat_anno.meta['task']['labels']: label_categories.add(label['name']) @@ -144,6 +146,8 @@ def convert_attrs(label, cvat_attrs): anno_group = shape_obj.group anno_label = map_label(shape_obj.label) anno_attr = convert_attrs(shape_obj.label, shape_obj.attributes) + anno_attr['occluded'] = shape_obj.occluded + anno_attr['z_order'] = shape_obj.z_order anno_points = shape_obj.points if shape_obj.type == ShapeType.POINTS: @@ -177,6 +181,8 @@ def __init__(self, url, db_task, user): def match_frame(item, cvat_task_anno): + is_video = cvat_task_anno.meta['task']['mode'] == 'interpolation' + frame_number = None if frame_number is None: try: @@ -193,6 +199,8 @@ def match_frame(item, cvat_task_anno): frame_number = int(item.id) except Exception: pass + if frame_number is None and is_video and item.id.startswith('frame_'): + frame_number = int(item.id[len('frame_'):]) if not frame_number in cvat_task_anno.frame_info: raise Exception("Could not match item id: '%s' with any task frame" % item.id) @@ -234,7 +242,7 @@ def import_dm_annotations(dm_dataset, cvat_task_anno): frame=frame_number, label=label_cat.items[ann.label].name, points=ann.points, - occluded=False, + occluded=ann.attributes.get('occluded') == True, group=group_map.get(ann.group, 0), attributes=[cvat_task_anno.Attribute(name=n, value=str(v)) for n, v in ann.attributes.items()], diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index d959f71c5b5..f3da0410623 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -2655,6 +2655,15 @@ def _get_initial_annotation(annotation_format): "points": [20.0, 0.1, 10, 3.22, 4, 7, 10, 30, 1, 2, 4.44, 5.55], "type": "polygon", "occluded": True + }, + { + "frame": 2, + "label_id": task["labels"][1]["id"], + "group": 1, + "attributes": [], + "points": [4, 7, 10, 30, 4, 5.55], + "type": "polygon", + "occluded": False }] tags_wo_attrs = [{ @@ -2711,6 +2720,12 @@ def _get_initial_annotation(annotation_format): elif annotation_format == "MOT CSV 1.0": annotations["tracks"] = rectangle_tracks_wo_attrs + elif annotation_format == "LabelMe ZIP 3.0 for images": + annotations["shapes"] = rectangle_shapes_with_attrs + \ + rectangle_shapes_wo_attrs + \ + polygon_shapes_wo_attrs + \ + polygon_shapes_with_attrs + return annotations response = self._get_annotation_formats(annotator) diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index fc0333ad0a5..de046528865 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -7,7 +7,7 @@ pylint==2.3.1 pylint-django==0.9.4 pylint-plugin-utils==0.2.6 rope==0.11 -wrapt==1.10.11 +wrapt==1.11.1 django-extensions==2.0.6 Werkzeug==0.15.3 snakeviz==0.4.2 diff --git a/datumaro/datumaro/plugins/coco_format/converter.py b/datumaro/datumaro/plugins/coco_format/converter.py index 39fe7b15402..403a6a83eb6 100644 --- a/datumaro/datumaro/plugins/coco_format/converter.py +++ b/datumaro/datumaro/plugins/coco_format/converter.py @@ -329,20 +329,24 @@ def save_categories(self, dataset): label_categories = dataset.categories().get(AnnotationType.label) if label_categories is None: return - points_categories = dataset.categories().get(AnnotationType.points) - if points_categories is None: - return - - for idx, kp_cat in points_categories.items.items(): - label_cat = label_categories.items[idx] + point_categories = dataset.categories().get(AnnotationType.points) + for idx, label_cat in enumerate(label_categories.items): cat = { 'id': 1 + idx, 'name': _cast(label_cat.name, str, ''), 'supercategory': _cast(label_cat.parent, str, ''), - 'keypoints': [str(l) for l in kp_cat.labels], - 'skeleton': [int(i) for i in kp_cat.adjacent], + 'keypoints': [], + 'skeleton': [], } + + if point_categories is not None: + kp_cat = point_categories.items.get(idx) + if kp_cat is not None: + cat.update({ + 'keypoints': [str(l) for l in kp_cat.labels], + 'skeleton': [int(i) for i in kp_cat.adjacent], + }) self.categories.append(cat) def save_annotations(self, item): @@ -447,14 +451,19 @@ class _Converter: def __init__(self, extractor, save_dir, tasks=None, save_images=False, segmentation_mode=None, crop_covered=False): - assert tasks is None or isinstance(tasks, (CocoTask, list)) + assert tasks is None or isinstance(tasks, (CocoTask, list, str)) if tasks is None: tasks = list(self._TASK_CONVERTER) elif isinstance(tasks, CocoTask): tasks = [tasks] + elif isinstance(tasks, str): + tasks = [CocoTask[tasks]] else: - for t in tasks: - assert t in CocoTask + for i, t in enumerate(tasks): + if isinstance(t, str): + tasks[i] = CocoTask[t] + else: + assert t in CocoTask, t self._tasks = tasks self._extractor = extractor @@ -546,9 +555,8 @@ def convert(self): task_conv.save_annotations(item) for task, task_conv in task_converters.items(): - if not task_conv.is_empty(): - task_conv.write(osp.join(self._ann_dir, - '%s_%s.json' % (task.name, subset_name))) + task_conv.write(osp.join(self._ann_dir, + '%s_%s.json' % (task.name, subset_name))) class CocoConverter(Converter, CliPlugin): @staticmethod diff --git a/datumaro/datumaro/plugins/yolo_format/extractor.py b/datumaro/datumaro/plugins/yolo_format/extractor.py index 7840b26c5ca..11e829d4a5b 100644 --- a/datumaro/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/datumaro/plugins/yolo_format/extractor.py @@ -90,7 +90,9 @@ def __init__(self, config_path, image_info=None): subset = YoloExtractor.Subset(subset_name, self) with open(list_path, 'r') as f: subset.items = OrderedDict( - (osp.splitext(osp.basename(p))[0], p.strip()) for p in f) + (osp.splitext(osp.basename(p.strip()))[0], p.strip()) + for p in f + ) for item_id, image_path in subset.items.items(): image_path = self._make_local_path(image_path) diff --git a/datumaro/tests/test_coco_format.py b/datumaro/tests/test_coco_format.py index 2caa03a7c09..7bf7247f59a 100644 --- a/datumaro/tests/test_coco_format.py +++ b/datumaro/tests/test_coco_format.py @@ -626,10 +626,13 @@ def __iter__(self): def categories(self): label_cat = LabelCategories() + point_cat = PointsCategories() for label in range(10): label_cat.add('label_' + str(label)) + point_cat.add(label) return { AnnotationType.label: label_cat, + AnnotationType.points: point_cat, } with TestDir() as test_dir: @@ -645,4 +648,4 @@ def __iter__(self): with TestDir() as test_dir: self._test_save_and_load(TestExtractor(), - CocoConverter(), test_dir) \ No newline at end of file + CocoConverter(tasks='image_info'), test_dir) \ No newline at end of file