Fixed track interpolation and frame extraction for avi videos

Author: JunweiLiang

Diff for: diva_io/LICENSE

@@ -0,0 +1,131 @@
+# DIVA IO Package
+
+Version 0.2
+
+Author: Lijun Yu
+
+Email: lijun@lj-y.com
+
+## Version History
+
+* 0.2
+  * **Real** random access in video loader.
+  * Add annotation converter.
+  * Warning control option.
+* 0.1
+  * Initial release of video loader.
+
+## Installation
+
+### Integration
+
+To use as a submodule in your git project, run
+
+```sh
+git submodule add https://github.com/Lijun-Yu/diva-io.git diva_io
+```
+
+### Requirements
+
+Environment requirements are listed in `environment.yml`.
+For the `av` package, I recommend you install it via `conda` by
+
+```sh
+conda install av -c conda-forge
+```
+
+as building from `pip` would require a lot of dependencies.
+
+## Video Loader
+
+A robust video loader that deals with missing frames in the [MEVA dataset](http://mevadata.org).
+
+This video loader is developed based on [`PyAV`](https://github.com/mikeboers/PyAV) package.
+The [`pims`](https://github.com/soft-matter/pims) package was also a good reference despite its compatibility issue with current `PyAV`.
+
+For the videos in the MEVA, using `cv2.VideoCapture` would result in wrong frame ids as it never counts the missing frames.
+If you are using MEVA, I suggest you change to this video loader ASAP.
+
+### Replace `cv2.VideoCapture`
+
+According to my test, this video loader returns the exact same frame as `cv2.VideoCapture` unless missing frame or decoding error occured.
+To replace the `cv2.VideoCapture` objects in legacy codes, simply change from
+
+```python
+import cv2
+cap = cv2.VideoCapture(video_path)
+```
+
+to
+
+```python
+from diva_io.video import VideoReader
+cap = VideoReader(video_path)
+```
+
+`VideoReader.read` follows the schema of `cv2.VideoCapture.read` but automatically inserts the missing frames while reading the video.
+
+### Iterator Interface
+
+```python
+video = VideoReader(video_path)
+for frame in video:
+    # frame is a diva_io.video.frame.Frame object
+    image = frame.numpy()
+    # image is an uint8 array in a shape of (height, width, channel[BGR])
+    # ... Do something with the image
+```
+
+### Random Access
+
+Random access of a frame requires decoding from the nearest key frame (approximately every 60 frames for MEVA).
+Averagely, this introduces a constant overhead of 0.1 seconds, which is much faster than iterating from the beginning.
+
+```python
+start_frame_id = 1500
+length = 100
+video.seek(start_frame_id)
+for frame in video.get_iter(length):
+    image = frame.numpy()
+    # ... Do something with the image
+```
+
+### Video Properties
+
+```python
+video.width # cap.get(cv2.CAP_PROP_FRAME_WIDTH)
+video.height # cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
+video.fps # cap.get(cv2.CAP_PROP_FPS)
+```
+
+### Other Interfaces
+
+For other usages, please see the comments in `video/reader.py`.
+
+## Annotation
+
+An annotation loader and converter for Kitware YML format in [meva-data-repo](https://gitlab.kitware.com/meva/meva-data-repo).
+
+Clone the meva-data-repo and set
+
+```python
+annotation_dir = 'path/to/meva-data-repo/annotation/DIVA-phase-2/MEVA/meva-annotations'
+```
+
+### Convert Annotation
+
+This is to convert the annotation from Kitware YML format to ActEV Scorer JSON format.
+Run the following command in shell outside the repo's director,
+
+```sh
+python -m diva_io.annotation.converter <annotation_dir> <output_dir>
+```
+
+### Read Annotation
+
+```python
+from diva_io.annotation import KitwareAnnotation
+video_name = '2018-03-11.11-15-04.11-20-04.school.G300'
+annotation = KitwareAnnotation(video_name, annotation_dir)
+# deal with annotation.raw_data
+```

Diff for: diva_io/README.md

@@ -0,0 +1 @@
+__author__ = 'Lijun Yu'

Diff for: diva_io/__init__.py

@@ -0,0 +1 @@
+from .kf1 import KitwareAnnotation

Diff for: diva_io/annotation/__init__.py

@@ -0,0 +1,65 @@
+import os
+import json
+import argparse
+import os.path as osp
+from progressbar import progressbar
+from concurrent.futures import ProcessPoolExecutor
+from ..utils import get_logger
+from .kf1 import KitwareAnnotation
+
+
+def _get_video_list(annotation_dir):
+    path = osp.join(annotation_dir, 'list-of-annotated-meva-clips.txt')
+    with open(path) as f:
+        video_list = [l.strip() for l in f][2:]
+    return video_list
+
+
+def _worker(job):
+    video_name, annotation_dir = job
+    annotation = KitwareAnnotation(video_name, annotation_dir)
+    return annotation.get_activities_official()
+
+
+def _get_official_format(video_list, annotation_dir):
+    jobs = [(video_name, annotation_dir) for video_name in video_list]
+    pool = ProcessPoolExecutor()
+    activities = []
+    for result in progressbar(pool.map(_worker, jobs)):
+        activities.extend(result)
+    reference = {'filesProcessed': video_list, 'activities': activities}
+    file_index = {video_name: {'framerate': 30.0, 'selected': {0: 1, 9000: 0}}
+                  for video_name in video_list}
+    return reference, file_index
+
+
+def _write_files(data_dict, output_dir):
+    os.makedirs(output_dir, exist_ok=True)
+    logger = get_logger(__name__)
+    for filename, data in data_dict.items():
+        path = osp.join(output_dir, filename + '.json')
+        if osp.exists(path):
+            logger.warning('Overwriting file %s', path)
+        with open(path, 'w') as f:
+            json.dump(data, f)
+
+
+def convert_annotation(annotation_dir, output_dir):
+    video_list = _get_video_list(annotation_dir)
+    reference, file_index = _get_official_format(video_list, annotation_dir)
+    data_dict = {'reference': reference, 'file-index': file_index}
+    _write_files(data_dict, output_dir)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        'Annotation Converter for KF1, from Kitware YML format to '
+        'ActEV Scorer JSON format.')
+    parser.add_argument('annotation_dir')
+    parser.add_argument('output_dir')
+    args = parser.parse_args()
+    convert_annotation(args.annotation_dir, args.output_dir)
+
+
+if __name__ == "__main__":
+    main()

Diff for: diva_io/annotation/converter.py

@@ -0,0 +1,97 @@
+import yaml
+import os.path as osp
+from collections import defaultdict
+
+
+FIELDS = ['activities', 'geom', 'types']
+
+
+class KitwareAnnotation(object):
+
+    def __init__(self, video_name: str, annotation_dir: str):
+        # Please explore the structure of raw_data yourself
+        self.video_name = video_name
+        self.raw_data = self._load_raw_data(video_name, annotation_dir)
+
+    def _split_meta(self, contents, key):
+        meta = []
+        i = 0
+        while i < len(contents) and 'meta' in contents[i]:
+            assert key not in contents[i]
+            meta.append(contents[i]['meta'])
+            i += 1
+        data = [content[key] for content in contents[i:]]
+        return meta, data
+
+    def _load_file(self, video_name, annotation_dir, field):
+        date, time_1, time_2 = video_name.split('.')[:3]
+        for time in [time_1, time_2]:
+            path = osp.join(annotation_dir, date, time[:2], '%s.%s.yml' % (
+                video_name, field))
+            if not osp.exists(path):
+                continue
+            with open(path) as f:
+                contents = yaml.load(f, Loader=yaml.FullLoader)
+            return contents
+        path = osp.join(annotation_dir, date, time_1[:2], '%s.%s.yml' % (
+            video_name, field))
+        raise FileNotFoundError(path)
+
+    def _load_raw_data(self, video_name, annotation_dir):
+        raw_data = {'meta': {}}
+        for field in FIELDS:
+            contents = self._load_file(video_name, annotation_dir, field)
+            key = field if field != 'activities' else 'act'
+            raw_data['meta'][field], raw_data[field] = self._split_meta(
+                contents, key)
+        objs = defaultdict(dict)
+        for obj in raw_data['geom']:
+            obj['g0'] = [int(x) for x in obj['g0'].split()]
+            objs[obj['id1']][obj['ts0']] = obj
+        for obj in raw_data['types']:
+            objs[obj['id1']]['type'] = [*obj['cset3'].keys()][0]
+        for act in raw_data['activities']:
+            for actor in act.get('actors', []):
+                obj = objs[actor['id1']]
+                geoms = []
+                for ts in actor['timespan']:
+                    start, end = ts['tsr0']
+                    for time in range(start, end + 1):
+                        geoms.append(obj[time])
+                actor['geoms'] = geoms
+                actor['type'] = obj['type']
+        return raw_data
+
+    def get_activities_official(self):
+        activities = []
+        for act in self.raw_data['activities']:
+            act_id = act['id2']
+            act_type = [*act['act2'].keys()][0]
+            if act_type.startswith('empty'):
+                continue
+            start, end = act['timespan'][0]['tsr0']
+            objects = []
+            for actor in act['actors']:
+                actor_id = actor['id1']
+                bbox_history = {}
+                for geom in actor['geoms']:
+                    frame_id = geom['ts0']
+                    x1, y1, x2, y2 = geom['g0']
+                    bbox_history[frame_id] = {
+                        'presenceConf': 1,
+                        'boundingBox': {
+                            'x': min(x1, x2), 'y': min(y1, y2),
+                            'w': abs(x2 - x1), 'h': abs(y2 - y1)}}
+                for frame_id in range(start, end + 1):
+                    if frame_id not in bbox_history:
+                        bbox_history[frame_id] = {}
+                obj = {'objectType': 'Vehicle', 'objectID': actor_id,
+                       'localization': {self.video_name: bbox_history}}
+                objects.append(obj)
+            activity = {
+                'activity': act_type, 'activityID': act_id,
+                'presenceConf': 1, 'alertFrame': start,
+                'localization': {self.video_name: {start: 1, end + 1: 0}},
+                'objects': objects}
+            activities.append(activity)
+        return activities

Diff for: diva_io/annotation/kf1.py

@@ -0,0 +1,8 @@
+name: diva_io
+channels:
+  - pkgs/main
+  - conda-forge
+dependencies:
+  - python
+  - numpy
+  - av

Diff for: diva_io/environment.yml

@@ -0,0 +1 @@
+from .log import get_logger

Diff for: diva_io/utils/__init__.py

@@ -0,0 +1,16 @@
+import logging
+
+
+def get_logger(name, level=logging.INFO, log_file=None):
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    formatter = logging.Formatter(
+        '%(asctime)s %(name)s %(levelname)s %(message)s')
+    handlers = [logging.StreamHandler()]
+    if log_file is not None:
+        handlers.append(logging.FileHandler(log_file))
+    for handler in handlers:
+        handler.setLevel(level)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger

Diff for: diva_io/utils/log.py

@@ -0,0 +1 @@
+from .reader import VideoReader