Add year argument in evaluation. Support evaluation on DAVIS 2016 (si…

…ngle-object). Adapted from patch at: davisvideochallenge#4 (comment).

davis2017/davis.py

@@ -8,31 +8,39 @@
 class DAVIS(object):
     SUBSET_OPTIONS = ['train', 'val', 'test-dev', 'test-challenge']
     TASKS = ['semi-supervised', 'unsupervised']
+    YEARS = ['2016', '2017', '2019']
     DATASET_WEB = 'https://davischallenge.org/davis2017/code.html'
     VOID_LABEL = 255
 
-    def __init__(self, root, task='unsupervised', subset='val', sequences='all', resolution='480p', codalab=False):
+    def __init__(self, root, task='unsupervised', subset='val', sequences='all', resolution='480p', codalab=False, year='2017'):
         """
         Class to read the DAVIS dataset
         :param root: Path to the DAVIS folder that contains JPEGImages, Annotations, etc. folders.
         :param task: Task to load the annotations, choose between semi-supervised or unsupervised.
         :param subset: Set to load the annotations
         :param sequences: Sequences to consider, 'all' to use all the sequences in a set.
         :param resolution: Specify the resolution to use the dataset, choose between '480' and 'Full-Resolution'
+        :param year: DAVIS dataset year.
         """
         if subset not in self.SUBSET_OPTIONS:
             raise ValueError(f'Subset should be in {self.SUBSET_OPTIONS}')
         if task not in self.TASKS:
             raise ValueError(f'The only tasks that are supported are {self.TASKS}')
+        if year not in self.YEARS:
+            raise ValueError(f'Year should be one of the following {self.YEARS}')
 
         self.task = task
         self.subset = subset
         self.root = root
         self.img_path = os.path.join(self.root, 'JPEGImages', resolution)
         annotations_folder = 'Annotations' if task == 'semi-supervised' else 'Annotations_unsupervised'
         self.mask_path = os.path.join(self.root, annotations_folder, resolution)
-        year = '2019' if task == 'unsupervised' and (subset == 'test-dev' or subset == 'test-challenge') else '2017'
-        self.imagesets_path = os.path.join(self.root, 'ImageSets', year)
+
+        self.year = year
+        if self.year == '2019' and not (task == 'unsupervised' and (subset == 'test-dev' or subset == 'test-challenge')):
+            raise ValueError("Set 'task' to 'unsupervised' and subset to 'test-dev' or 'test-challenge'")
+
+        self.imagesets_path = os.path.join(self.root, 'ImageSets', self.year)
 
         self._check_directories()
 
@@ -95,6 +103,10 @@ def get_all_masks(self, sequence, separate_objects_masks=False):
             tmp = tmp * np.arange(1, num_objects + 1)[:, None, None, None]
             masks = (tmp == masks[None, ...])
             masks = masks > 0
+        else:
+            # for single object evaluation (e.g. DAVIS2016)
+            masks = np.expand_dims(masks, axis=0)
+            masks = masks > 0
         return masks, masks_void, masks_id
 
     def get_sequences(self):

davis2017/evaluation.py

@@ -12,17 +12,19 @@
 
 
 class DAVISEvaluation(object):
-    def __init__(self, davis_root, task, gt_set, sequences='all', codalab=False):
+    def __init__(self, davis_root, task, gt_set, sequences='all', codalab=False, year='2017'):
         """
         Class to evaluate DAVIS sequences from a certain set and for a certain task
         :param davis_root: Path to the DAVIS folder that contains JPEGImages, Annotations, etc. folders.
         :param task: Task to compute the evaluation, chose between semi-supervised or unsupervised.
-        :param gt_set: Set to compute the evaluation
+        :param gt_set: Set to compute the evaluation.
         :param sequences: Sequences to consider for the evaluation, 'all' to use all the sequences in a set.
+        :param year: DAVIS dataset year.
         """
         self.davis_root = davis_root
         self.task = task
-        self.dataset = DAVIS(root=davis_root, task=task, subset=gt_set, sequences=sequences, codalab=codalab)
+        self.year = year
+        self.dataset = DAVIS(root=davis_root, task=task, subset=gt_set, sequences=sequences, codalab=codalab, year=self.year)
 
     @staticmethod
     def _evaluate_semisupervised(all_gt_masks, all_res_masks, all_void_masks, metric):
@@ -77,10 +79,12 @@ def evaluate(self, res_path, metric=('J', 'F'), debug=False):
         if 'F' in metric:
             metrics_res['F'] = {"M": [], "R": [], "D": [], "M_per_object": {}}
 
+        separate_objects_masks = self.year != '2016'
+
         # Sweep all sequences
         results = Results(root_dir=res_path)
         for seq in tqdm(list(self.dataset.get_sequences())):
-            all_gt_masks, all_void_masks, all_masks_id = self.dataset.get_all_masks(seq, True)
+            all_gt_masks, all_void_masks, all_masks_id = self.dataset.get_all_masks(seq, separate_objects_masks)
             if self.task == 'semi-supervised':
                 all_gt_masks, all_masks_id = all_gt_masks[:, 1:-1, :, :], all_masks_id[1:-1]
             all_res_masks = results.read_masks(seq, all_masks_id)

evaluation_method.py

@@ -18,6 +18,8 @@
 parser.add_argument('--set', type=str, help='Subset to evaluate the results', default='val')
 parser.add_argument('--task', type=str, help='Task to evaluate the results', default='unsupervised',
                     choices=['semi-supervised', 'unsupervised'])
+parser.add_argument("--year", type=str, help="Davis dataset year (default: 2017)", default='2017',
+                    choices=['2016', '2017', '2019'])
 parser.add_argument('--results_path', type=str, help='Path to the folder containing the sequences folders',
                     required=True)
 args, _ = parser.parse_known_args()
@@ -34,7 +36,7 @@
 else:
     print(f'Evaluating sequences for the {args.task} task...')
     # Create dataset and evaluate
-    dataset_eval = DAVISEvaluation(davis_root=args.davis_path, task=args.task, gt_set=args.set)
+    dataset_eval = DAVISEvaluation(davis_root=args.davis_path, task=args.task, gt_set=args.set, year=args.year)
     metrics_res = dataset_eval.evaluate(args.results_path)
     J, F = metrics_res['J'], metrics_res['F']