diff --git a/davis2017/davis.py b/davis2017/davis.py index d831be6..d8d628f 100644 --- a/davis2017/davis.py +++ b/davis2017/davis.py @@ -8,10 +8,11 @@ class DAVIS(object): SUBSET_OPTIONS = ['train', 'val', 'test-dev', 'test-challenge'] TASKS = ['semi-supervised', 'unsupervised'] + YEARS = ['2016', '2017', '2019'] DATASET_WEB = 'https://davischallenge.org/davis2017/code.html' VOID_LABEL = 255 - def __init__(self, root, task='unsupervised', subset='val', sequences='all', resolution='480p', codalab=False): + def __init__(self, root, task='unsupervised', subset='val', sequences='all', resolution='480p', codalab=False, year='2017'): """ Class to read the DAVIS dataset :param root: Path to the DAVIS folder that contains JPEGImages, Annotations, etc. folders. @@ -19,11 +20,14 @@ def __init__(self, root, task='unsupervised', subset='val', sequences='all', res :param subset: Set to load the annotations :param sequences: Sequences to consider, 'all' to use all the sequences in a set. :param resolution: Specify the resolution to use the dataset, choose between '480' and 'Full-Resolution' + :param year: DAVIS dataset year. """ if subset not in self.SUBSET_OPTIONS: raise ValueError(f'Subset should be in {self.SUBSET_OPTIONS}') if task not in self.TASKS: raise ValueError(f'The only tasks that are supported are {self.TASKS}') + if year not in self.YEARS: + raise ValueError(f'Year should be one of the following {self.YEARS}') self.task = task self.subset = subset @@ -31,8 +35,12 @@ def __init__(self, root, task='unsupervised', subset='val', sequences='all', res self.img_path = os.path.join(self.root, 'JPEGImages', resolution) annotations_folder = 'Annotations' if task == 'semi-supervised' else 'Annotations_unsupervised' self.mask_path = os.path.join(self.root, annotations_folder, resolution) - year = '2019' if task == 'unsupervised' and (subset == 'test-dev' or subset == 'test-challenge') else '2017' - self.imagesets_path = os.path.join(self.root, 'ImageSets', year) + + self.year = year + if self.year == '2019' and not (task == 'unsupervised' and (subset == 'test-dev' or subset == 'test-challenge')): + raise ValueError("Set 'task' to 'unsupervised' and subset to 'test-dev' or 'test-challenge'") + + self.imagesets_path = os.path.join(self.root, 'ImageSets', self.year) self._check_directories() @@ -95,6 +103,10 @@ def get_all_masks(self, sequence, separate_objects_masks=False): tmp = tmp * np.arange(1, num_objects + 1)[:, None, None, None] masks = (tmp == masks[None, ...]) masks = masks > 0 + else: + # for single object evaluation (e.g. DAVIS2016) + masks = np.expand_dims(masks, axis=0) + masks = masks > 0 return masks, masks_void, masks_id def get_sequences(self): diff --git a/davis2017/evaluation.py b/davis2017/evaluation.py index 7bfb80f..75c513d 100644 --- a/davis2017/evaluation.py +++ b/davis2017/evaluation.py @@ -12,17 +12,19 @@ class DAVISEvaluation(object): - def __init__(self, davis_root, task, gt_set, sequences='all', codalab=False): + def __init__(self, davis_root, task, gt_set, sequences='all', codalab=False, year='2017'): """ Class to evaluate DAVIS sequences from a certain set and for a certain task :param davis_root: Path to the DAVIS folder that contains JPEGImages, Annotations, etc. folders. :param task: Task to compute the evaluation, chose between semi-supervised or unsupervised. - :param gt_set: Set to compute the evaluation + :param gt_set: Set to compute the evaluation. :param sequences: Sequences to consider for the evaluation, 'all' to use all the sequences in a set. + :param year: DAVIS dataset year. """ self.davis_root = davis_root self.task = task - self.dataset = DAVIS(root=davis_root, task=task, subset=gt_set, sequences=sequences, codalab=codalab) + self.year = year + self.dataset = DAVIS(root=davis_root, task=task, subset=gt_set, sequences=sequences, codalab=codalab, year=self.year) @staticmethod def _evaluate_semisupervised(all_gt_masks, all_res_masks, all_void_masks, metric): @@ -77,10 +79,12 @@ def evaluate(self, res_path, metric=('J', 'F'), debug=False): if 'F' in metric: metrics_res['F'] = {"M": [], "R": [], "D": [], "M_per_object": {}} + separate_objects_masks = self.year != '2016' + # Sweep all sequences results = Results(root_dir=res_path) for seq in tqdm(list(self.dataset.get_sequences())): - all_gt_masks, all_void_masks, all_masks_id = self.dataset.get_all_masks(seq, True) + all_gt_masks, all_void_masks, all_masks_id = self.dataset.get_all_masks(seq, separate_objects_masks) if self.task == 'semi-supervised': all_gt_masks, all_masks_id = all_gt_masks[:, 1:-1, :, :], all_masks_id[1:-1] all_res_masks = results.read_masks(seq, all_masks_id) diff --git a/evaluation_method.py b/evaluation_method.py index 04f67d1..f7b2723 100644 --- a/evaluation_method.py +++ b/evaluation_method.py @@ -18,6 +18,8 @@ parser.add_argument('--set', type=str, help='Subset to evaluate the results', default='val') parser.add_argument('--task', type=str, help='Task to evaluate the results', default='unsupervised', choices=['semi-supervised', 'unsupervised']) +parser.add_argument("--year", type=str, help="Davis dataset year (default: 2017)", default='2017', + choices=['2016', '2017', '2019']) parser.add_argument('--results_path', type=str, help='Path to the folder containing the sequences folders', required=True) args, _ = parser.parse_known_args() @@ -34,7 +36,7 @@ else: print(f'Evaluating sequences for the {args.task} task...') # Create dataset and evaluate - dataset_eval = DAVISEvaluation(davis_root=args.davis_path, task=args.task, gt_set=args.set) + dataset_eval = DAVISEvaluation(davis_root=args.davis_path, task=args.task, gt_set=args.set, year=args.year) metrics_res = dataset_eval.evaluate(args.results_path) J, F = metrics_res['J'], metrics_res['F']