diff --git a/fsdet/checkpoint/__init__.py b/fsdet/checkpoint/__init__.py index 83dfe9f4e..441936613 100755 --- a/fsdet/checkpoint/__init__.py +++ b/fsdet/checkpoint/__init__.py @@ -1,4 +1,3 @@ -from . import catalog as _UNUSED # register the handler from .detection_checkpoint import DetectionCheckpointer __all__ = ["DetectionCheckpointer"] \ No newline at end of file diff --git a/fsdet/checkpoint/detection_checkpoint.py b/fsdet/checkpoint/detection_checkpoint.py index 2c0f97d7e..6509bab8f 100644 --- a/fsdet/checkpoint/detection_checkpoint.py +++ b/fsdet/checkpoint/detection_checkpoint.py @@ -1,6 +1,6 @@ import pickle +from fsdet.utils.file_io import PathManager from fvcore.common.checkpoint import Checkpointer -from fvcore.common.file_io import PathManager import detectron2.utils.comm as comm from detectron2.checkpoint.c2_model_loading import align_and_update_state_dicts @@ -20,6 +20,7 @@ def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables): save_to_disk=is_main_process if save_to_disk is None else save_to_disk, **checkpointables, ) + self.path_manager = PathManager def _load_file(self, filename): if filename.endswith(".pkl"): @@ -46,12 +47,22 @@ def _load_model(self, checkpoint): if checkpoint.get("matching_heuristics", False): self._convert_ndarray_to_tensor(checkpoint["model"]) # convert weights by name-matching heuristics - model_state_dict = self.model.state_dict() - align_and_update_state_dicts( - model_state_dict, + checkpoint["model"] = align_and_update_state_dicts( + self.model.state_dict(), checkpoint["model"], c2_conversion=checkpoint.get("__author__", None) == "Caffe2", ) - checkpoint["model"] = model_state_dict # for non-caffe2 models, use standard ways to load it - super()._load_model(checkpoint) + incompatible = super()._load_model(checkpoint) + + model_buffers = dict(self.model.named_buffers(recurse=False)) + for k in ["pixel_mean", "pixel_std"]: + # Ignore missing key message about pixel_mean/std. + # Though they may be missing in old checkpoints, they will be correctly + # initialized from config anyway. + if k in model_buffers: + try: + incompatible.missing_keys.remove(k) + except ValueError: + pass + return incompatible diff --git a/fsdet/data/meta_coco.py b/fsdet/data/meta_coco.py index f4f615102..c139323a1 100644 --- a/fsdet/data/meta_coco.py +++ b/fsdet/data/meta_coco.py @@ -1,5 +1,4 @@ import numpy as np -from fvcore.common.file_io import PathManager from pycocotools.coco import COCO import contextlib @@ -7,6 +6,7 @@ import os from detectron2.data import DatasetCatalog, MetadataCatalog from detectron2.structures import BoxMode +from fsdet.utils.file_io import PathManager """ This file contains functions to parse COCO-format annotations into dicts in "Detectron2 format". diff --git a/fsdet/data/meta_lvis.py b/fsdet/data/meta_lvis.py index 8ae7f57f1..4eaaba440 100644 --- a/fsdet/data/meta_lvis.py +++ b/fsdet/data/meta_lvis.py @@ -1,4 +1,4 @@ -from fvcore.common.file_io import PathManager +from fsdet.utils.file_io import PathManager from fvcore.common.timer import Timer import logging diff --git a/fsdet/data/meta_pascal_voc.py b/fsdet/data/meta_pascal_voc.py index 57e3aac90..2dd1132dc 100644 --- a/fsdet/data/meta_pascal_voc.py +++ b/fsdet/data/meta_pascal_voc.py @@ -1,5 +1,5 @@ import numpy as np -from fvcore.common.file_io import PathManager +from fsdet.utils.file_io import PathManager import os import xml.etree.ElementTree as ET diff --git a/fsdet/engine/defaults.py b/fsdet/engine/defaults.py index 893b719fa..8d570c0c2 100755 --- a/fsdet/engine/defaults.py +++ b/fsdet/engine/defaults.py @@ -11,7 +11,6 @@ import os from collections import OrderedDict import torch -from fvcore.common.file_io import PathManager from fvcore.nn.precise_bn import get_bn_modules from torch.nn.parallel import DistributedDataParallel @@ -25,6 +24,7 @@ verify_results, ) from fsdet.modeling import build_model +from fsdet.utils.file_io import PathManager from detectron2.data import ( MetadataCatalog, build_detection_test_loader, @@ -40,7 +40,6 @@ JSONWriter, TensorboardXWriter, ) - from detectron2.utils.logger import setup_logger diff --git a/fsdet/engine/hooks.py b/fsdet/engine/hooks.py index 413c19253..84db5312d 100644 --- a/fsdet/engine/hooks.py +++ b/fsdet/engine/hooks.py @@ -3,13 +3,14 @@ import os import time import torch -from fvcore.common.file_io import PathManager import detectron2.utils.comm as comm from detectron2.config import global_cfg from detectron2.engine.train_loop import HookBase from detectron2.evaluation.testing import flatten_results_dict +from fsdet.utils.file_io import PathManager + __all__ = ["EvalHookFsdet"] diff --git a/fsdet/evaluation/coco_evaluation.py b/fsdet/evaluation/coco_evaluation.py index 509082d85..3051d49b1 100755 --- a/fsdet/evaluation/coco_evaluation.py +++ b/fsdet/evaluation/coco_evaluation.py @@ -8,7 +8,6 @@ import os import torch from collections import OrderedDict -from fvcore.common.file_io import PathManager from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval from tabulate import tabulate @@ -20,6 +19,7 @@ from detectron2.utils.logger import create_small_table from fsdet.evaluation.evaluator import DatasetEvaluator +from fsdet.utils.file_io import PathManager class COCOEvaluator(DatasetEvaluator): diff --git a/fsdet/evaluation/lvis_evaluation.py b/fsdet/evaluation/lvis_evaluation.py index 5b14492b2..afaba052b 100755 --- a/fsdet/evaluation/lvis_evaluation.py +++ b/fsdet/evaluation/lvis_evaluation.py @@ -5,7 +5,6 @@ import os from collections import OrderedDict import torch -from fvcore.common.file_io import PathManager import detectron2.utils.comm as comm from detectron2.data import MetadataCatalog @@ -13,6 +12,7 @@ from fsdet.evaluation.coco_evaluation import instances_to_coco_json from fsdet.evaluation.evaluator import DatasetEvaluator +from fsdet.utils.file_io import PathManager class LVISEvaluator(DatasetEvaluator): diff --git a/fsdet/checkpoint/catalog.py b/fsdet/utils/file_io.py old mode 100755 new mode 100644 similarity index 53% rename from fsdet/checkpoint/catalog.py rename to fsdet/utils/file_io.py index 7b59b9574..7b562b85d --- a/fsdet/checkpoint/catalog.py +++ b/fsdet/utils/file_io.py @@ -1,7 +1,15 @@ -"""FS-Det Model Handler.""" -from fvcore.common.file_io import PathHandler, PathManager +from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler +from iopath.common.file_io import PathManager as PathManagerBase -import logging +__all__ = ["PathManager", "PathHandler"] + + +PathManager = PathManagerBase() +""" +This is a detectron2 project-specific PathManager. +We try to stay away from global PathManager in fvcore as it +introduces potential conflicts among other libraries. +""" class FsDetHandler(PathHandler): @@ -23,4 +31,6 @@ def _open(self, path, mode="r", **kwargs): return PathManager.open(self._get_local_path(path), mode, **kwargs) +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) PathManager.register_handler(FsDetHandler())