Moved from fvcore's PathManager to iopath's

fsdet/checkpoint/__init__.py

@@ -1,4 +1,3 @@
-from . import catalog as _UNUSED  # register the handler
 from .detection_checkpoint import DetectionCheckpointer
 
 __all__ = ["DetectionCheckpointer"]

fsdet/checkpoint/detection_checkpoint.py

@@ -1,6 +1,6 @@
 import pickle
+from fsdet.utils.file_io import PathManager
 from fvcore.common.checkpoint import Checkpointer
-from fvcore.common.file_io import PathManager
 
 import detectron2.utils.comm as comm
 from detectron2.checkpoint.c2_model_loading import align_and_update_state_dicts
@@ -20,6 +20,7 @@ def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables):
             save_to_disk=is_main_process if save_to_disk is None else save_to_disk,
             **checkpointables,
         )
+        self.path_manager = PathManager
 
     def _load_file(self, filename):
         if filename.endswith(".pkl"):
@@ -46,12 +47,22 @@ def _load_model(self, checkpoint):
         if checkpoint.get("matching_heuristics", False):
             self._convert_ndarray_to_tensor(checkpoint["model"])
             # convert weights by name-matching heuristics
-            model_state_dict = self.model.state_dict()
-            align_and_update_state_dicts(
-                model_state_dict,
+            checkpoint["model"] = align_and_update_state_dicts(
+                self.model.state_dict(),
                 checkpoint["model"],
                 c2_conversion=checkpoint.get("__author__", None) == "Caffe2",
             )
-            checkpoint["model"] = model_state_dict
         # for non-caffe2 models, use standard ways to load it
-        super()._load_model(checkpoint)
+        incompatible = super()._load_model(checkpoint)
+
+        model_buffers = dict(self.model.named_buffers(recurse=False))
+        for k in ["pixel_mean", "pixel_std"]:
+            # Ignore missing key message about pixel_mean/std.
+            # Though they may be missing in old checkpoints, they will be correctly
+            # initialized from config anyway.
+            if k in model_buffers:
+                try:
+                    incompatible.missing_keys.remove(k)
+                except ValueError:
+                    pass
+        return incompatible

fsdet/data/meta_coco.py

@@ -1,12 +1,12 @@
 import numpy as np
-from fvcore.common.file_io import PathManager
 from pycocotools.coco import COCO
 
 import contextlib
 import io
 import os
 from detectron2.data import DatasetCatalog, MetadataCatalog
 from detectron2.structures import BoxMode
+from fsdet.utils.file_io import PathManager
 
 """
 This file contains functions to parse COCO-format annotations into dicts in "Detectron2 format".

fsdet/data/meta_lvis.py

@@ -1,4 +1,4 @@
-from fvcore.common.file_io import PathManager
+from fsdet.utils.file_io import PathManager
 from fvcore.common.timer import Timer
 
 import logging

fsdet/data/meta_pascal_voc.py

@@ -1,5 +1,5 @@
 import numpy as np
-from fvcore.common.file_io import PathManager
+from fsdet.utils.file_io import PathManager
 
 import os
 import xml.etree.ElementTree as ET

fsdet/engine/defaults.py

@@ -11,7 +11,6 @@
 import os
 from collections import OrderedDict
 import torch
-from fvcore.common.file_io import PathManager
 from fvcore.nn.precise_bn import get_bn_modules
 from torch.nn.parallel import DistributedDataParallel
 
@@ -25,6 +24,7 @@
     verify_results,
 )
 from fsdet.modeling import build_model
+from fsdet.utils.file_io import PathManager
 from detectron2.data import (
     MetadataCatalog,
     build_detection_test_loader,
@@ -40,7 +40,6 @@
     JSONWriter,
     TensorboardXWriter,
 )
-
 from detectron2.utils.logger import setup_logger
 
 

fsdet/engine/hooks.py

@@ -3,13 +3,14 @@
 import os
 import time
 import torch
-from fvcore.common.file_io import PathManager
 
 import detectron2.utils.comm as comm
 from detectron2.config import global_cfg
 from detectron2.engine.train_loop import HookBase
 from detectron2.evaluation.testing import flatten_results_dict
 
+from fsdet.utils.file_io import PathManager
+
 __all__ = ["EvalHookFsdet"]
 
 

fsdet/evaluation/coco_evaluation.py

@@ -8,7 +8,6 @@
 import os
 import torch
 from collections import OrderedDict
-from fvcore.common.file_io import PathManager
 from pycocotools.coco import COCO
 from pycocotools.cocoeval import COCOeval
 from tabulate import tabulate
@@ -20,6 +19,7 @@
 from detectron2.utils.logger import create_small_table
 
 from fsdet.evaluation.evaluator import DatasetEvaluator
+from fsdet.utils.file_io import PathManager
 
 
 class COCOEvaluator(DatasetEvaluator):

fsdet/evaluation/lvis_evaluation.py

@@ -5,14 +5,14 @@
 import os
 from collections import OrderedDict
 import torch
-from fvcore.common.file_io import PathManager
 
 import detectron2.utils.comm as comm
 from detectron2.data import MetadataCatalog
 from detectron2.utils.logger import create_small_table
 
 from fsdet.evaluation.coco_evaluation import instances_to_coco_json
 from fsdet.evaluation.evaluator import DatasetEvaluator
+from fsdet.utils.file_io import PathManager
 
 
 class LVISEvaluator(DatasetEvaluator):

fsdet/utils/file_io.py

@@ -0,0 +1,56 @@
+from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler
+from iopath.common.file_io import PathManager as PathManagerBase
+
+__all__ = ["PathManager", "PathHandler"]
+
+
+PathManager = PathManagerBase()
+"""
+This is a detectron2 project-specific PathManager.
+We try to stay away from global PathManager in fvcore as it
+introduces potential conflicts among other libraries.
+"""
+
+
+class Detectron2Handler(PathHandler):
+    """
+    Resolve anything that's hosted under detectron2's namespace.
+    """
+
+    PREFIX = "detectron2://"
+    S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
+
+    def _get_supported_prefixes(self):
+        return [self.PREFIX]
+
+    def _get_local_path(self, path):
+        name = path[len(self.PREFIX) :]
+        return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name)
+
+    def _open(self, path, mode="r", **kwargs):
+        return PathManager.open(self._get_local_path(path), mode, **kwargs)
+
+
+class FsDetHandler(PathHandler):
+    """
+    Resolve anything that's in FsDet model zoo.
+    """
+
+    PREFIX = "fsdet://"
+    URL_PREFIX = "http://dl.yf.io/fs-det/models/"
+
+    def _get_supported_prefixes(self):
+        return [self.PREFIX]
+
+    def _get_local_path(self, path):
+        name = path[len(self.PREFIX) :]
+        return PathManager.get_local_path(self.URL_PREFIX + name)
+
+    def _open(self, path, mode="r", **kwargs):
+        return PathManager.open(self._get_local_path(path), mode, **kwargs)
+
+
+PathManager.register_handler(HTTPURLHandler())
+PathManager.register_handler(OneDrivePathHandler())
+PathManager.register_handler(Detectron2Handler())
+PathManager.register_handler(FsDetHandler())