Make detectron2 usable without building C extensions

Summary:
The C code in detectron2 are:
* deformable conv & rotated boxes ops
* fast COCO evaluation

So all the core features should be able to work without building any C code.

This PR make D2 (11528ce) usable without compiling C extensions by making some import statements optional. This will make it easier to use D2 (11528ce) where the compilation toolchain & build system is different.

(There is an open internal task about this as well)

Pull Request resolved: #3909

Reviewed By: wat3rBro

Differential Revision: D33713296

Pulled By: zhanghang1989

fbshipit-source-id: e6605367164400546133fd2ab5589c92d7226482

.circleci/config.yml

@@ -130,6 +130,7 @@ install_detectron2: &install_detectron2
       command: |
         # Remove first, in case it's in the CI cache
         pip uninstall -y detectron2
+
         pip install --progress-bar off -e .[all]
         python -m detectron2.utils.collect_env
         ./datasets/prepare_for_tests.sh
@@ -140,6 +141,17 @@ run_unittests: &run_unittests
       command: |
         pytest -v --durations=15 tests  # parallel causes some random failures
 
+uninstall_tests: &uninstall_tests
+  - run:
+      name: Run Tests After Uninstalling
+      command: |
+        pip uninstall -y detectron2
+        # Remove built binaries
+        rm -rf build/ detectron2/*.so
+        # Tests that code is importable without installation
+        PYTHONPATH=. ./.circleci/import-tests.sh
+
+
 # -------------------------------------------------------------------------------------
 # Jobs to run
 # -------------------------------------------------------------------------------------
@@ -163,6 +175,7 @@ jobs:
       - <<: *install_linux_dep
       - <<: *install_detectron2
       - <<: *run_unittests
+      - <<: *uninstall_tests
 
       - save_cache:
           paths:
@@ -188,6 +201,7 @@ jobs:
       - <<: *install_linux_dep
       - <<: *install_detectron2
       - <<: *run_unittests
+      - <<: *uninstall_tests
 
       - save_cache:
           paths:

.circleci/import-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# Test that import works without building detectron2.
+
+# Check that _C is not importable
+python -c "from detectron2 import _C" > /dev/null 2>&1 && {
+  echo "This test should be run without building detectron2."
+  exit 1
+}
+
+# Check that other modules are still importable, even when _C is not importable
+python -c "from detectron2 import modeling"
+python -c "from detectron2 import modeling, data"
+python -c "from detectron2 import evaluation, export, checkpoint"
+python -c "from detectron2 import utils, engine"

.flake8

@@ -2,7 +2,7 @@
 # Keep in sync with setup.cfg which is used for source packages.
 
 [flake8]
-ignore = W503, E203, E221, C901, C408, E741, C407, B017
+ignore = W503, E203, E221, C901, C408, E741, C407, B017, F811
 max-line-length = 100
 max-complexity = 18
 select = B,C,E,F,W,T4,B9

.github/workflows/workflow.yml

@@ -51,17 +51,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Cache dependencies
         uses: actions/cache@v2
         with:
           path: |
-            ${{ env.pythonLocation }}/lib/python3.6/site-packages
+            ${{ env.pythonLocation }}/lib/python3.7/site-packages
             ~/.torch
-          key: ${{ runner.os }}-torch${{ matrix.torch }}-${{ hashFiles('setup.py') }}-20210420
+          key: ${{ runner.os }}-torch${{ matrix.torch }}-${{ hashFiles('setup.py') }}-20220119
 
       - name: Install dependencies
         run: |

detectron2/evaluation/coco_evaluation.py

@@ -19,13 +19,17 @@
 from detectron2.config import CfgNode
 from detectron2.data import MetadataCatalog
 from detectron2.data.datasets.coco import convert_to_coco_json
-from detectron2.evaluation.fast_eval_api import COCOeval_opt
 from detectron2.structures import Boxes, BoxMode, pairwise_iou
 from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import create_small_table
 
 from .evaluator import DatasetEvaluator
 
+try:
+    from detectron2.evaluation.fast_eval_api import COCOeval_opt
+except ImportError:
+    COCOeval_opt = COCOeval
+
 
 class COCOEvaluator(DatasetEvaluator):
     """
@@ -93,6 +97,10 @@ def __init__(
         self._logger = logging.getLogger(__name__)
         self._distributed = distributed
         self._output_dir = output_dir
+
+        if use_fast_impl and (COCOeval_opt is COCOeval):
+            self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.")
+            use_fast_impl = False
         self._use_fast_impl = use_fast_impl
 
         # COCOeval requires the limit on the number of detections per image (maxDets) to be a list

detectron2/layers/deform_conv.py

@@ -8,7 +8,7 @@
 from torch.nn.modules.utils import _pair
 from torchvision.ops import deform_conv2d
 
-from detectron2 import _C
+from detectron2.utils.develop import create_dummy_class, create_dummy_func
 
 from .wrappers import _NewEmptyTensorOp
 
@@ -47,6 +47,7 @@ def forward(
         ctx.bufs_ = [input.new_empty(0), input.new_empty(0)]  # columns, ones
 
         if not input.is_cuda:
+            # TODO: let torchvision support full features of our deformconv.
             if deformable_groups != 1:
                 raise NotImplementedError(
                     "Deformable Conv with deformable_groups != 1 is not supported on CPUs!"
@@ -499,3 +500,15 @@ def extra_repr(self):
         tmpstr += ", deformable_groups=" + str(self.deformable_groups)
         tmpstr += ", bias=" + str(self.with_bias)
         return tmpstr
+
+
+try:
+    from detectron2 import _C
+except ImportError:
+    # TODO: register ops natively so there is no need to import _C.
+    _msg = "detectron2 is not compiled successfully, please build following the instructions!"
+    _args = ("detectron2._C", _msg)
+    DeformConv = create_dummy_class("DeformConv", *_args)
+    ModulatedDeformConv = create_dummy_class("ModulatedDeformConv", *_args)
+    deform_conv = create_dummy_func("deform_conv", *_args)
+    modulated_deform_conv = create_dummy_func("modulated_deform_conv", *_args)

detectron2/utils/develop.py

@@ -0,0 +1,59 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+""" Utilities for developers only.
+These are not visible to users (not automatically imported). And should not
+appeared in docs."""
+# adapted from https://github.com/tensorpack/tensorpack/blob/master/tensorpack/utils/develop.py
+
+
+def create_dummy_class(klass, dependency, message=""):
+    """
+    When a dependency of a class is not available, create a dummy class which throws ImportError
+    when used.
+
+    Args:
+        klass (str): name of the class.
+        dependency (str): name of the dependency.
+        message: extra message to print
+    Returns:
+        class: a class object
+    """
+    err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass)
+    if message:
+        err = err + " " + message
+
+    class _DummyMetaClass(type):
+        # throw error on class attribute access
+        def __getattr__(_, __):
+            raise ImportError(err)
+
+    class _Dummy(object, metaclass=_DummyMetaClass):
+        # throw error on constructor
+        def __init__(self, *args, **kwargs):
+            raise ImportError(err)
+
+    return _Dummy
+
+
+def create_dummy_func(func, dependency, message=""):
+    """
+    When a dependency of a function is not available, create a dummy function which throws
+    ImportError when used.
+
+    Args:
+        func (str): name of the function.
+        dependency (str or list[str]): name(s) of the dependency.
+        message: extra message to print
+    Returns:
+        function: a function object
+    """
+    err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func)
+    if message:
+        err = err + " " + message
+
+    if isinstance(dependency, (list, tuple)):
+        dependency = ",".join(dependency)
+
+    def _dummy(*args, **kwargs):
+        raise ImportError(err)
+
+    return _dummy

tools/train_net.py

@@ -19,7 +19,6 @@
 import logging
 import os
 from collections import OrderedDict
-import torch
 
 import detectron2.utils.comm as comm
 from detectron2.checkpoint import DetectionCheckpointer