diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b19244db0c..eca033f085 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -26,12 +26,12 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.3.0, 1.5.1, 1.6.0, 1.7.0]
+        torch: [1.3.0, 1.5.0, 1.6.0, 1.7.0]
         include:
           - torch: 1.3.0
             torchvision: 0.4.1
-          - torch: 1.5.1
-            torchvision: 0.6.1
+          - torch: 1.5.0
+            torchvision: 0.6.0
           - torch: 1.6.0
             torchvision: 0.7.0
           - torch: 1.7.0
@@ -60,7 +60,7 @@ jobs:
       - name: Install MMCV
         run: pip install mmcv-full==1.2.4 -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html
       - name: Install MMDet
-        run: pip install -q git+https://github.com/open-mmlab/mmdetection/
+        run: pip install git+https://github.com/open-mmlab/mmdetection/
       - name: Install unittest dependencies
         run: pip install -r requirements/tests.txt -r requirements/optional.txt
       - name: Build and install
@@ -80,14 +80,14 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.3.0, 1.5.1+cu101, 1.6.0+cu101, 1.7.0+cu101]
+        torch: [1.3.0, 1.5.0+cu101, 1.6.0+cu101, 1.7.0+cu101]
         include:
           - torch: 1.3.0
             torchvision: 0.4.1
             mmcv: "cu101/torch1.3.0"
-          - torch: 1.5.1+cu101
-            torchvision: 0.6.1+cu101
-            mmcv: "cu101/torch1.5.1"
+          - torch: 1.5.0+cu101
+            torchvision: 0.6.0+cu101
+            mmcv: "cu101/torch1.5.0"
           - torch: 1.6.0+cu101
             torchvision: 0.7.0+cu101
             mmcv: "cu101/torch1.6.0"
diff --git a/mmaction/core/bbox/assigners/max_iou_assigner_ava.py b/mmaction/core/bbox/assigners/max_iou_assigner_ava.py
index 17d4d22482..2af7c2915f 100644
--- a/mmaction/core/bbox/assigners/max_iou_assigner_ava.py
+++ b/mmaction/core/bbox/assigners/max_iou_assigner_ava.py
@@ -3,14 +3,15 @@
 import torch
 
 try:
-    import mmdet  # noqa
     from mmdet.core.bbox import AssignResult, MaxIoUAssigner
     from mmdet.core.bbox.builder import BBOX_ASSIGNERS
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use AssignResult, MaxIoUAssigner '
                   'and BBOX_ASSIGNERS')
+    mmdet_imported = False
 
-if 'mmdet' in dir():
+if mmdet_imported:
 
     @BBOX_ASSIGNERS.register_module()
     class MaxIoUAssignerAVA(MaxIoUAssigner):
diff --git a/mmaction/models/backbones/resnet3d.py b/mmaction/models/backbones/resnet3d.py
index 4b91f06f51..db9a6a4d4e 100644
--- a/mmaction/models/backbones/resnet3d.py
+++ b/mmaction/models/backbones/resnet3d.py
@@ -12,10 +12,11 @@
 from ..registry import BACKBONES
 
 try:
-    import mmdet  # noqa
     from mmdet.models.builder import SHARED_HEADS as MMDET_SHARED_HEADS
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use MMDET_SHARED_HEADS')
+    mmdet_imported = False
 
 
 class BasicBlock3d(nn.Module):
@@ -999,5 +1000,5 @@ def train(self, mode=True):
                     m.eval()
 
 
-if 'mmdet' in dir():
+if mmdet_imported:
     MMDET_SHARED_HEADS.register_module()(ResNet3dLayer)
diff --git a/mmaction/models/backbones/resnet3d_slowfast.py b/mmaction/models/backbones/resnet3d_slowfast.py
index 02b5f13120..38fb2fc282 100644
--- a/mmaction/models/backbones/resnet3d_slowfast.py
+++ b/mmaction/models/backbones/resnet3d_slowfast.py
@@ -11,10 +11,11 @@
 from .resnet3d import ResNet3d
 
 try:
-    import mmdet  # noqa
     from mmdet.models import BACKBONES as MMDET_BACKBONES
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use MMDET_BACKBONES')
+    mmdet_imported = False
 
 
 class ResNet3dPathway(ResNet3d):
@@ -509,5 +510,5 @@ def forward(self, x):
         return out
 
 
-if 'mmdet' in dir():
+if mmdet_imported:
     MMDET_BACKBONES.register_module()(ResNet3dSlowFast)
diff --git a/mmaction/models/backbones/resnet3d_slowonly.py b/mmaction/models/backbones/resnet3d_slowonly.py
index 8e39bd288a..f5aef34d70 100644
--- a/mmaction/models/backbones/resnet3d_slowonly.py
+++ b/mmaction/models/backbones/resnet3d_slowonly.py
@@ -4,10 +4,11 @@
 from .resnet3d_slowfast import ResNet3dPathway
 
 try:
-    import mmdet  # noqa
     from mmdet.models.builder import BACKBONES as MMDET_BACKBONES
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use MMDET_BACKBONES')
+    mmdet_imported = False
 
 
 @BACKBONES.register_module()
@@ -50,5 +51,5 @@ def __init__(self,
         assert not self.lateral
 
 
-if 'mmdet' in dir():
+if mmdet_imported:
     MMDET_BACKBONES.register_module()(ResNet3dSlowOnly)
diff --git a/mmaction/models/heads/bbox_head.py b/mmaction/models/heads/bbox_head.py
index 208e2f3897..357d4e5a0f 100644
--- a/mmaction/models/heads/bbox_head.py
+++ b/mmaction/models/heads/bbox_head.py
@@ -7,10 +7,11 @@
 from mmaction.core.bbox import bbox_target
 
 try:
-    import mmdet  # noqa
     from mmdet.models.builder import HEADS as MMDET_HEADS
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use MMDET_HEADS')
+    mmdet_imported = False
 
 
 class BBoxHeadAVA(nn.Module):
@@ -217,5 +218,5 @@ def _bbox_crop_undo(bboxes, crop_quadruple):
         return bboxes, scores
 
 
-if 'mmdet' in dir():
+if mmdet_imported:
     MMDET_HEADS.register_module()(BBoxHeadAVA)
diff --git a/mmaction/models/heads/roi_head.py b/mmaction/models/heads/roi_head.py
index 45a36df528..75b715e107 100644
--- a/mmaction/models/heads/roi_head.py
+++ b/mmaction/models/heads/roi_head.py
@@ -5,15 +5,16 @@
 from mmaction.core.bbox import bbox2result
 
 try:
-    import mmdet  # noqa
     from mmdet.core.bbox import bbox2roi
     from mmdet.models import HEADS as MMDET_HEADS
     from mmdet.models.roi_heads import StandardRoIHead
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use bbox2roi, MMDET_HEADS '
                   'and StandardRoIHead')
+    mmdet_imported = False
 
-if 'mmdet' in dir():
+if mmdet_imported:
 
     @MMDET_HEADS.register_module()
     class AVARoIHead(StandardRoIHead):
diff --git a/mmaction/models/roi_extractors/single_straight3d.py b/mmaction/models/roi_extractors/single_straight3d.py
index 294cce7a94..0665ae38c2 100644
--- a/mmaction/models/roi_extractors/single_straight3d.py
+++ b/mmaction/models/roi_extractors/single_straight3d.py
@@ -9,10 +9,11 @@
     warnings.warn('Please install mmcv-full to use RoIAlign and RoIPool')
 
 try:
-    import mmdet  # noqa
     from mmdet.models import ROI_EXTRACTORS
+    mmdet_imported = True
 except (ImportError, ModuleNotFoundError):
     warnings.warn('Please install mmdet to use ROI_EXTRACTORS')
+    mmdet_imported = False
 
 
 class SingleRoIExtractor3D(nn.Module):
@@ -101,5 +102,5 @@ def forward(self, feat, rois):
         return torch.stack(roi_feats, dim=2)
 
 
-if 'mmdet' in dir():
+if mmdet_imported:
     ROI_EXTRACTORS.register_module()(SingleRoIExtractor3D)