[Refactor] Refactor fileio (#2543)

## Motivation Use the new fileio from mmengine open-mmlab/mmengine#533 ## Modification 1. Use `mmengine.fileio` to repalce FileClient in mmseg/datasets 2. Use `mmengine.fileio` to repalce FileClient in mmseg/datasets/transforms 3. Use `mmengine.fileio` to repalce FileClient in mmseg/visualization ## BC-breaking (Optional) we modify all the dataset configurations, so please use the latest config file.
open-mmlab · Feb 1, 2023 · 124b87c · 124b87c
1 parent 698dee9
commit 124b87c
Show file tree

Hide file tree

Showing 30 changed files with 134 additions and 150 deletions.
diff --git a/.circleci/test.yml b/.circleci/test.yml
@@ -61,7 +61,7 @@ jobs:
           command: |
             pip install git+https://github.com/open-mmlab/mmengine.git@main
             pip install -U openmim
-            mim install 'mmcv==2.0.0rc3'
+            mim install 'mmcv>=2.0.0rc4'
             pip install git+https://github.com/open-mmlab/mmclassification@dev-1.x
             mim install 'mmdet==3.0.0rc5'
             pip install -r requirements/tests.txt -r requirements/optional.txt
@@ -107,7 +107,7 @@ jobs:
           command: |
             docker exec mmseg pip install -e /mmengine
             docker exec mmseg pip install -U openmim
-            docker exec mmseg mim install 'mmcv==2.0.0rc3'
+            docker exec mmseg mim install 'mmcv>=2.0.0rc4'
             docker exec mmseg pip install -e /mmclassification
             docker exec mmseg mim install 'mmdet==3.0.0rc5'
             docker exec mmseg pip install -r requirements/tests.txt -r requirements/optional.txt

diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml
@@ -44,7 +44,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -100,7 +100,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -166,7 +166,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -209,7 +209,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -244,7 +244,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies

diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml
@@ -44,7 +44,7 @@ jobs:
         run: |
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -100,7 +100,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies
@@ -135,7 +135,7 @@ jobs:
           python -V
           pip install -U openmim
           pip install git+https://github.com/open-mmlab/mmengine.git
-          mim install 'mmcv==2.0.0rc3'
+          mim install 'mmcv>=2.0.0rc4'
           pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
           mim install 'mmdet==3.0.0rc5'
       - name: Install unittest dependencies

diff --git a/configs/_base_/datasets/ade20k.py b/configs/_base_/datasets/ade20k.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/ade20k_640x640.py b/configs/_base_/datasets/ade20k_640x640.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/chase_db1.py b/configs/_base_/datasets/chase_db1.py
@@ -26,7 +26,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/cityscapes.py b/configs/_base_/datasets/cityscapes.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/coco-stuff10k.py b/configs/_base_/datasets/coco-stuff10k.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/coco-stuff164k.py b/configs/_base_/datasets/coco-stuff164k.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/drive.py b/configs/_base_/datasets/drive.py
@@ -26,7 +26,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/hrf.py b/configs/_base_/datasets/hrf.py
@@ -26,7 +26,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/isaid.py b/configs/_base_/datasets/isaid.py
@@ -32,7 +32,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/loveda.py b/configs/_base_/datasets/loveda.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/pascal_context_59.py b/configs/_base_/datasets/pascal_context_59.py
@@ -28,7 +28,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/pascal_voc12.py b/configs/_base_/datasets/pascal_voc12.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/pascal_voc12_aug.py b/configs/_base_/datasets/pascal_voc12_aug.py
@@ -27,7 +27,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/potsdam.py b/configs/_base_/datasets/potsdam.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/stare.py b/configs/_base_/datasets/stare.py
@@ -26,7 +26,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/configs/_base_/datasets/vaihingen.py b/configs/_base_/datasets/vaihingen.py
@@ -25,7 +25,7 @@
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/docs/en/migration/interface.md b/docs/en/migration/interface.md
@@ -237,7 +237,7 @@ test_pipeline = [
 ]
 img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75]
 tta_pipeline = [
-    dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')),
+    dict(type='LoadImageFromFile', backend_args=dict(backend='local')),
     dict(
         type='TestTimeAug',
         transforms=[

diff --git a/docs/en/notes/faq.md b/docs/en/notes/faq.md
@@ -4,37 +4,19 @@ We list some common troubles faced by many users and their corresponding solutio
 
 ## Installation
 
-The compatible MMSegmentation and MMCV versions are as below. Please install the correct version of MMCV to avoid installation issues.
+The compatible MMSegmentation, MMCV and MMEngine versions are as below. Please install the correct versions of them to avoid installation issues.
 
-| MMSegmentation version |          MMCV version          | MMClassification (optional) version | MMDetection (optional) version |
-| :--------------------: | :----------------------------: | :---------------------------------: | :----------------------------: |
-|   1.x/dev-1.x branch   |        mmcv == 2.0.0rc3        |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4, \<=3.0.0rc5>  |
-|        1.0.0rc4        |        mmcv == 2.0.0rc3        |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4, \<=3.0.0rc5>  |
-|        1.0.0rc3        |        mmcv == 2.0.0rc3        |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4  \<=3.0.0rc5>  |
-|        1.0.0rc2        |        mmcv == 2.0.0rc3        |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4  \<=3.0.0rc5>  |
-|        1.0.0rc1        | mmcv >= 2.0.0rc1, \<=2.0.0rc3> |           mmcls>=1.0.0rc0           |          Not required          |
-|        1.0.0rc0        | mmcv >= 2.0.0rc1, \<=2.0.0rc3> |           mmcls>=1.0.0rc0           |          Not required          |
-|         master         |   mmcv-full>=1.4.4, \<=1.6.0   |       mmcls>=0.20.1, \<=1.0.0       |          Not required          |
-|         0.24.1         |   mmcv-full>=1.4.4, \<=1.6.0   |       mmcls>=0.20.1, \<=1.0.0       |          Not required          |
-|         0.23.0         |   mmcv-full>=1.4.4, \<=1.6.0   |       mmcls>=0.20.1, \<=1.0.0       |          Not required          |
-|         0.22.0         |   mmcv-full>=1.4.4, \<=1.6.0   |       mmcls>=0.20.1, \<=1.0.0       |          Not required          |
-|         0.21.1         |   mmcv-full>=1.4.4, \<=1.6.0   |            Not required             |          Not required          |
-|         0.20.2         |  mmcv-full>=1.3.13, \<=1.6.0   |            Not required             |          Not required          |
-|         0.19.0         |  mmcv-full>=1.3.13, \<1.3.17   |            Not required             |          Not required          |
-|         0.18.0         |  mmcv-full>=1.3.13, \<1.3.17   |            Not required             |          Not required          |
-|         0.17.0         |   mmcv-full>=1.3.7, \<1.3.17   |            Not required             |          Not required          |
-|         0.16.0         |   mmcv-full>=1.3.7, \<1.3.17   |            Not required             |          Not required          |
-|         0.15.0         |   mmcv-full>=1.3.7, \<1.3.17   |            Not required             |          Not required          |
-|         0.14.1         |   mmcv-full>=1.3.7, \<1.3.17   |            Not required             |          Not required          |
-|         0.14.0         |   mmcv-full>=1.3.1, \<1.3.2    |            Not required             |          Not required          |
-|         0.13.0         |   mmcv-full>=1.3.1, \<1.3.2    |            Not required             |          Not required          |
-|         0.12.0         |   mmcv-full>=1.1.4, \<1.3.2    |            Not required             |          Not required          |
-|         0.11.0         |   mmcv-full>=1.1.4, \<1.3.0    |            Not required             |          Not required          |
-|         0.10.0         |   mmcv-full>=1.1.4, \<1.3.0    |            Not required             |          Not required          |
-|         0.9.0          |   mmcv-full>=1.1.4, \<1.3.0    |            Not required             |          Not required          |
-|         0.8.0          |   mmcv-full>=1.1.4, \<1.2.0    |            Not required             |          Not required          |
-|         0.7.0          |   mmcv-full>=1.1.2, \<1.2.0    |            Not required             |          Not required          |
-|         0.6.0          |   mmcv-full>=1.1.2, \<1.2.0    |            Not required             |          Not required          |
+| MMSegmentation version |          MMCV version          | MMEngine version  | MMClassification (optional) version | MMDetection (optional) version |
+| :--------------------: | :----------------------------: | :---------------: | :---------------------------------: | :----------------------------: |
+|     dev-1.x branch     |        mmcv >= 2.0.0rc4        | MMEngine >= 0.2.0 |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4, \<=3.0.0rc5>  |
+|       1.x branch       |        mmcv == 2.0.0rc3        | MMEngine >= 0.2.0 |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4, \<=3.0.0rc5>  |
+|        1.0.0rc4        |        mmcv == 2.0.0rc3        | MMEngine >= 0.1.0 |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4, \<=3.0.0rc5>  |
+|        1.0.0rc3        |        mmcv == 2.0.0rc3        | MMEngine >= 0.1.0 |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4  \<=3.0.0rc5>  |
+|        1.0.0rc2        |        mmcv == 2.0.0rc3        | MMEngine >= 0.1.0 |           mmcls>=1.0.0rc0           | mmdet>=3.0.0rc4  \<=3.0.0rc5>  |
+|        1.0.0rc1        | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | MMEngine >= 0.1.0 |           mmcls>=1.0.0rc0           |          Not required          |
+|        1.0.0rc0        | mmcv >= 2.0.0rc1, \<=2.0.0rc3> | MMEngine >= 0.1.0 |           mmcls>=1.0.0rc0           |          Not required          |
+
+Notes: To install MMSegmentation 0.x and master branch, please refer to [the faq 0.x document](https://mmsegmentation.readthedocs.io/en/latest/faq.html#installation) to check compatible versions of MMCV.
 
 ## How to know the number of GPUs needed to train the model
 

diff --git a/mmseg/__init__.py b/mmseg/__init__.py
@@ -7,9 +7,9 @@
 
 from .version import __version__, version_info
 
-MMCV_MIN = '2.0.0rc3'
-MMCV_MAX = '2.0.0rc3'
-MMENGINE_MIN = '0.1.0'
+MMCV_MIN = '2.0.0rc4'
+MMCV_MAX = '2.1.0'
+MMENGINE_MIN = '0.2.0'
 MMENGINE_MAX = '1.0.0'
 
 
@@ -58,9 +58,9 @@ def digit_version(version_str: str, length: int = 4):
 mmcv_version = digit_version(mmcv.__version__)
 
 
-assert (mmcv_min_version <= mmcv_version <= mmcv_max_version), \
+assert (mmcv_min_version <= mmcv_version < mmcv_max_version), \
     f'MMCV=={mmcv.__version__} is used but incompatible. ' \
-    f'Please install mmcv==2.0.0rc3.'
+    f'Please install mmcv>=2.0.0rc4.'
 
 mmengine_min_version = digit_version(MMENGINE_MIN)
 mmengine_max_version = digit_version(MMENGINE_MAX)

diff --git a/mmseg/datasets/basesegdataset.py b/mmseg/datasets/basesegdataset.py
@@ -4,6 +4,7 @@
 from typing import Callable, Dict, List, Optional, Sequence, Union
 
 import mmengine
+import mmengine.fileio as fileio
 import numpy as np
 from mmengine.dataset import BaseDataset, Compose
 
@@ -72,9 +73,10 @@ class BaseSegDataset(BaseDataset):
         ignore_index (int): The label index to be ignored. Default: 255
         reduce_zero_label (bool): Whether to mark label zero as ignored.
             Default to False.
-        file_client_args (dict): Arguments to instantiate a FileClient.
-            See :class:`mmengine.fileio.FileClient` for details.
-            Defaults to ``dict(backend='disk')``.
+        backend_args (dict): Arguments to instantiate a file backend.
+            See https://mmengine.readthedocs.io/en/latest/api/fileio.htm
+            for details. Defaults to ``dict(backend='local')``
+            Notes: mmcv>=2.0.0rc4, mmengine>=0.2.0 required.
     """
     METAINFO: dict = dict()
 
@@ -95,16 +97,14 @@ def __init__(
         max_refetch: int = 1000,
         ignore_index: int = 255,
         reduce_zero_label: bool = False,
-        file_client_args: dict = dict(backend='disk')
+        backend_args: dict = dict(backend='local')
     ) -> None:
 
         self.img_suffix = img_suffix
         self.seg_map_suffix = seg_map_suffix
         self.ignore_index = ignore_index
         self.reduce_zero_label = reduce_zero_label
-        self.file_client_args = file_client_args
-        self.file_client = mmengine.FileClient.infer_client(
-            self.file_client_args)
+        self.backend_args = backend_args.copy()
 
         self.data_root = data_root
         self.data_prefix = copy.copy(data_prefix)
@@ -239,7 +239,7 @@ def load_data_list(self) -> List[dict]:
         ann_dir = self.data_prefix.get('seg_map_path', None)
         if osp.isfile(self.ann_file):
             lines = mmengine.list_from_file(
-                self.ann_file, file_client_args=self.file_client_args)
+                self.ann_file, backend_args=self.backend_args)
             for line in lines:
                 img_name = line.strip()
                 data_info = dict(
@@ -252,11 +252,12 @@ def load_data_list(self) -> List[dict]:
                 data_info['seg_fields'] = []
                 data_list.append(data_info)
         else:
-            for img in self.file_client.list_dir_or_file(
+            for img in fileio.list_dir_or_file(
                     dir_path=img_dir,
                     list_dir=False,
                     suffix=self.img_suffix,
-                    recursive=True):
+                    recursive=True,
+                    backend_args=self.backend_args):
                 data_info = dict(img_path=osp.join(img_dir, img))
                 if ann_dir is not None:
                     seg_map = img.replace(self.img_suffix, self.seg_map_suffix)

diff --git a/mmseg/datasets/isaid.py b/mmseg/datasets/isaid.py
@@ -1,4 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import mmengine.fileio as fileio
+
 from mmseg.registry import DATASETS
 from .basesegdataset import BaseSegDataset
 
@@ -33,4 +35,5 @@ def __init__(self,
             seg_map_suffix=seg_map_suffix,
             ignore_index=ignore_index,
             **kwargs)
-        assert self.file_client.exists(self.data_prefix['img_path'])
+        assert fileio.exists(
+            self.data_prefix['img_path'], backend_args=self.backend_args)