From 166ca02363ddd2e0a623f0393e9be0cc54243bf3 Mon Sep 17 00:00:00 2001
From: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com>
Date: Tue, 15 Feb 2022 23:43:20 +0800
Subject: [PATCH] [Refactor] Move fileio directory from utils (#23)
* move fileio directory from utils
* remove CephBackend from fileio and clear some deprecated docstring
* fix format
* fix a spellling error
---
mmengine/__init__.py | 1 +
mmengine/{utils => }/fileio/__init__.py | 0
mmengine/{utils => }/fileio/file_client.py | 47 ++-----------------
.../{utils => }/fileio/handlers/__init__.py | 0
mmengine/{utils => }/fileio/handlers/base.py | 0
.../fileio/handlers/json_handler.py | 0
.../fileio/handlers/pickle_handler.py | 0
.../fileio/handlers/yaml_handler.py | 0
mmengine/{utils => }/fileio/io.py | 16 +++----
mmengine/{utils => }/fileio/parse.py | 16 +++----
mmengine/utils/__init__.py | 5 +-
tests/test_data/test_base_dataset.py | 1 +
12 files changed, 19 insertions(+), 67 deletions(-)
rename mmengine/{utils => }/fileio/__init__.py (100%)
rename mmengine/{utils => }/fileio/file_client.py (96%)
rename mmengine/{utils => }/fileio/handlers/__init__.py (100%)
rename mmengine/{utils => }/fileio/handlers/base.py (100%)
rename mmengine/{utils => }/fileio/handlers/json_handler.py (100%)
rename mmengine/{utils => }/fileio/handlers/pickle_handler.py (100%)
rename mmengine/{utils => }/fileio/handlers/yaml_handler.py (100%)
rename mmengine/{utils => }/fileio/io.py (91%)
rename mmengine/{utils => }/fileio/parse.py (84%)
diff --git a/mmengine/__init__.py b/mmengine/__init__.py
index 916c3628..1e4ec2f9 100644
--- a/mmengine/__init__.py
+++ b/mmengine/__init__.py
@@ -1,3 +1,4 @@
# Copyright (c) OpenMMLab. All rights reserved.
# flake8: noqa
+from .fileio import *
from .utils import *
diff --git a/mmengine/utils/fileio/__init__.py b/mmengine/fileio/__init__.py
similarity index 100%
rename from mmengine/utils/fileio/__init__.py
rename to mmengine/fileio/__init__.py
diff --git a/mmengine/utils/fileio/file_client.py b/mmengine/fileio/file_client.py
similarity index 96%
rename from mmengine/utils/fileio/file_client.py
rename to mmengine/fileio/file_client.py
index afc19bb7..955643ee 100644
--- a/mmengine/utils/fileio/file_client.py
+++ b/mmengine/fileio/file_client.py
@@ -4,7 +4,6 @@ import os
import os.path as osp
import re
import tempfile
-import warnings
from abc import ABCMeta, abstractmethod
from contextlib import contextmanager
from pathlib import Path
@@ -43,47 +42,8 @@ class BaseStorageBackend(metaclass=ABCMeta):
pass
-class CephBackend(BaseStorageBackend):
- """Ceph storage backend (for internal use).
-
- Args:
- path_mapping (dict|None): path mapping dict from local path to Petrel
- path. When ``path_mapping={'src': 'dst'}``, ``src`` in ``filepath``
- will be replaced by ``dst``. Default: None.
-
- .. warning::
- :class:`mmcv.fileio.file_client.CephBackend` will be deprecated,
- please use :class:`mmcv.fileio.file_client.PetrelBackend` instead.
- """
-
- def __init__(self, path_mapping=None):
- try:
- import ceph
- except ImportError:
- raise ImportError('Please install ceph to enable CephBackend.')
-
- warnings.warn(
- 'CephBackend will be deprecated, please use PetrelBackend instead',
- DeprecationWarning)
- self._client = ceph.S3Client()
- assert isinstance(path_mapping, dict) or path_mapping is None
- self.path_mapping = path_mapping
-
- def get(self, filepath):
- filepath = str(filepath)
- if self.path_mapping is not None:
- for k, v in self.path_mapping.items():
- filepath = filepath.replace(k, v)
- value = self._client.Get(filepath)
- value_buf = memoryview(value)
- return value_buf
-
- def get_text(self, filepath, encoding=None):
- raise NotImplementedError
-
-
class PetrelBackend(BaseStorageBackend):
- """Petrel storage backend (for internal use).
+ """Petrel storage backend (for internal usage).
PetrelBackend supports reading and writing data to multiple clusters.
If the file path contains the cluster name, PetrelBackend will read data
@@ -152,8 +112,8 @@ class PetrelBackend(BaseStorageBackend):
Returns:
memoryview: A memory view of expected bytes object to avoid
- copying. The memoryview object can be converted to bytes by
- ``value_buf.tobytes()``.
+ copying. The memoryview object can be converted to bytes by
+ ``value_buf.tobytes()``.
"""
filepath = self._map_path(filepath)
filepath = self._format_path(filepath)
@@ -770,7 +730,6 @@ class FileClient:
_backends = {
'disk': HardDiskBackend,
- 'ceph': CephBackend,
'memcached': MemcachedBackend,
'lmdb': LmdbBackend,
'petrel': PetrelBackend,
diff --git a/mmengine/utils/fileio/handlers/__init__.py b/mmengine/fileio/handlers/__init__.py
similarity index 100%
rename from mmengine/utils/fileio/handlers/__init__.py
rename to mmengine/fileio/handlers/__init__.py
diff --git a/mmengine/utils/fileio/handlers/base.py b/mmengine/fileio/handlers/base.py
similarity index 100%
rename from mmengine/utils/fileio/handlers/base.py
rename to mmengine/fileio/handlers/base.py
diff --git a/mmengine/utils/fileio/handlers/json_handler.py b/mmengine/fileio/handlers/json_handler.py
similarity index 100%
rename from mmengine/utils/fileio/handlers/json_handler.py
rename to mmengine/fileio/handlers/json_handler.py
diff --git a/mmengine/utils/fileio/handlers/pickle_handler.py b/mmengine/fileio/handlers/pickle_handler.py
similarity index 100%
rename from mmengine/utils/fileio/handlers/pickle_handler.py
rename to mmengine/fileio/handlers/pickle_handler.py
diff --git a/mmengine/utils/fileio/handlers/yaml_handler.py b/mmengine/fileio/handlers/yaml_handler.py
similarity index 100%
rename from mmengine/utils/fileio/handlers/yaml_handler.py
rename to mmengine/fileio/handlers/yaml_handler.py
diff --git a/mmengine/utils/fileio/io.py b/mmengine/fileio/io.py
similarity index 91%
rename from mmengine/utils/fileio/io.py
rename to mmengine/fileio/io.py
index fbfa93d3..8a451f1d 100644
--- a/mmengine/utils/fileio/io.py
+++ b/mmengine/fileio/io.py
@@ -2,7 +2,7 @@
from io import BytesIO, StringIO
from pathlib import Path
-from mmengine import is_list_of, is_str
+from ..utils import is_list_of, is_str
from .file_client import FileClient
from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler
@@ -20,9 +20,8 @@ def load(file, file_format=None, file_client_args=None, **kwargs):
This method provides a unified api for loading data from serialized files.
- Note:
- In v1.3.16 and later, ``load`` supports loading data from serialized
- files those can be storaged in different backends.
+ ``load`` supports loading data from serialized files those can be storaged
+ in different backends.
Args:
file (str or :obj:`Path` or file-like object): Filename or a file-like
@@ -32,7 +31,7 @@ def load(file, file_format=None, file_client_args=None, **kwargs):
Currently supported formats include "json", "yaml/yml" and
"pickle/pkl".
file_client_args (dict, optional): Arguments to instantiate a
- FileClient. See :class:`mmcv.fileio.FileClient` for details.
+ FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None.
Examples:
@@ -72,9 +71,8 @@ def dump(obj, file=None, file_format=None, file_client_args=None, **kwargs):
This method provides a unified api for dumping data as strings or to files,
and also supports custom arguments for each file format.
- Note:
- In v1.3.16 and later, ``dump`` supports dumping data as strings or to
- files which is saved to different backends.
+ ``dump`` supports dumping data as strings or to files which is saved to
+ different backends.
Args:
obj (any): The python object to be dumped.
@@ -83,7 +81,7 @@ def dump(obj, file=None, file_format=None, file_client_args=None, **kwargs):
specified by the filename or file-like object.
file_format (str, optional): Same as :func:`load`.
file_client_args (dict, optional): Arguments to instantiate a
- FileClient. See :class:`mmcv.fileio.FileClient` for details.
+ FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None.
Examples:
diff --git a/mmengine/utils/fileio/parse.py b/mmengine/fileio/parse.py
similarity index 84%
rename from mmengine/utils/fileio/parse.py
rename to mmengine/fileio/parse.py
index 582221a3..8353b622 100644
--- a/mmengine/utils/fileio/parse.py
+++ b/mmengine/fileio/parse.py
@@ -12,10 +12,8 @@ def list_from_file(filename,
file_client_args=None):
"""Load a text file and parse the content as a list of strings.
- Note:
- In v1.3.16 and later, ``list_from_file`` supports loading a text file
- which can be storaged in different backends and parsing the content as
- a list for strings.
+ ``list_from_file`` supports loading a text file which can be storaged in
+ different backends and parsing the content as a list for strings.
Args:
filename (str): Filename.
@@ -25,7 +23,7 @@ def list_from_file(filename,
zeros and negatives mean no limitation.
encoding (str): Encoding used to open the file. Default utf-8.
file_client_args (dict, optional): Arguments to instantiate a
- FileClient. See :class:`mmcv.fileio.FileClient` for details.
+ FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None.
Examples:
@@ -61,10 +59,8 @@ def dict_from_file(filename,
whitespaces or tabs. The first column will be parsed as dict keys, and
the following columns will be parsed as dict values.
- Note:
- In v1.3.16 and later, ``dict_from_file`` supports loading a text file
- which can be storaged in different backends and parsing the content as
- a dict.
+ ``dict_from_file`` supports loading a text file which can be storaged in
+ different backends and parsing the content as a dict.
Args:
filename(str): Filename.
@@ -72,7 +68,7 @@ def dict_from_file(filename,
type conversion will be performed if specified.
encoding (str): Encoding used to open the file. Default utf-8.
file_client_args (dict, optional): Arguments to instantiate a
- FileClient. See :class:`mmcv.fileio.FileClient` for details.
+ FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None.
Examples:
diff --git a/mmengine/utils/__init__.py b/mmengine/utils/__init__.py
index e709a4ef..952b2dc3 100644
--- a/mmengine/utils/__init__.py
+++ b/mmengine/utils/__init__.py
@@ -1,6 +1,4 @@
# Copyright (c) OpenMMLab. All rights reserved.
-from .fileio import (FileClient, dict_from_file, dump, list_from_file, load,
- register_handler)
from .misc import (check_prerequisites, concat_list, deprecated_api_warning,
has_method, import_modules_from_strings, is_list_of,
is_method_overridden, is_seq_of, is_str, is_tuple_of,
@@ -17,6 +15,5 @@ __all__ = [
'is_filepath', 'fopen', 'check_file_exist', 'mkdir_or_exist', 'symlink',
'scandir', 'deprecated_api_warning', 'import_modules_from_strings',
'to_1tuple', 'to_2tuple', 'to_3tuple', 'to_4tuple', 'to_ntuple',
- 'is_method_overridden', 'has_method', 'dict_from_file', 'list_from_file',
- 'register_handler', 'dump', 'load', 'FileClient'
+ 'is_method_overridden', 'has_method'
]
diff --git a/tests/test_data/test_base_dataset.py b/tests/test_data/test_base_dataset.py
index ff4f60fc..2a33427c 100644
--- a/tests/test_data/test_base_dataset.py
+++ b/tests/test_data/test_base_dataset.py
@@ -1,3 +1,4 @@
+# Copyright (c) OpenMMLab. All rights reserved.
import os.path as osp
from unittest.mock import MagicMock
--
GitLab