diff --git a/dvc/path_info.py b/dvc/path_info.py
index 4f757b00b6..e502130aa8 100644
--- a/dvc/path_info.py
+++ b/dvc/path_info.py
@@ -33,10 +33,6 @@ def __new__(cls, *args):
             cls = WindowsPathInfo if os.name == "nt" else PosixPathInfo
         return cls._from_parts(args)
 
-    @classmethod
-    def from_posix(cls, s):
-        return cls(PosixPathInfo(s))
-
     def as_posix(self):
         f = self._flavour
         # Unlike original implementation [1] that uses `str()` we actually need
diff --git a/dvc/remote/base.py b/dvc/remote/base.py
index c349a7a12b..d719a9f8a2 100644
--- a/dvc/remote/base.py
+++ b/dvc/remote/base.py
@@ -21,7 +21,7 @@
     RemoteCacheRequiredError,
 )
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo, URLInfo
+from dvc.path_info import PathInfo, URLInfo, WindowsPathInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
@@ -285,10 +285,13 @@ def load_dir_cache(self, checksum):
             )
             return []
 
-        for info in d:
-            # NOTE: here is a BUG, see comment to .as_posix() below
-            relative_path = PathInfo.from_posix(info[self.PARAM_RELPATH])
-            info[self.PARAM_RELPATH] = relative_path.fspath
+        if self.path_cls == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                info[self.PARAM_RELPATH] = info[self.PARAM_RELPATH].replace(
+                    "/", self.path_cls.sep
+                )
 
         return d