perf:optimize typing in megfile (#380)

.github/workflows/publish-docs.yml

@@ -12,8 +12,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [3.9]
+        os: [ubuntu-22.04]
+        python-version: [3.10]
 
     steps:
       - name: Checkout Github Repository

.github/workflows/run-tests.yml

@@ -12,7 +12,7 @@ jobs:
     name: Unit Test (py${{ matrix.python-version}}, ${{ matrix.os }})
     strategy:
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -58,8 +58,8 @@ jobs:
     name: Style Check (py${{matrix.python-version}}, ${{ matrix.os }})
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [3.9]
+        os: [ubuntu-22.04]
+        python-version: ["3.10"]
 
     steps:
       - name: Checkout Github Repository
@@ -95,8 +95,8 @@ jobs:
     name: Static Check (pytype, py${{matrix.python-version}}, ${{ matrix.os }})
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [3.9]
+        os: [ubuntu-22.04]
+        python-version: ["3.10"]
 
     steps:
       - name: Checkout Github Repository
@@ -132,8 +132,8 @@ jobs:
     name: Security Check (bandit, py${{ matrix.python-version }}, ${{ matrix.os }})
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [3.9]
+        os: [ubuntu-22.04]
+        python-version: ["3.10"]
 
     steps:
       - name: Checkout Github Repository
@@ -181,8 +181,8 @@ jobs:
     name: Static Check (pyre, py${{matrix.python-version}}, ${{ matrix.os }})
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [3.9]
+        os: [ubuntu-22.04]
+        python-version: ["3.10"]
 
     steps:
       - name: Checkout Github Repository
@@ -212,7 +212,6 @@ jobs:
       - name: Run pyre-check
         continue-on-error: true
         run: |
-          pip install --upgrade pyre-check==0.9.6
           make pyre_check
 
       - name: Expose SARIF Results

.pyre_configuration

@@ -1,4 +1,5 @@
 {
+  "site_package_search_strategy": "pep561",
   "search_path": [
     {
       "site-package": "boto3"
@@ -10,7 +11,10 @@
       "site-package": "requests"
     },
     {
-      "site-package": "smart_open"
+      "site-package": "urllib3"
+    },
+    {
+      "site-package": "hdfs"
     }
   ],
   "source_directories": [

CHANGELOG.md

@@ -153,7 +153,7 @@
 
 ## 2.2.3 - 2023.08.25
 - feat
-    - add `force` param in sync methods for copy file forcely
+    - add `force` param in sync methods for copy file forcible
     - add `-f` / `--force` in `refile sync`
     - check same file in copy, and raise `SameFileError`
 - perf

megfile/cli.py

@@ -321,7 +321,7 @@ def rm(path: str, recursive: bool):
     '-f',
     '--force',
     is_flag=True,
-    help='Copy files forcely, ignore same files.')
+    help='Copy files forcible, ignore same files.')
 @click.option('-q', '--quiet', is_flag=True, help='Not show any progress log.')
 @click.option('--skip', is_flag=True, help='Skip existed files.')
 def sync(
@@ -419,8 +419,8 @@ def touch(path: str):
 @cli.command(short_help='Concatenate any files and send them to stdout.')
 @click.argument('path')
 def cat(path: str):
-    with smart_open(path, 'rb') as file:
-        shutil.copyfileobj(file, sys.stdout.buffer)
+    with smart_open(path, 'rb') as f:
+        shutil.copyfileobj(f, sys.stdout.buffer)  # pytype: disable=wrong-arg-types
 
 
 @cli.command(

megfile/errors.py

@@ -1,3 +1,4 @@
+# pyre-ignore-all-errors[16]
 import time
 from contextlib import contextmanager
 from functools import wraps
@@ -78,16 +79,16 @@ def full_error_message(error):
 
 
 def client_error_code(error: ClientError) -> str:
-    error_data = error.response.get('Error', {})  # pytype: disable=attribute-error
+    error_data = error.response.get('Error', {})
     return error_data.get('Code') or error_data.get('code', 'Unknown')
 
 
 def client_error_message(error: ClientError) -> str:
-    return error.response.get('Error', {}).get('Message', 'Unknown')  # pytype: disable=attribute-error
+    return error.response.get('Error', {}).get('Message', 'Unknown')
 
 
 def param_validation_error_report(error: ParamValidationError) -> str:
-    return error.kwargs.get('report', 'Unknown')  # pytype: disable=attribute-error
+    return error.kwargs.get('report', 'Unknown')
 
 
 s3_retry_exceptions = [
@@ -106,12 +107,13 @@ def param_validation_error_report(error: ParamValidationError) -> str:
 ]
 if hasattr(botocore.exceptions,
            'ResponseStreamingError'):  # backport botocore==1.23.24
-    s3_retry_exceptions.append(botocore.exceptions.ResponseStreamingError)
-s3_retry_exceptions = tuple(s3_retry_exceptions)
+    s3_retry_exceptions.append(
+        botocore.exceptions.ResponseStreamingError)  # pyre-ignore[6]
+s3_retry_exceptions = tuple(s3_retry_exceptions)  # pyre-ignore[9]
 
 
 def s3_should_retry(error: Exception) -> bool:
-    if isinstance(error, s3_retry_exceptions):
+    if isinstance(error, s3_retry_exceptions):  # pyre-ignore[6]
         return True
     if isinstance(error, botocore.exceptions.ClientError):
         return client_error_code(error) in (
@@ -170,12 +172,12 @@ def _create_missing_ok_generator(generator, missing_ok: bool, error: Exception):
         yield from generator
         return
 
-    zero_elum = True
+    zero_elem = True
     for item in generator:
-        zero_elum = False
+        zero_elem = False
         yield item
 
-    if zero_elum:
+    if zero_elem:
         raise error
 
 
@@ -319,7 +321,7 @@ class ProtocolNotFoundError(Exception):
     pass
 
 
-def translate_fs_error(fs_error: Exception, fs_path: PathLike):
+def translate_fs_error(fs_error: Exception, fs_path: PathLike) -> Exception:
     if isinstance(fs_error, OSError):
         if fs_error.filename is None:
             fs_error.filename = fs_path
@@ -371,8 +373,7 @@ def translate_s3_error(s3_error: Exception, s3_url: PathLike) -> Exception:
     return S3UnknownError(s3_error, s3_url)
 
 
-def translate_http_error(
-        http_error: Optional[Exception], http_url: str) -> Exception:
+def translate_http_error(http_error: Exception, http_url: str) -> Exception:
     '''Generate exception according to http_error and status_code
 
     .. note ::
@@ -407,20 +408,23 @@ def s3_error_code_should_retry(error: str) -> bool:
     return False
 
 
-def translate_hdfs_error(hdfs_error: Exception, hdfs_path: PathLike):
+def translate_hdfs_error(
+        hdfs_error: Exception, hdfs_path: PathLike) -> Exception:
     from megfile.lib.hdfs_tools import hdfs_api
 
+    # pytype: disable=attribute-error
     if hdfs_api and isinstance(hdfs_error, hdfs_api.HdfsError):
-        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:  # pytype: disable=attribute-error
+        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:
             return IsADirectoryError('Is a directory: %r' % hdfs_path)
-        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:  # pytype: disable=attribute-error
+        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:
             return NotADirectoryError('Not a directory: %r' % hdfs_path)
-        elif hdfs_error.status_code in (401, 403):  # pytype: disable=attribute-error
+        elif hdfs_error.status_code in (401, 403):
             return PermissionError('Permission denied: %r' % hdfs_path)
-        elif hdfs_error.status_code == 400:  # pytype: disable=attribute-error
-            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')  # pytype: disable=attribute-error
-        elif hdfs_error.status_code == 404:  # pytype: disable=attribute-error
+        elif hdfs_error.status_code == 400:
+            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')
+        elif hdfs_error.status_code == 404:
             return FileNotFoundError(f'No match file: {hdfs_path}')
+    # pytype: enable=attribute-error
     return hdfs_error
 
 

megfile/fs.py

@@ -161,10 +161,10 @@ def fs_isfile(path: PathLike, followlinks: bool = False) -> bool:
 
 def fs_listdir(path: PathLike) -> List[str]:
     '''
-    Get all contents of given fs path. The result is in acsending alphabetical order.
+    Get all contents of given fs path. The result is in ascending alphabetical order.
 
     :param path: Given path
-    :returns: All contents have in the path in acsending alphabetical order
+    :returns: All contents have in the path in ascending alphabetical order
     '''
     return FSPath(path).listdir()
 
@@ -353,7 +353,7 @@ def fs_sync(
     :param src_path: Given path
     :param dst_path: Target file path
     :param followlinks: False if regard symlink as file, else True
-    :param force: Sync file forcely, do not ignore same files, priority is higher than 'overwrite', default is False
+    :param force: Sync file forcible, do not ignore same files, priority is higher than 'overwrite', default is False
     :param overwrite: whether or not overwrite file when exists, default is True
     '''
     return FSPath(src_path).sync(dst_path, followlinks, force, overwrite)
@@ -364,7 +364,7 @@ def fs_symlink(src_path: PathLike, dst_path: PathLike) -> None:
     Create a symbolic link pointing to src_path named dst_path.
 
     :param src_path: Given path
-    :param dst_path: Desination path
+    :param dst_path: Destination path
     '''
     return FSPath(src_path).symlink(dst_path)