refactor: unify temp path to dot cache jina (#5058)

* refactor: unify temp path to dot cache jina

* refactor: unify temp path to dot cache jina

* refactor: move the executor's secret.key to .cache/jina/secrets/ path

* style: remove the print log

* refactor: use the inode of the executor file path as the path to the executor's secret.key

* refactor: rename parameter names and update doc of function

* fix: fix the test push in hubbio

Co-authored-by: floralatin <dragondao@hotmail.com>

jina/__init__.py

@@ -13,6 +13,7 @@
 import signal as _signal
 import sys as _sys
 import warnings as _warnings
+from pathlib import Path as _Path
 
 import docarray as _docarray
 
@@ -107,7 +108,7 @@ def _warning_on_one_line(message, category, filename, lineno, *args, **kwargs):
     'JINA_RANDOM_PORT_MIN',
     'JINA_DISABLE_HEALTHCHECK_LOGS',
     'JINA_LOCKS_ROOT',
-    'JINA_OPTOUT_TELEMETRY'
+    'JINA_OPTOUT_TELEMETRY',
 )
 
 __default_host__ = _os.environ.get(
@@ -128,6 +129,9 @@ def _warning_on_one_line(message, category, filename, lineno, *args, **kwargs):
 __resources_path__ = _os.path.join(
     _os.path.dirname(_sys.modules['jina'].__file__), 'resources'
 )
+__cache_path__ = f'{_os.path.expanduser("~")}/.cache/{__package__}'
+if not _Path(__cache_path__).exists():
+    _Path(__cache_path__).mkdir(parents=True, exist_ok=True)
 
 _names_with_underscore = [
     '__version__',

jina/hubble/helper.py

@@ -17,22 +17,22 @@
 from typing import Dict, Optional, Tuple
 from urllib.parse import urljoin, urlparse
 
-from jina import __resources_path__
+from jina import __cache_path__, __resources_path__
 from jina.enums import BetterEnum
 from jina.helper import get_request_header as _get_request_header_main
-from jina.importer import ImportExtensions
-from jina.logging.predefined import default_logger
 from jina.hubble.requirements import (
-    get_env_variables,
     check_env_variable,
+    expand_env_variables,
+    get_env_variables,
     parse_requirement,
-    expand_env_variables
 )
+from jina.importer import ImportExtensions
+from jina.logging.predefined import default_logger
 
 
 @lru_cache()
 def _get_hub_root() -> Path:
-    hub_root = Path(os.environ.get('JINA_HUB_ROOT', Path.home().joinpath('.jina')))
+    hub_root = Path(os.environ.get('JINA_HUB_ROOT', __cache_path__))
 
     if not hub_root.exists():
         hub_root.mkdir(parents=True, exist_ok=True)
@@ -490,6 +490,7 @@ def is_requirements_installed(
         return isinstance(ex, VersionConflict)
     return True
 
+
 def get_requirements_env_variables(requirements_file: 'Path') -> list:
     """get the env variables in requirements.txt
     :param requirements_file: the requirements.txt file
@@ -507,15 +508,17 @@ def get_requirements_env_variables(requirements_file: 'Path') -> list:
 
     return env_variables
 
+
 def check_requirements_env_variable(env_variable: str) -> bool:
-    """ 
+    """
     check the environment variables is limited
     to uppercase letter and number and the `_` (underscore).
     :param env_variable: env_variable in the requirements.txt file
     :return: True or False if not satisfied
     """
     return check_env_variable(env_variable)
 
+
 def replace_requirements_env_variables(requirements_file: 'Path') -> list:
     """replace the environment variables in requirements.txt
     :param requirements_file: the requirements.txt file
@@ -531,7 +534,7 @@ def replace_requirements_env_variables(requirements_file: 'Path') -> list:
                 line = expand_env_variables(line)
                 env_variables.append(line)
     return env_variables
-    
+
 
 def _get_install_options(requirements_file: 'Path', excludes: Tuple[str] = ('jina',)):
     with requirements_file.open() as requirements:
@@ -675,4 +678,4 @@ def get_hubble_error_message(hubble_structured_error: dict) -> Tuple[str, str]:
     https://github.com/jina-ai/executor-normalizer
 '''
 
-    return (msg, original_msg)
+    return (msg, original_msg)

jina/hubble/hubapi.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 from typing import Tuple
 
+from jina import __cache_path__
 from jina.helper import random_identity
 from jina.hubble import HubExecutor
 from jina.hubble.helper import (
@@ -14,6 +15,10 @@
     unpack_package,
 )
 
+import os
+
+SECRET_PATH = 'secrets'
+
 
 def get_dist_path(uuid: str, tag: str) -> Tuple[Path, Path]:
     """Get the package path according ID and TAG
@@ -57,19 +62,35 @@ def get_lockfile() -> str:
     """
     return str(get_hub_packages_dir() / 'LOCK')
 
+def get_secret_path(inode: str) -> 'Path': 
+    """Get the path of secrets
+    :param inode: the inode of the executor path
+    :return: the path of secrets
+    """
+
+    pre_path = Path(f'{__cache_path__}/{SECRET_PATH}/{inode}')
+
+    return pre_path
+
 
 def load_secret(work_path: 'Path') -> Tuple[str, str]:
     """Get the UUID and Secret from local
-
-    :param work_path: the local package directory
+    :param work_path: the work path of the executor
     :return: the UUID and secret
     """
+
     from cryptography.fernet import Fernet
 
-    config = work_path / '.jina'
+    preConfig = work_path / '.jina'
+    config = get_secret_path(os.stat(work_path).st_ino)
     config.mkdir(parents=True, exist_ok=True)
 
     local_id_file = config / 'secret.key'
+    pre_secret_file = preConfig / 'secret.key'
+
+    if pre_secret_file.exists() and not local_id_file.exists():
+        shutil.copyfile(pre_secret_file, local_id_file)
+
     uuid8 = None
     secret = None
     if local_id_file.exists():
@@ -90,14 +111,13 @@ def load_secret(work_path: 'Path') -> Tuple[str, str]:
 
 def dump_secret(work_path: 'Path', uuid8: str, secret: str):
     """Dump the UUID and Secret into local file
-
-    :param work_path: the local package directory
+    :param work_path: the work path of the executor
     :param uuid8: the ID of the executor
     :param secret: the access secret
     """
     from cryptography.fernet import Fernet
 
-    config = work_path / '.jina'
+    config = get_secret_path(os.stat(work_path).st_ino);
     config.mkdir(parents=True, exist_ok=True)
 
     local_id_file = config / 'secret.key'

jina/hubble/hubio.py

@@ -14,17 +14,17 @@
 from jina.hubble import HubExecutor
 from jina.hubble.helper import (
     archive_package,
+    check_requirements_env_variable,
     disk_cache_offline,
     download_with_resume,
     get_cache_db,
     get_download_cache_dir,
     get_hubble_error_message,
     get_hubble_url_v2,
     get_request_header,
+    get_requirements_env_variables,
     parse_hub_uri,
     upload_file,
-    get_requirements_env_variables,
-    check_requirements_env_variable
 )
 from jina.hubble.hubapi import (
     dump_secret,
@@ -347,43 +347,53 @@ def push(self) -> None:
                 )
 
             dockerfile = dockerfile.relative_to(work_path)
-        
+
         build_env = None
         if self.args.build_env:
             build_envs = self.args.build_env.strip().split()
             build_env_dict = {}
             for index, env in enumerate(build_envs):
                 env_list = env.split('=')
-                if (len(env_list) != 2):
-                    raise Exception( f'The --build-env parameter: `{env}` is wrong format. you can use: `--build-env {env}=YOUR_VALUE`.')
+                if len(env_list) != 2:
+                    raise Exception(
+                        f'The --build-env parameter: `{env}` is wrong format. you can use: `--build-env {env}=YOUR_VALUE`.'
+                    )
                 if check_requirements_env_variable(env_list[0]) is False:
-                    raise Exception( f'The --build-env parameter key:`{env_list[0]}` can only consist of uppercase letter and number and underline.')
+                    raise Exception(
+                        f'The --build-env parameter key:`{env_list[0]}` can only consist of uppercase letter and number and underline.'
+                    )
                 build_env_dict[env_list[0]] = env_list[1]
-            build_env = build_env_dict if len(list(build_env_dict.keys()))>0 else None
+            build_env = build_env_dict if len(list(build_env_dict.keys())) > 0 else None
 
         requirements_file = work_path / 'requirements.txt'
 
         requirements_env_variables = []
-        if requirements_file.exists(): 
-            requirements_env_variables = get_requirements_env_variables(requirements_file)
+        if requirements_file.exists():
+            requirements_env_variables = get_requirements_env_variables(
+                requirements_file
+            )
             for index, env in enumerate(requirements_env_variables):
                 if check_requirements_env_variable(env) is False:
-                   raise Exception( f'The requirements.txt environment variables:`${env}` can only consist of uppercase letter and number and underline.')
+                    raise Exception(
+                        f'The requirements.txt environment variables:`${env}` can only consist of uppercase letter and number and underline.'
+                    )
 
         if len(requirements_env_variables) and not build_env:
-            env_variables_str = ','.join(requirements_env_variables);
-            error_str= f'The requirements.txt set environment variables as follows:`{env_variables_str}` should use `--build-env';
+            env_variables_str = ','.join(requirements_env_variables)
+            error_str = f'The requirements.txt set environment variables as follows:`{env_variables_str}` should use `--build-env'
             for item in requirements_env_variables:
-                error_str+= f' {item}=YOUR_VALUE'
+                error_str += f' {item}=YOUR_VALUE'
             raise Exception(f'{error_str}` to add it.')
         elif len(requirements_env_variables) and build_env:
             build_env_keys = list(build_env.keys())
-            diff_env_variables = list(set(requirements_env_variables).difference(set(build_env_keys)))
+            diff_env_variables = list(
+                set(requirements_env_variables).difference(set(build_env_keys))
+            )
             if len(diff_env_variables):
                 diff_env_variables_str = ",".join(diff_env_variables)
-                error_str= f'The requirements.txt set environment variables as follows:`{diff_env_variables_str}` should use `--build-env';
+                error_str = f'The requirements.txt set environment variables as follows:`{diff_env_variables_str}` should use `--build-env'
                 for item in diff_env_variables:
-                    error_str+= f' {item}=YOUR_VALUE'
+                    error_str += f' {item}=YOUR_VALUE'
                 raise Exception(f'{error_str}` to add it.')
 
         console = get_rich_console()
@@ -421,9 +431,9 @@ def push(self) -> None:
                 if dockerfile:
                     form_data['dockerfile'] = str(dockerfile)
 
-                if build_env: 
+                if build_env:
                     form_data['buildEnv'] = json.dumps(build_env)
-
+                
                 uuid8, secret = load_secret(work_path)
                 if self.args.force_update or uuid8:
                     form_data['id'] = self.args.force_update or uuid8
@@ -606,7 +616,6 @@ def _get_prettyprint_usage(self, console, executor_name, usage_kind=None):
 
         console.print(Panel(param_str, title='Usage', expand=False, width=100))
 
-
     def _prettyprint_build_env_usage(self, console, build_env, usage_kind=None):
         from rich import box
         from rich.panel import Panel
@@ -620,12 +629,17 @@ def _prettyprint_build_env_usage(self, console, build_env, usage_kind=None):
         param_str.add_column('Your value')
 
         for index, item in enumerate(build_env):
-            param_str.add_row(
-                f'{item}',
-                'your value'
-            )
+            param_str.add_row(f'{item}', 'your value')
 
-        console.print(Panel(param_str, title='build_env', subtitle='You have to set the above environment variables', expand=False, width=100))
+        console.print(
+            Panel(
+                param_str,
+                title='build_env',
+                subtitle='You have to set the above environment variables',
+                expand=False,
+                width=100,
+            )
+        )
 
     @staticmethod
     @disk_cache_offline(cache_file=str(get_cache_db()))
@@ -698,7 +712,7 @@ def _send_request_with_retry(url, **kwargs):
             image_name=image_name,
             archive_url=resp['package']['download'],
             md5sum=resp['package']['md5'],
-            build_env=buildEnv.keys() if buildEnv else []
+            build_env=buildEnv.keys() if buildEnv else [],
         )
 
     @staticmethod
@@ -854,7 +868,7 @@ def pull(self) -> str:
                 )
 
                 build_env = executor.build_env
-   
+
                 presented_id = executor.name if executor.name else executor.uuid
                 executor_name = (
                     f'{presented_id}'
@@ -887,8 +901,8 @@ def pull(self) -> str:
                     import filelock
 
                     if build_env:
-                       self._prettyprint_build_env_usage(console,build_env)
-                   
+                        self._prettyprint_build_env_usage(console, build_env)
+
                     with filelock.FileLock(get_lockfile(), timeout=-1):
                         try:
                             pkg_path, pkg_dist_path = get_dist_path_of_executor(

jina/orchestrate/helper.py

@@ -1,6 +1,8 @@
 import os
 from pathlib import Path
 
+from jina import __cache_path__
+
 
 def generate_default_volume_and_workspace(workspace_id=''):
     """automatically generate a docker volume, and an Executor workspace inside it
@@ -17,7 +19,7 @@ def generate_default_volume_and_workspace(workspace_id=''):
             path=os.path.abspath(default_workspace), start=Path.home()
         )
     else:  # fallback if no custom volume and no default workspace
-        workspace = os.path.join('.jina', 'executor-workspace')
+        workspace = os.path.join(__cache_path__, 'executor-workspace')
         host_addr = os.path.join(
             Path.home(),
             workspace,

jina/serve/executors/decorators.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Union
 
+from jina import __cache_path__
 from jina.helper import convert_tuple_to_list, iscoroutinefunction
 from jina.importer import ImportExtensions
 from jina.serve.executors.metas import get_default_metas
@@ -17,9 +18,7 @@
 @functools.lru_cache()
 def _get_locks_root() -> Path:
     locks_root = Path(
-        os.environ.get(
-            'JINA_LOCKS_ROOT', Path.home().joinpath('.jina').joinpath('locks')
-        )
+        os.environ.get('JINA_LOCKS_ROOT', os.path.join(__cache_path__, 'locks'))
     )
 
     if not locks_root.exists():