diff --git a/jina/__init__.py b/jina/__init__.py index a9bee58eae1b3..2211b09025191 100644 --- a/jina/__init__.py +++ b/jina/__init__.py @@ -13,6 +13,7 @@ import signal as _signal import sys as _sys import warnings as _warnings +from pathlib import Path as _Path import docarray as _docarray @@ -107,7 +108,7 @@ def _warning_on_one_line(message, category, filename, lineno, *args, **kwargs): 'JINA_RANDOM_PORT_MIN', 'JINA_DISABLE_HEALTHCHECK_LOGS', 'JINA_LOCKS_ROOT', - 'JINA_OPTOUT_TELEMETRY' + 'JINA_OPTOUT_TELEMETRY', ) __default_host__ = _os.environ.get( @@ -128,6 +129,9 @@ def _warning_on_one_line(message, category, filename, lineno, *args, **kwargs): __resources_path__ = _os.path.join( _os.path.dirname(_sys.modules['jina'].__file__), 'resources' ) +__cache_path__ = f'{_os.path.expanduser("~")}/.cache/{__package__}' +if not _Path(__cache_path__).exists(): + _Path(__cache_path__).mkdir(parents=True, exist_ok=True) _names_with_underscore = [ '__version__', diff --git a/jina/hubble/helper.py b/jina/hubble/helper.py index 28a502da487b6..9f99e30ab3837 100644 --- a/jina/hubble/helper.py +++ b/jina/hubble/helper.py @@ -17,22 +17,22 @@ from typing import Dict, Optional, Tuple from urllib.parse import urljoin, urlparse -from jina import __resources_path__ +from jina import __cache_path__, __resources_path__ from jina.enums import BetterEnum from jina.helper import get_request_header as _get_request_header_main -from jina.importer import ImportExtensions -from jina.logging.predefined import default_logger from jina.hubble.requirements import ( - get_env_variables, check_env_variable, + expand_env_variables, + get_env_variables, parse_requirement, - expand_env_variables ) +from jina.importer import ImportExtensions +from jina.logging.predefined import default_logger @lru_cache() def _get_hub_root() -> Path: - hub_root = Path(os.environ.get('JINA_HUB_ROOT', Path.home().joinpath('.jina'))) + hub_root = Path(os.environ.get('JINA_HUB_ROOT', __cache_path__)) if not hub_root.exists(): hub_root.mkdir(parents=True, exist_ok=True) @@ -490,6 +490,7 @@ def is_requirements_installed( return isinstance(ex, VersionConflict) return True + def get_requirements_env_variables(requirements_file: 'Path') -> list: """get the env variables in requirements.txt :param requirements_file: the requirements.txt file @@ -507,8 +508,9 @@ def get_requirements_env_variables(requirements_file: 'Path') -> list: return env_variables + def check_requirements_env_variable(env_variable: str) -> bool: - """ + """ check the environment variables is limited to uppercase letter and number and the `_` (underscore). :param env_variable: env_variable in the requirements.txt file @@ -516,6 +518,7 @@ def check_requirements_env_variable(env_variable: str) -> bool: """ return check_env_variable(env_variable) + def replace_requirements_env_variables(requirements_file: 'Path') -> list: """replace the environment variables in requirements.txt :param requirements_file: the requirements.txt file @@ -531,7 +534,7 @@ def replace_requirements_env_variables(requirements_file: 'Path') -> list: line = expand_env_variables(line) env_variables.append(line) return env_variables - + def _get_install_options(requirements_file: 'Path', excludes: Tuple[str] = ('jina',)): with requirements_file.open() as requirements: @@ -675,4 +678,4 @@ def get_hubble_error_message(hubble_structured_error: dict) -> Tuple[str, str]: https://github.com/jina-ai/executor-normalizer ''' - return (msg, original_msg) \ No newline at end of file + return (msg, original_msg) diff --git a/jina/hubble/hubapi.py b/jina/hubble/hubapi.py index d908452a7bd43..bbbc011b3a852 100644 --- a/jina/hubble/hubapi.py +++ b/jina/hubble/hubapi.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Tuple +from jina import __cache_path__ from jina.helper import random_identity from jina.hubble import HubExecutor from jina.hubble.helper import ( @@ -14,6 +15,10 @@ unpack_package, ) +import os + +SECRET_PATH = 'secrets' + def get_dist_path(uuid: str, tag: str) -> Tuple[Path, Path]: """Get the package path according ID and TAG @@ -57,19 +62,35 @@ def get_lockfile() -> str: """ return str(get_hub_packages_dir() / 'LOCK') +def get_secret_path(inode: str) -> 'Path': + """Get the path of secrets + :param inode: the inode of the executor path + :return: the path of secrets + """ + + pre_path = Path(f'{__cache_path__}/{SECRET_PATH}/{inode}') + + return pre_path + def load_secret(work_path: 'Path') -> Tuple[str, str]: """Get the UUID and Secret from local - - :param work_path: the local package directory + :param work_path: the work path of the executor :return: the UUID and secret """ + from cryptography.fernet import Fernet - config = work_path / '.jina' + preConfig = work_path / '.jina' + config = get_secret_path(os.stat(work_path).st_ino) config.mkdir(parents=True, exist_ok=True) local_id_file = config / 'secret.key' + pre_secret_file = preConfig / 'secret.key' + + if pre_secret_file.exists() and not local_id_file.exists(): + shutil.copyfile(pre_secret_file, local_id_file) + uuid8 = None secret = None if local_id_file.exists(): @@ -90,14 +111,13 @@ def load_secret(work_path: 'Path') -> Tuple[str, str]: def dump_secret(work_path: 'Path', uuid8: str, secret: str): """Dump the UUID and Secret into local file - - :param work_path: the local package directory + :param work_path: the work path of the executor :param uuid8: the ID of the executor :param secret: the access secret """ from cryptography.fernet import Fernet - config = work_path / '.jina' + config = get_secret_path(os.stat(work_path).st_ino); config.mkdir(parents=True, exist_ok=True) local_id_file = config / 'secret.key' diff --git a/jina/hubble/hubio.py b/jina/hubble/hubio.py index e47d10acfa41d..5e880b4354d48 100644 --- a/jina/hubble/hubio.py +++ b/jina/hubble/hubio.py @@ -14,6 +14,7 @@ from jina.hubble import HubExecutor from jina.hubble.helper import ( archive_package, + check_requirements_env_variable, disk_cache_offline, download_with_resume, get_cache_db, @@ -21,10 +22,9 @@ get_hubble_error_message, get_hubble_url_v2, get_request_header, + get_requirements_env_variables, parse_hub_uri, upload_file, - get_requirements_env_variables, - check_requirements_env_variable ) from jina.hubble.hubapi import ( dump_secret, @@ -347,43 +347,53 @@ def push(self) -> None: ) dockerfile = dockerfile.relative_to(work_path) - + build_env = None if self.args.build_env: build_envs = self.args.build_env.strip().split() build_env_dict = {} for index, env in enumerate(build_envs): env_list = env.split('=') - if (len(env_list) != 2): - raise Exception( f'The --build-env parameter: `{env}` is wrong format. you can use: `--build-env {env}=YOUR_VALUE`.') + if len(env_list) != 2: + raise Exception( + f'The --build-env parameter: `{env}` is wrong format. you can use: `--build-env {env}=YOUR_VALUE`.' + ) if check_requirements_env_variable(env_list[0]) is False: - raise Exception( f'The --build-env parameter key:`{env_list[0]}` can only consist of uppercase letter and number and underline.') + raise Exception( + f'The --build-env parameter key:`{env_list[0]}` can only consist of uppercase letter and number and underline.' + ) build_env_dict[env_list[0]] = env_list[1] - build_env = build_env_dict if len(list(build_env_dict.keys()))>0 else None + build_env = build_env_dict if len(list(build_env_dict.keys())) > 0 else None requirements_file = work_path / 'requirements.txt' requirements_env_variables = [] - if requirements_file.exists(): - requirements_env_variables = get_requirements_env_variables(requirements_file) + if requirements_file.exists(): + requirements_env_variables = get_requirements_env_variables( + requirements_file + ) for index, env in enumerate(requirements_env_variables): if check_requirements_env_variable(env) is False: - raise Exception( f'The requirements.txt environment variables:`${env}` can only consist of uppercase letter and number and underline.') + raise Exception( + f'The requirements.txt environment variables:`${env}` can only consist of uppercase letter and number and underline.' + ) if len(requirements_env_variables) and not build_env: - env_variables_str = ','.join(requirements_env_variables); - error_str= f'The requirements.txt set environment variables as follows:`{env_variables_str}` should use `--build-env'; + env_variables_str = ','.join(requirements_env_variables) + error_str = f'The requirements.txt set environment variables as follows:`{env_variables_str}` should use `--build-env' for item in requirements_env_variables: - error_str+= f' {item}=YOUR_VALUE' + error_str += f' {item}=YOUR_VALUE' raise Exception(f'{error_str}` to add it.') elif len(requirements_env_variables) and build_env: build_env_keys = list(build_env.keys()) - diff_env_variables = list(set(requirements_env_variables).difference(set(build_env_keys))) + diff_env_variables = list( + set(requirements_env_variables).difference(set(build_env_keys)) + ) if len(diff_env_variables): diff_env_variables_str = ",".join(diff_env_variables) - error_str= f'The requirements.txt set environment variables as follows:`{diff_env_variables_str}` should use `--build-env'; + error_str = f'The requirements.txt set environment variables as follows:`{diff_env_variables_str}` should use `--build-env' for item in diff_env_variables: - error_str+= f' {item}=YOUR_VALUE' + error_str += f' {item}=YOUR_VALUE' raise Exception(f'{error_str}` to add it.') console = get_rich_console() @@ -421,9 +431,9 @@ def push(self) -> None: if dockerfile: form_data['dockerfile'] = str(dockerfile) - if build_env: + if build_env: form_data['buildEnv'] = json.dumps(build_env) - + uuid8, secret = load_secret(work_path) if self.args.force_update or uuid8: form_data['id'] = self.args.force_update or uuid8 @@ -606,7 +616,6 @@ def _get_prettyprint_usage(self, console, executor_name, usage_kind=None): console.print(Panel(param_str, title='Usage', expand=False, width=100)) - def _prettyprint_build_env_usage(self, console, build_env, usage_kind=None): from rich import box from rich.panel import Panel @@ -620,12 +629,17 @@ def _prettyprint_build_env_usage(self, console, build_env, usage_kind=None): param_str.add_column('Your value') for index, item in enumerate(build_env): - param_str.add_row( - f'{item}', - 'your value' - ) + param_str.add_row(f'{item}', 'your value') - console.print(Panel(param_str, title='build_env', subtitle='You have to set the above environment variables', expand=False, width=100)) + console.print( + Panel( + param_str, + title='build_env', + subtitle='You have to set the above environment variables', + expand=False, + width=100, + ) + ) @staticmethod @disk_cache_offline(cache_file=str(get_cache_db())) @@ -698,7 +712,7 @@ def _send_request_with_retry(url, **kwargs): image_name=image_name, archive_url=resp['package']['download'], md5sum=resp['package']['md5'], - build_env=buildEnv.keys() if buildEnv else [] + build_env=buildEnv.keys() if buildEnv else [], ) @staticmethod @@ -854,7 +868,7 @@ def pull(self) -> str: ) build_env = executor.build_env - + presented_id = executor.name if executor.name else executor.uuid executor_name = ( f'{presented_id}' @@ -887,8 +901,8 @@ def pull(self) -> str: import filelock if build_env: - self._prettyprint_build_env_usage(console,build_env) - + self._prettyprint_build_env_usage(console, build_env) + with filelock.FileLock(get_lockfile(), timeout=-1): try: pkg_path, pkg_dist_path = get_dist_path_of_executor( diff --git a/jina/orchestrate/helper.py b/jina/orchestrate/helper.py index 360fc7d838ee3..967cd9ad5145c 100644 --- a/jina/orchestrate/helper.py +++ b/jina/orchestrate/helper.py @@ -1,6 +1,8 @@ import os from pathlib import Path +from jina import __cache_path__ + def generate_default_volume_and_workspace(workspace_id=''): """automatically generate a docker volume, and an Executor workspace inside it @@ -17,7 +19,7 @@ def generate_default_volume_and_workspace(workspace_id=''): path=os.path.abspath(default_workspace), start=Path.home() ) else: # fallback if no custom volume and no default workspace - workspace = os.path.join('.jina', 'executor-workspace') + workspace = os.path.join(__cache_path__, 'executor-workspace') host_addr = os.path.join( Path.home(), workspace, diff --git a/jina/serve/executors/decorators.py b/jina/serve/executors/decorators.py index 7abcf79fe2374..52ebb3d226911 100644 --- a/jina/serve/executors/decorators.py +++ b/jina/serve/executors/decorators.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Union +from jina import __cache_path__ from jina.helper import convert_tuple_to_list, iscoroutinefunction from jina.importer import ImportExtensions from jina.serve.executors.metas import get_default_metas @@ -17,9 +18,7 @@ @functools.lru_cache() def _get_locks_root() -> Path: locks_root = Path( - os.environ.get( - 'JINA_LOCKS_ROOT', Path.home().joinpath('.jina').joinpath('locks') - ) + os.environ.get('JINA_LOCKS_ROOT', os.path.join(__cache_path__, 'locks')) ) if not locks_root.exists(): diff --git a/tests/unit/hubble/test_hubio.py b/tests/unit/hubble/test_hubio.py index efc5c622c0cf8..4a43dad235988 100644 --- a/tests/unit/hubble/test_hubio.py +++ b/tests/unit/hubble/test_hubio.py @@ -12,14 +12,14 @@ import pytest import requests import yaml -from jina.hubble import hubio +from jina.hubble import hubio from jina.hubble.helper import ( _get_auth_token, _get_hub_config, _get_hub_root, disk_cache_offline, - get_requirements_env_variables + get_requirements_env_variables, ) from jina.hubble.hubio import HubExecutor, HubIO from jina.parsers.hubble import ( @@ -27,6 +27,9 @@ set_hub_pull_parser, set_hub_push_parser, ) +from jina.hubble.hubapi import ( + get_secret_path +) cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -164,11 +167,10 @@ def _mock_post(url, data, headers=None, stream=True): args = set_hub_push_parser().parse_args(_args_list) - result = HubIO(args).push() - + result = HubIO(args).push() - # remove .jina - exec_config_path = os.path.join(exec_path, '.jina') + + exec_config_path = get_secret_path(os.stat(exec_path).st_ino) shutil.rmtree(exec_config_path) _, mock_kwargs = mock.call_args_list[0] @@ -187,10 +189,12 @@ def _mock_post(url, data, headers=None, stream=True): assert form_data['id'] == ['UUID8'] else: assert form_data.get('id') is None - + if build_env: print(form_data['buildEnv']) - assert form_data['buildEnv'] == ['{"DOMAIN": "github.com", "DOWNLOAD": "download"}'] + assert form_data['buildEnv'] == [ + '{"DOMAIN": "github.com", "DOWNLOAD": "download"}' + ] else: assert form_data.get('buildEnv') is None @@ -228,7 +232,14 @@ def _mock_post(url, data, headers=None, stream=True): @pytest.mark.parametrize('mode', ['--public', '--private']) @pytest.mark.parametrize('build_env', ['TEST_TOKEN_ccc=ghp_I1cCzUY', 'NO123123']) def test_push_wrong_build_env( - mocker, monkeypatch, path, mode, tmpdir, env_variable_format_error, env_variable_consist_error, build_env + mocker, + monkeypatch, + path, + mode, + tmpdir, + env_variable_format_error, + env_variable_consist_error, + build_env, ): mock = mocker.Mock() @@ -246,15 +257,19 @@ def _mock_post(url, data, headers=None, stream=True): if build_env: _args_list.extend(['--build-env', build_env]) - + args = set_hub_push_parser().parse_args(_args_list) with pytest.raises(Exception) as info: - result = HubIO(args).push() - - assert ( - env_variable_format_error.format(build_env=build_env) in str( info.value ) - or env_variable_consist_error.format(build_env_key=build_env.split('=')[0]) in str( info.value )) + result = HubIO(args).push() + + assert env_variable_format_error.format(build_env=build_env) in str( + info.value + ) or env_variable_consist_error.format( + build_env_key=build_env.split('=')[0] + ) in str( + info.value + ) @pytest.mark.parametrize( @@ -267,7 +282,13 @@ def _mock_post(url, data, headers=None, stream=True): @pytest.mark.parametrize('mode', ['--public', '--private']) @pytest.mark.parametrize('requirements_file', ['requirements.txt']) def test_push_requirements_file_require_set_env_variables( - mocker, monkeypatch, path, mode, tmpdir, requirements_file_need_build_env_error, requirements_file + mocker, + monkeypatch, + path, + mode, + tmpdir, + requirements_file_need_build_env_error, + requirements_file, ): mock = mocker.Mock() @@ -285,12 +306,17 @@ def _mock_post(url, data, headers=None, stream=True): args = set_hub_push_parser().parse_args(_args_list) - requirements_file = os.path.join(exec_path,requirements_file) - requirements_file_env_variables = get_requirements_env_variables(Path(requirements_file)) - + requirements_file = os.path.join(exec_path, requirements_file) + requirements_file_env_variables = get_requirements_env_variables( + Path(requirements_file) + ) + with pytest.raises(Exception) as info: - result = HubIO(args).push() - assert requirements_file_need_build_env_error.format(env_variables_str=','.join(requirements_file_env_variables)) in str( info.value ) + result = HubIO(args).push() + + assert requirements_file_need_build_env_error.format( + env_variables_str=','.join(requirements_file_env_variables) + ) in str(info.value) @pytest.mark.parametrize( @@ -323,14 +349,20 @@ def _mock_post(url, data, headers=None, stream=True): args = set_hub_push_parser().parse_args(_args_list) - requirements_file = os.path.join(exec_path,'requirements.txt') - requirements_file_env_variables = get_requirements_env_variables(Path(requirements_file)) - diff_env_variables = list(set(requirements_file_env_variables).difference(set([build_env]))) + requirements_file = os.path.join(exec_path, 'requirements.txt') + requirements_file_env_variables = get_requirements_env_variables( + Path(requirements_file) + ) + diff_env_variables = list( + set(requirements_file_env_variables).difference(set([build_env])) + ) with pytest.raises(Exception) as info: - result = HubIO(args).push() + result = HubIO(args).push() - assert diff_env_variables_error.format(env_variables_str=','.join(diff_env_variables)) in str( info.value ) + assert diff_env_variables_error.format( + env_variables_str=','.join(diff_env_variables) + ) in str(info.value) @pytest.mark.parametrize( @@ -365,8 +397,10 @@ def _mock_post(url, data, headers=None, stream=True): args = set_hub_push_parser().parse_args(_args_list) args.dockerfile = dockerfile + with pytest.raises(Exception) as info: HubIO(args).push() + assert expected_error.format(dockerfile=dockerfile, work_path=args.path) in str( info.value @@ -391,10 +425,6 @@ def _mock_post(url, data, headers, stream): args = set_hub_push_parser().parse_args(_args_list) HubIO(args).push() - # remove .jina - exec_config_path = os.path.join(exec_path, '.jina') - shutil.rmtree(exec_config_path) - assert mock.call_count == 1 _, kwargs = mock.call_args_list[0] @@ -514,8 +544,9 @@ def iter_content(self, buffer=32 * 1024): def status_code(self): return self.response_code + @pytest.mark.parametrize('executor_name', ['alias_dummy', None]) -@pytest.mark.parametrize('build_env', [['DOWNLOAD','DOMAIN'], None]) +@pytest.mark.parametrize('build_env', [['DOWNLOAD', 'DOMAIN'], None]) def test_pull(test_envs, mocker, monkeypatch, executor_name, build_env): mock = mocker.Mock() @@ -538,7 +569,7 @@ def _mock_fetch( md5sum=None, visibility=True, archive_url=None, - build_env=build_env + build_env=build_env, ), False, ) @@ -558,7 +589,7 @@ def _mock_head(url): monkeypatch.setattr(requests, 'get', _mock_download) monkeypatch.setattr(requests, 'head', _mock_head) - def _mock_get_prettyprint_usage(self,console, executor_name, usage_kind=None): + def _mock_get_prettyprint_usage(self, console, executor_name, usage_kind=None): mock(console=console) mock(usage_kind=usage_kind) print('_mock_get_prettyprint_usage executor_name:', executor_name) @@ -898,4 +929,3 @@ def _mock_post(url, json, headers=None): host, port = HubIO.deploy_public_sandbox(args) assert host == 'http://test_new_deployment.com' assert port == 4322 -