From c6b89399ae488d0bb74f6e3737ff357860729f13 Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Wed, 8 Feb 2023 16:22:32 +0400 Subject: [PATCH 1/4] ini file API_URL set hidden, ConfigEntity.LOGGING_LEVEL set Literal [logging levels] --- .../lib/app/interface/base_interface.py | 8 -- .../lib/app/interface/cli_interface.py | 1 - src/superannotate/lib/core/entities/base.py | 79 ++++++++++--------- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 5b2f60198..3b66ca13a 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -108,14 +108,6 @@ def _retrieve_configs_from_env() -> typing.Union[ConfigEntity, None]: config.VERIFY_SSL = verify_ssl return config - @property - def host(self): - return self._host - - @property - def token(self): - return self._token - class Tracker: def get_mp_instance(self) -> Mixpanel: diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index dde5036d4..cae13d740 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -55,7 +55,6 @@ def init(token: str): config_parser["DEFAULT"] = { "API_TOKEN": token, - "API_URL": constances.BACKEND_URL, "LOGGING_LEVEL": "INFO", "LOGGING_PATH": constances.LOG_FILE_LOCATION, } diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 863193a32..f31583b11 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -20,6 +20,7 @@ from pydantic.utils import ROOT_KEY from pydantic.utils import sequence_like from pydantic.utils import ValueItems +from typing_extensions import Literal DATE_TIME_FORMAT_ERROR_MESSAGE = ( "does not match expected format YYYY-MM-DDTHH:MM:SS.fffZ" @@ -41,14 +42,14 @@ class BaseModel(PydanticBaseModel): """ def _iter( - self, - to_dict: bool = False, - by_alias: bool = False, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, + self, + to_dict: bool = False, + by_alias: bool = False, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, ) -> "TupleGenerator": # noqa # Merge field set excludes with explicit exclude parameter with explicit overriding field set options. @@ -63,7 +64,7 @@ def _iter( include=include, exclude=exclude, exclude_unset=exclude_unset # type: ignore ) if allowed_keys is None and not ( - by_alias or exclude_unset or exclude_defaults or exclude_none + by_alias or exclude_unset or exclude_defaults or exclude_none ): # huge boost for plain _iter() yield from self.__dict__.items() @@ -74,15 +75,15 @@ def _iter( for field_key, v in self.__dict__.items(): if (allowed_keys is not None and field_key not in allowed_keys) or ( - exclude_none and v is None + exclude_none and v is None ): continue if exclude_defaults: model_field = self.__fields__.get(field_key) if ( - not getattr(model_field, "required", True) - and getattr(model_field, "default", _missing) == v + not getattr(model_field, "required", True) + and getattr(model_field, "default", _missing) == v ): continue @@ -107,15 +108,15 @@ def _iter( @classmethod @no_type_check def _get_value( - cls, - v: Any, - to_dict: bool, - by_alias: bool, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], - exclude_unset: bool, - exclude_defaults: bool, - exclude_none: bool, + cls, + v: Any, + to_dict: bool, + by_alias: bool, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], + exclude_unset: bool, + exclude_defaults: bool, + exclude_none: bool, ) -> Any: if isinstance(v, PydanticBaseModel): @@ -147,7 +148,7 @@ def _get_value( ) for k_, v_ in v.items() if (not value_exclude or not value_exclude.is_excluded(k_)) - and (not value_include or value_include.is_included(k_)) + and (not value_include or value_include.is_included(k_)) } elif sequence_like(v): @@ -164,7 +165,7 @@ def _get_value( ) for i, v_ in enumerate(v) if (not value_exclude or not value_exclude.is_excluded(i)) - and (not value_include or value_include.is_included(i)) + and (not value_include or value_include.is_included(i)) ) return ( @@ -173,9 +174,9 @@ def _get_value( else v.__class__(seq_args) ) elif ( - isinstance(v, BaseTitledEnum) - and getattr(cls.Config, "use_enum_names", False) - and to_dict + isinstance(v, BaseTitledEnum) + and getattr(cls.Config, "use_enum_names", False) + and to_dict ): return v.name elif isinstance(v, Enum) and getattr(cls.Config, "use_enum_values", False): @@ -184,18 +185,18 @@ def _get_value( return v def json( - self, - *, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - by_alias: bool = False, - skip_defaults: Optional[bool] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - encoder: Optional[Callable[[Any], Any]] = None, - models_as_dict: bool = True, - **dumps_kwargs: Any, + self, + *, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + by_alias: bool = False, + skip_defaults: Optional[bool] = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + encoder: Optional[Callable[[Any], Any]] = None, + models_as_dict: bool = True, + **dumps_kwargs: Any, ) -> str: """ Generate a JSON representation of the model, `include` and `exclude` arguments as per `dict()`. @@ -294,7 +295,7 @@ def map_fields(entity: dict) -> dict: class ConfigEntity(BaseModel): API_TOKEN: str API_URL: str = BACKEND_URL - LOGGING_LEVEL: str = "INFO" + LOGGING_LEVEL: Literal["INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"] = "INFO" LOGGING_PATH: str = f"{LOG_FILE_LOCATION}" VERIFY_SSL: bool = True ANNOTATION_CHUNK_SIZE = 5000 From 995d9a8d473b967c2afe118ac22338eb5ee9505e Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Wed, 8 Feb 2023 17:10:57 +0400 Subject: [PATCH 2/4] fix CLI init test --- src/superannotate/lib/core/entities/base.py | 76 ++++++++++----------- tests/integration/test_cli.py | 1 - 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index f31583b11..3575bd443 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -42,14 +42,14 @@ class BaseModel(PydanticBaseModel): """ def _iter( - self, - to_dict: bool = False, - by_alias: bool = False, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, + self, + to_dict: bool = False, + by_alias: bool = False, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, ) -> "TupleGenerator": # noqa # Merge field set excludes with explicit exclude parameter with explicit overriding field set options. @@ -64,7 +64,7 @@ def _iter( include=include, exclude=exclude, exclude_unset=exclude_unset # type: ignore ) if allowed_keys is None and not ( - by_alias or exclude_unset or exclude_defaults or exclude_none + by_alias or exclude_unset or exclude_defaults or exclude_none ): # huge boost for plain _iter() yield from self.__dict__.items() @@ -75,15 +75,15 @@ def _iter( for field_key, v in self.__dict__.items(): if (allowed_keys is not None and field_key not in allowed_keys) or ( - exclude_none and v is None + exclude_none and v is None ): continue if exclude_defaults: model_field = self.__fields__.get(field_key) if ( - not getattr(model_field, "required", True) - and getattr(model_field, "default", _missing) == v + not getattr(model_field, "required", True) + and getattr(model_field, "default", _missing) == v ): continue @@ -108,15 +108,15 @@ def _iter( @classmethod @no_type_check def _get_value( - cls, - v: Any, - to_dict: bool, - by_alias: bool, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], - exclude_unset: bool, - exclude_defaults: bool, - exclude_none: bool, + cls, + v: Any, + to_dict: bool, + by_alias: bool, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]], + exclude_unset: bool, + exclude_defaults: bool, + exclude_none: bool, ) -> Any: if isinstance(v, PydanticBaseModel): @@ -148,7 +148,7 @@ def _get_value( ) for k_, v_ in v.items() if (not value_exclude or not value_exclude.is_excluded(k_)) - and (not value_include or value_include.is_included(k_)) + and (not value_include or value_include.is_included(k_)) } elif sequence_like(v): @@ -165,7 +165,7 @@ def _get_value( ) for i, v_ in enumerate(v) if (not value_exclude or not value_exclude.is_excluded(i)) - and (not value_include or value_include.is_included(i)) + and (not value_include or value_include.is_included(i)) ) return ( @@ -174,9 +174,9 @@ def _get_value( else v.__class__(seq_args) ) elif ( - isinstance(v, BaseTitledEnum) - and getattr(cls.Config, "use_enum_names", False) - and to_dict + isinstance(v, BaseTitledEnum) + and getattr(cls.Config, "use_enum_names", False) + and to_dict ): return v.name elif isinstance(v, Enum) and getattr(cls.Config, "use_enum_values", False): @@ -185,18 +185,18 @@ def _get_value( return v def json( - self, - *, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - by_alias: bool = False, - skip_defaults: Optional[bool] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - encoder: Optional[Callable[[Any], Any]] = None, - models_as_dict: bool = True, - **dumps_kwargs: Any, + self, + *, + include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, + by_alias: bool = False, + skip_defaults: Optional[bool] = None, + exclude_unset: bool = False, + exclude_defaults: bool = False, + exclude_none: bool = False, + encoder: Optional[Callable[[Any], Any]] = None, + models_as_dict: bool = True, + **dumps_kwargs: Any, ) -> str: """ Generate a JSON representation of the model, `include` and `exclude` arguments as per `dict()`. diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 6c18f24ce..0041b97d6 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -216,6 +216,5 @@ def test_init(self): config = ConfigParser() config.read(config_ini_path) assert config["DEFAULT"]["API_TOKEN"] == _token - assert config["DEFAULT"]["API_URL"] == constants.BACKEND_URL assert config["DEFAULT"]["LOGGING_LEVEL"] == "INFO" assert config["DEFAULT"]["LOGGING_PATH"] == f"{constants.LOG_FILE_LOCATION}" From e10e7e90daefe67fae4d09a6140192235d9e76ab Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Thu, 9 Feb 2023 10:25:55 +0400 Subject: [PATCH 3/4] fix config entity logging levels --- src/superannotate/lib/core/entities/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index 3575bd443..b4bb266df 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -295,7 +295,9 @@ def map_fields(entity: dict) -> dict: class ConfigEntity(BaseModel): API_TOKEN: str API_URL: str = BACKEND_URL - LOGGING_LEVEL: Literal["INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"] = "INFO" + LOGGING_LEVEL: Literal[ + "NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" + ] = "INFO" LOGGING_PATH: str = f"{LOG_FILE_LOCATION}" VERIFY_SSL: bool = True ANNOTATION_CHUNK_SIZE = 5000 From 99abd711e51e293056258198705541229c43341a Mon Sep 17 00:00:00 2001 From: Narek Mkhitaryan Date: Thu, 9 Feb 2023 15:11:29 +0400 Subject: [PATCH 4/4] config.ini token config rename --- src/superannotate/lib/app/interface/base_interface.py | 6 +++--- src/superannotate/lib/app/interface/cli_interface.py | 2 +- src/superannotate/lib/core/entities/base.py | 7 +++++-- tests/integration/test_cli.py | 2 +- tests/unit/test_init.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/superannotate/lib/app/interface/base_interface.py b/src/superannotate/lib/app/interface/base_interface.py index 3b66ca13a..de70a5764 100644 --- a/src/superannotate/lib/app/interface/base_interface.py +++ b/src/superannotate/lib/app/interface/base_interface.py @@ -28,7 +28,7 @@ class BaseInterfaceFacade: def __init__(self, token: str = None, config_path: str = None): if token: - config = ConfigEntity(API_TOKEN=token) + config = ConfigEntity(SA_TOKEN=token) elif config_path: config_path = Path(config_path) if not Path(config_path).is_file() or not os.access(config_path, os.R_OK): @@ -70,7 +70,7 @@ def _retrieve_configs_from_json(path: Path) -> typing.Union[ConfigEntity]: with open(path) as json_file: json_data = json.load(json_file) token = json_data["token"] - config = ConfigEntity(API_TOKEN=token) + config = ConfigEntity(SA_TOKEN=token) host = json_data.get("main_endpoint") verify_ssl = json_data.get("ssl_verify") if host: @@ -99,7 +99,7 @@ def _retrieve_configs_from_env() -> typing.Union[ConfigEntity, None]: token = os.environ.get("SA_TOKEN") if not token: return None - config = ConfigEntity(API_TOKEN=token) + config = ConfigEntity(**dict(os.environ)) host = os.environ.get("SA_URL") verify_ssl = not os.environ.get("SA_SSL", "True").lower() in ("false", "f", "0") if host: diff --git a/src/superannotate/lib/app/interface/cli_interface.py b/src/superannotate/lib/app/interface/cli_interface.py index cae13d740..487df666a 100644 --- a/src/superannotate/lib/app/interface/cli_interface.py +++ b/src/superannotate/lib/app/interface/cli_interface.py @@ -54,7 +54,7 @@ def init(token: str): config_parser.optionxform = str config_parser["DEFAULT"] = { - "API_TOKEN": token, + "SA_TOKEN": token, "LOGGING_LEVEL": "INFO", "LOGGING_PATH": constances.LOG_FILE_LOCATION, } diff --git a/src/superannotate/lib/core/entities/base.py b/src/superannotate/lib/core/entities/base.py index b4bb266df..9825a0f48 100644 --- a/src/superannotate/lib/core/entities/base.py +++ b/src/superannotate/lib/core/entities/base.py @@ -293,8 +293,8 @@ def map_fields(entity: dict) -> dict: class ConfigEntity(BaseModel): - API_TOKEN: str - API_URL: str = BACKEND_URL + API_TOKEN: str = Field(alias="SA_TOKEN") + API_URL: str = Field(alias="SA_URL", default=BACKEND_URL) LOGGING_LEVEL: Literal[ "NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" ] = "INFO" @@ -304,3 +304,6 @@ class ConfigEntity(BaseModel): ITEM_CHUNK_SIZE = 2000 MAX_THREAD_COUNT = 4 MAX_COROUTINE_COUNT = 8 + + class Config: + extra = Extra.ignore diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 0041b97d6..e44f6ea53 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -215,6 +215,6 @@ def test_init(self): self.safe_run(self._cli.init, _token) config = ConfigParser() config.read(config_ini_path) - assert config["DEFAULT"]["API_TOKEN"] == _token + assert config["DEFAULT"]["SA_TOKEN"] == _token assert config["DEFAULT"]["LOGGING_LEVEL"] == "INFO" assert config["DEFAULT"]["LOGGING_PATH"] == f"{constants.LOG_FILE_LOCATION}" diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index fa5892bd9..f6f81f694 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -62,7 +62,7 @@ def test_init_via_config_ini(self, get_team_use_case): config_parser = ConfigParser() config_parser.optionxform = str config_parser["DEFAULT"] = { - "API_TOKEN": self._token, + "SA_TOKEN": self._token, "LOGGING_LEVEL": "DEBUG", } config_parser.write(config_ini)