From 7b2e5ba3005714cb66eab737fbea0fa4ac44886b Mon Sep 17 00:00:00 2001 From: ptiurin Date: Thu, 21 Dec 2023 12:18:08 +0000 Subject: [PATCH 1/8] build: Relax pydantic version requirement --- setup.cfg | 2 +- src/firebolt/model/V1/__init__.py | 5 ++++- src/firebolt/model/V1/binding.py | 14 +++++++------- src/firebolt/model/V1/engine.py | 28 ++++++++++++++-------------- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/setup.cfg b/setup.cfg index a6ff1afbf1..1b5ea5933c 100755 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ install_requires = cryptography>=3.4.0 httpcore<0.17.3 httpx[http2]==0.24.0 - pydantic[dotenv]>=1.8.2,<2.0 + pydantic[dotenv]>=1.8.2 python-dateutil>=2.8.2 readerwriterlock>=1.0.9 sqlparse>=0.4.2 diff --git a/src/firebolt/model/V1/__init__.py b/src/firebolt/model/V1/__init__.py index 83992de4b6..1a1114c232 100644 --- a/src/firebolt/model/V1/__init__.py +++ b/src/firebolt/model/V1/__init__.py @@ -5,9 +5,12 @@ class FireboltBaseModel(BaseModel): + + # Using Pydantic 1.* config class for backwards compatibility class Config: - allow_population_by_field_name = True extra = "forbid" + allow_population_by_field_name = True # Pydantic 1.8 + populate_by_name = True # Pydantic 2.0 def jsonable_dict(self, *args: Any, **kwargs: Any) -> dict: """ diff --git a/src/firebolt/model/V1/binding.py b/src/firebolt/model/V1/binding.py index c9628d337e..acb3784afc 100644 --- a/src/firebolt/model/V1/binding.py +++ b/src/firebolt/model/V1/binding.py @@ -21,13 +21,13 @@ class Binding(FireboltBaseModel): is_default_engine: bool = Field(alias="engine_is_default") # optional - current_status: Optional[str] - health_status: Optional[str] - create_time: Optional[datetime] - create_actor: Optional[str] - last_update_time: Optional[datetime] - last_update_actor: Optional[str] - desired_status: Optional[str] + current_status: Optional[str] = None + health_status: Optional[str] = None + create_time: Optional[datetime] = None + create_actor: Optional[str] = None + last_update_time: Optional[datetime] = None + last_update_actor: Optional[str] = None + desired_status: Optional[str] = None @property def database_id(self) -> str: diff --git a/src/firebolt/model/V1/engine.py b/src/firebolt/model/V1/engine.py index 97356b5290..4a97eb4c4c 100644 --- a/src/firebolt/model/V1/engine.py +++ b/src/firebolt/model/V1/engine.py @@ -50,7 +50,7 @@ class EngineSettings(FireboltBaseModel): """ preset: str - auto_stop_delay_duration: str = Field(regex=r"^[0-9]+[sm]$|^0$") + auto_stop_delay_duration: str = Field(pattern=r"^[0-9]+[sm]$|^0$") minimum_logging_level: str is_read_only: bool warm_up: str @@ -97,30 +97,30 @@ class Engine(FireboltBaseModel): _service: EngineService = PrivateAttr() # required - name: str = Field(min_length=1, max_length=255, regex=r"^[0-9a-zA-Z_]+$") + name: str = Field(min_length=1, max_length=255, pattern=r"^[0-9a-zA-Z_]+$") compute_region_key: RegionKey = Field(alias="compute_region_id") settings: EngineSettings # optional key: Optional[EngineKey] = Field(None, alias="id") - description: Optional[str] - emoji: Optional[str] - current_status: Optional[EngineStatus] - current_status_summary: Optional[EngineStatusSummary] + description: Optional[str] = None + emoji: Optional[str] = None + current_status: Optional[EngineStatus] = None + current_status_summary: Optional[EngineStatusSummary] = None latest_revision_key: Optional[EngineRevisionKey] = Field( None, alias="latest_revision_id" ) - endpoint: Optional[str] + endpoint: Optional[str] = None endpoint_serving_revision_key: Optional[EngineRevisionKey] = Field( None, alias="endpoint_serving_revision_id" ) - create_time: Optional[datetime] - create_actor: Optional[str] - last_update_time: Optional[datetime] - last_update_actor: Optional[str] - last_use_time: Optional[datetime] - desired_status: Optional[str] - health_status: Optional[str] + create_time: Optional[datetime] = None + create_actor: Optional[str] = None + last_update_time: Optional[datetime] = None + last_update_actor: Optional[str] = None + last_use_time: Optional[datetime] = None + desired_status: Optional[str] = None + health_status: Optional[str] = None endpoint_desired_revision_key: Optional[EngineRevisionKey] = Field( None, alias="endpoint_desired_revision_id" ) From 0d495ed75662464b1800e183588d2aa164941d27 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Thu, 21 Dec 2023 15:19:31 +0000 Subject: [PATCH 2/8] Bump mypy version satisfy its complaints --- setup.cfg | 2 +- src/firebolt/async_db/cursor.py | 2 +- src/firebolt/client/auth/base.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1b5ea5933c..ef942ab8ef 100755 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ ciso8601 = dev = allure-pytest==2.* devtools==0.7.0 - mypy==0.910 + mypy==1.7.1 pre-commit==2.15.0 pyfakefs>=4.5.3 pytest==7.2.0 diff --git a/src/firebolt/async_db/cursor.py b/src/firebolt/async_db/cursor.py index fa254a6024..923b8064e7 100644 --- a/src/firebolt/async_db/cursor.py +++ b/src/firebolt/async_db/cursor.py @@ -395,7 +395,7 @@ def __exit__( self.close() def __await__(self) -> Iterator: - pass + yield None async def __aexit__( self, exc_type: type, exc_val: Exception, exc_tb: TracebackType diff --git a/src/firebolt/client/auth/base.py b/src/firebolt/client/auth/base.py index 3a9cdac134..a5f7e77c44 100644 --- a/src/firebolt/client/auth/base.py +++ b/src/firebolt/client/auth/base.py @@ -1,3 +1,4 @@ +from abc import abstractmethod from time import time from typing import AsyncGenerator, Generator, Optional @@ -44,7 +45,7 @@ def copy(self) -> "Auth": Returns: Auth: Auth object """ - return Auth(self._use_token_cache) + return self.__class__(self._use_token_cache) @property def token(self) -> Optional[str]: @@ -103,6 +104,7 @@ def _cache_token(self) -> None: if self._token and self._expires: self._token_storage.cache_token(self._token, self._expires) + @abstractmethod def get_new_token_generator(self) -> Generator[Request, Response, None]: """Generate requests needed to create a new token session.""" From ceb64a0556f6e5efcce1c45a296f1937863b67da Mon Sep 17 00:00:00 2001 From: ptiurin Date: Thu, 21 Dec 2023 15:46:59 +0000 Subject: [PATCH 3/8] mypy 1.* --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ef942ab8ef..6a7870b04e 100755 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ ciso8601 = dev = allure-pytest==2.* devtools==0.7.0 - mypy==1.7.1 + mypy==1.* pre-commit==2.15.0 pyfakefs>=4.5.3 pytest==7.2.0 From 05934e09000c42dd54ccc914d097f2972c7b74e9 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Thu, 21 Dec 2023 16:13:05 +0000 Subject: [PATCH 4/8] limit below 3 just in case --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6a7870b04e..2d028fd069 100755 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ install_requires = cryptography>=3.4.0 httpcore<0.17.3 httpx[http2]==0.24.0 - pydantic[dotenv]>=1.8.2 + pydantic[dotenv]>=1.8.2,<3.0.0 python-dateutil>=2.8.2 readerwriterlock>=1.0.9 sqlparse>=0.4.2 From 366f3fc8ec6abff1f90f093e15593936f91da789 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Wed, 17 Jan 2024 14:49:54 +0000 Subject: [PATCH 5/8] Add v2 pydantic support to new classes from main --- src/firebolt/model/V1/database.py | 31 +++++++++++++++--------------- src/firebolt/model/V1/provider.py | 6 +++--- src/firebolt/model/V1/region.py | 6 +++--- src/firebolt/service/V1/binding.py | 1 + src/firebolt/service/V1/region.py | 7 +++++++ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/firebolt/model/V1/database.py b/src/firebolt/model/V1/database.py index 27d868088d..6ad19c1ef6 100644 --- a/src/firebolt/model/V1/database.py +++ b/src/firebolt/model/V1/database.py @@ -42,24 +42,24 @@ class Database(FireboltBaseModel): _service: DatabaseService = PrivateAttr() # required - name: str = Field(min_length=1, max_length=255, regex=r"^[0-9a-zA-Z_]+$") + name: str = Field(min_length=1, max_length=255, pattern=r"^[0-9a-zA-Z_]+$") compute_region_key: RegionKey = Field(alias="compute_region_id") # optional - database_key: Optional[DatabaseKey] = Field(None, alias="id") - description: Optional[str] = Field(None, max_length=255) - emoji: Optional[str] = Field(None, max_length=255) - current_status: Optional[str] - health_status: Optional[str] - data_size_full: Optional[int] - data_size_compressed: Optional[int] - is_system_database: Optional[bool] - storage_bucket_name: Optional[str] - create_time: Optional[datetime] - create_actor: Optional[str] - last_update_time: Optional[datetime] - last_update_actor: Optional[str] - desired_status: Optional[str] + database_key: Optional[DatabaseKey] = Field(default=None, alias="id") + description: Optional[str] = Field(default=None, max_length=255) + emoji: Optional[str] = Field(default=None, max_length=255) + current_status: Optional[str] = None + health_status: Optional[str] = None + data_size_full: Optional[int] = None + data_size_compressed: Optional[int] = None + is_system_database: Optional[bool] = None + storage_bucket_name: Optional[str] = None + create_time: Optional[datetime] = None + create_actor: Optional[str] = None + last_update_time: Optional[datetime] = None + last_update_actor: Optional[str] = None + desired_status: Optional[str] = None @classmethod def parse_obj_with_service( @@ -142,6 +142,7 @@ class _DatabaseUpdateRequest(FireboltBaseModel): self.description = description + assert self.database_id is not None, "Database must have database_id" logger.info( f"Updating Database (database_id={self.database_id}, " f"name={self.name}, description={self.description})" diff --git a/src/firebolt/model/V1/provider.py b/src/firebolt/model/V1/provider.py index 01ac13176e..dc4e406e1b 100644 --- a/src/firebolt/model/V1/provider.py +++ b/src/firebolt/model/V1/provider.py @@ -11,6 +11,6 @@ class Provider(FireboltBaseModel, frozen=True): # type: ignore name: str # optional - create_time: Optional[datetime] - display_name: Optional[str] - last_update_time: Optional[datetime] + create_time: Optional[datetime] = None + display_name: Optional[str] = None + last_update_time: Optional[datetime] = None diff --git a/src/firebolt/model/V1/region.py b/src/firebolt/model/V1/region.py index 9fc2c605df..bb984d7ea1 100644 --- a/src/firebolt/model/V1/region.py +++ b/src/firebolt/model/V1/region.py @@ -16,6 +16,6 @@ class Region(FireboltBaseModel): name: str # optional - display_name: Optional[str] - create_time: Optional[datetime] - last_update_time: Optional[datetime] + display_name: Optional[str] = None + create_time: Optional[datetime] = None + last_update_time: Optional[datetime] = None diff --git a/src/firebolt/service/V1/binding.py b/src/firebolt/service/V1/binding.py index e58bac95e4..f21b09cea9 100644 --- a/src/firebolt/service/V1/binding.py +++ b/src/firebolt/service/V1/binding.py @@ -125,6 +125,7 @@ def create( f"to Database (database_id={database.database_id}, " f"name={database.name})" ) + assert database.database_id is not None, "Database must have database_id" binding = Binding( binding_key=BindingKey( account_id=self.account_id, diff --git a/src/firebolt/service/V1/region.py b/src/firebolt/service/V1/region.py index a6676b7943..710894b896 100644 --- a/src/firebolt/service/V1/region.py +++ b/src/firebolt/service/V1/region.py @@ -61,6 +61,13 @@ def get_by_key(self, key: RegionKey) -> Region: def get_by_id(self, id_: str) -> Region: """Get an AWS region by region_id.""" + # error if provider_id is not set + if not self.resource_manager.provider_id: + raise ValueError( + "provider_id parameter must be set when initializing " + "the resource manager." + ) + return self.get_by_key( RegionKey(provider_id=self.resource_manager.provider_id, region_id=id_) ) From 8d99e081390a320734564d305579b92febf9780e Mon Sep 17 00:00:00 2001 From: ptiurin Date: Wed, 17 Jan 2024 15:15:04 +0000 Subject: [PATCH 6/8] use aliases --- src/firebolt/service/V1/binding.py | 4 ++-- src/firebolt/service/V1/database.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/firebolt/service/V1/binding.py b/src/firebolt/service/V1/binding.py index f21b09cea9..a40283d7b7 100644 --- a/src/firebolt/service/V1/binding.py +++ b/src/firebolt/service/V1/binding.py @@ -127,12 +127,12 @@ def create( ) assert database.database_id is not None, "Database must have database_id" binding = Binding( - binding_key=BindingKey( + binding_id=BindingKey( account_id=self.account_id, database_id=database.database_id, engine_id=engine.engine_id, ), - is_default_engine=is_default_engine, + engine_is_default=is_default_engine, ) response = self.client.post( diff --git a/src/firebolt/service/V1/database.py b/src/firebolt/service/V1/database.py index 2673f0a740..bb882da13a 100644 --- a/src/firebolt/service/V1/database.py +++ b/src/firebolt/service/V1/database.py @@ -110,7 +110,7 @@ class _DatabaseCreateRequest(FireboltBaseModel): else: region_key = self.resource_manager.regions.get_by_name(name=region).key database = Database( - name=name, compute_region_key=region_key, description=description + name=name, compute_region_id=region_key, description=description ) logger.info(f"Creating Database (name={name})") From 86c166371a7c1a6b74e1d9987c164eab7c6e6953 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Wed, 17 Jan 2024 15:17:53 +0000 Subject: [PATCH 7/8] fix binding alias --- src/firebolt/service/V1/binding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firebolt/service/V1/binding.py b/src/firebolt/service/V1/binding.py index a40283d7b7..61d38a92e2 100644 --- a/src/firebolt/service/V1/binding.py +++ b/src/firebolt/service/V1/binding.py @@ -127,7 +127,7 @@ def create( ) assert database.database_id is not None, "Database must have database_id" binding = Binding( - binding_id=BindingKey( + id=BindingKey( account_id=self.account_id, database_id=database.database_id, engine_id=engine.engine_id, From cb7f4ef98ba51ddc180f2609bdfb48260980d5ab Mon Sep 17 00:00:00 2001 From: ptiurin Date: Wed, 17 Jan 2024 18:18:38 +0000 Subject: [PATCH 8/8] add pydantic.mypy plugin --- setup.cfg | 1 + src/firebolt/service/V1/binding.py | 4 ++-- src/firebolt/service/V1/database.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2d028fd069..6f4f071355 100755 --- a/setup.cfg +++ b/setup.cfg @@ -71,6 +71,7 @@ firebolt = py.typed disallow_untyped_defs = True show_error_codes = True files = src/ +plugins = pydantic.mypy [flake8] exclude = tests/* diff --git a/src/firebolt/service/V1/binding.py b/src/firebolt/service/V1/binding.py index 61d38a92e2..f21b09cea9 100644 --- a/src/firebolt/service/V1/binding.py +++ b/src/firebolt/service/V1/binding.py @@ -127,12 +127,12 @@ def create( ) assert database.database_id is not None, "Database must have database_id" binding = Binding( - id=BindingKey( + binding_key=BindingKey( account_id=self.account_id, database_id=database.database_id, engine_id=engine.engine_id, ), - engine_is_default=is_default_engine, + is_default_engine=is_default_engine, ) response = self.client.post( diff --git a/src/firebolt/service/V1/database.py b/src/firebolt/service/V1/database.py index bb882da13a..2673f0a740 100644 --- a/src/firebolt/service/V1/database.py +++ b/src/firebolt/service/V1/database.py @@ -110,7 +110,7 @@ class _DatabaseCreateRequest(FireboltBaseModel): else: region_key = self.resource_manager.regions.get_by_name(name=region).key database = Database( - name=name, compute_region_id=region_key, description=description + name=name, compute_region_key=region_key, description=description ) logger.info(f"Creating Database (name={name})")