From 4e4738e9c3edbf906844854f903d785e71900f06 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 19 Sep 2024 20:02:37 +0900 Subject: [PATCH 01/27] feat: Upgrade the pydantic version Pydantic used to be in 1.10.5 now moving to up to v2 accepting from v2.x to the latest --- setup.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..89e0f21 --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""Aleph Message - Python library for the Aleph.im message specification +(c) 2022 OKESO for Aleph.im +""" + +import os +import re + + +def get_version(): + version_file = os.path.join("aleph_message", "__init__.py") + initfile_lines = open(version_file, "rt").readlines() + version = r"^__version__ = ['\"]([^'\"]*)['\"]" + + for line in initfile_lines: + mo = re.search(version, line, re.M) + if mo: + return mo.group(1) + raise RuntimeError(f"Unable to find version string in {version_file}.") + + +from setuptools import setup + +# allow setup.py to be run from any path +os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + +with open("README.md") as file: + long_description = file.read() + +setup( + name="aleph-message", + version=get_version(), + description="Aleph.im message specification ", + long_description=long_description, + long_description_content_type="text/markdown", + author="Hugo Herter", + author_email="git@hugoherter.com", + url="https://github.com/aleph-im/aleph-message", + packages=[ + "aleph_message", + "aleph_message.models", + "aleph_message.models.execution", + ], + package_data={ + "aleph_message": ["py.typed"], + "aleph_message.models": ["py.typed"], + "aleph_message.models.execution": ["py.typed"], + }, + data_files=[], + install_requires=[ + "pydantic>=2", + "typing_extensions>=4.5.0", + ], + license="MIT", + platform="any", + keywords="aleph.im message validation specification", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Topic :: System :: Distributed Computing", + ], +) From 21a27a98b34254246a1ec1211e898a734302df14 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 27 Sep 2024 22:45:12 +0900 Subject: [PATCH 02/27] Fix: Refactor ItemHash to align with Pydantic v2 validation schema. Replaced `__get_pydantic_core_schema__` with a more efficient schema handling using `core_schema.str_schema()` and custom validation for ItemHash. --- aleph_message/models/item_hash.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/aleph_message/models/item_hash.py b/aleph_message/models/item_hash.py index e029416..7037a99 100644 --- a/aleph_message/models/item_hash.py +++ b/aleph_message/models/item_hash.py @@ -2,7 +2,8 @@ from functools import lru_cache from ..exceptions import UnknownHashError - +from pydantic_core import CoreSchema, core_schema +from pydantic import GetCoreSchemaHandler class ItemType(str, Enum): """Item storage options""" @@ -45,18 +46,21 @@ def __new__(cls, value: str): return obj @classmethod - def __get_validators__(cls): - # one or more validators may be yielded which will be called in the - # order to validate the input, each validator will receive as an input - # the value returned from the previous validator - yield cls.validate + def __get_pydantic_core_schema__(cls, source, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + # This function validates the input after the initial type validation (as a string). + # The returned value from this function will be used as the final validated value. + + # Return a string schema and add a post-validation function to convert to ItemHash + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.str_schema() + ) @classmethod def validate(cls, v): if not isinstance(v, str): raise TypeError("Item hash must be a string") - - return cls(v) + return cls(v) # Convert to ItemHash def __repr__(self): return f"" From 2404dfe57762fed3238d73cab2d906c667d95bad Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 27 Sep 2024 22:57:39 +0900 Subject: [PATCH 03/27] Fix: Refactor classes and functions to align with Pydantic v2. - Updated code to explicitly specify optional keys where necessary. - Replaced direct `.get` calls with `data.get()` to handle new validation logic. - Migrated model configuration to use `model_config = ConfigDict(extra="forbid")` or `model_config = ConfigDict(extra="allow")` in place of Pydantic v1's configuration style. --- aleph_message/models/abstract.py | 5 +-- aleph_message/models/execution/environment.py | 25 +++++------- aleph_message/models/execution/instance.py | 12 +++++- aleph_message/models/execution/program.py | 8 +++- aleph_message/models/execution/volume.py | 40 +++++++++++++------ 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/aleph_message/models/abstract.py b/aleph_message/models/abstract.py index f272dbd..2af6f8b 100644 --- a/aleph_message/models/abstract.py +++ b/aleph_message/models/abstract.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Extra +from pydantic import BaseModel, ConfigDict def hashable(obj): @@ -24,5 +24,4 @@ class BaseContent(BaseModel): address: str time: float - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index d277f36..c517fc8 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -3,7 +3,7 @@ from enum import Enum from typing import List, Literal, Optional, Union -from pydantic import Extra, Field, validator +from pydantic import ConfigDict, Field, field_validator from ...utils import Mebibytes from ..abstract import HashableModel @@ -13,8 +13,7 @@ class Subscription(HashableModel): """A subscription is used to trigger a program in response to a FunctionTrigger.""" - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") class FunctionTriggers(HashableModel): @@ -29,8 +28,7 @@ class FunctionTriggers(HashableModel): description="Persist the execution of the program instead of running it on demand.", ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class NetworkProtocol(str, Enum): @@ -85,8 +83,7 @@ class CpuProperties(HashableModel): description="CPU features required by the virtual machine. Examples: 'sev', 'sev_es', 'sev_snp'.", ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class GpuDeviceClass(str, Enum): @@ -153,8 +150,7 @@ class TrustedExecutionEnvironment(HashableModel): description="Policy of the TEE. Default value is 0x01 for SEV without debugging.", ) - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") class InstanceEnvironment(HashableModel): @@ -171,9 +167,9 @@ class InstanceEnvironment(HashableModel): reproducible: bool = False shared_cache: bool = False - @validator("trusted_execution", pre=True) + @field_validator("trusted_execution", mode="before") def check_hypervisor(cls, v, values): - if v and values.get("hypervisor") != HypervisorType.qemu: + if v and values.data.get("hypervisor") != HypervisorType.qemu: raise ValueError("Trusted Execution Environment is only supported for QEmu") return v @@ -190,8 +186,7 @@ class NodeRequirements(HashableModel): default=None, description="Terms and conditions of this CRN" ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class HostRequirements(HashableModel): @@ -205,6 +200,4 @@ class HostRequirements(HashableModel): default=None, description="GPUs needed to pass-through from the host" ) - class Config: - # Allow users to add custom requirements - extra = Extra.allow + model_config = ConfigDict(extra="allow") diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index e4120b2..5bf7f1b 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -1,6 +1,7 @@ from __future__ import annotations +from typing import Optional, List -from pydantic import Field, root_validator +from pydantic import Field, field_validator from aleph_message.models.abstract import HashableModel @@ -21,11 +22,20 @@ class RootfsVolume(HashableModel): persistence: VolumePersistence # Use the same size constraint as persistent volumes for now size_mib: PersistentVolumeSizeMib + forgotten_by: Optional[List[str]] = None + + @field_validator('size_mib', mode="before") + def convert_size_mib(cls, v): + if isinstance(v, int): + return PersistentVolumeSizeMib(persistent_volume_size=v) + return v class InstanceContent(BaseExecutableContent): """Message content for scheduling a VM instance on the network.""" + metadata: Optional[dict] = None + payment: Optional[dict] = None environment: InstanceEnvironment = Field( description="Properties of the instance execution environment" ) diff --git a/aleph_message/models/execution/program.py b/aleph_message/models/execution/program.py index 8afb6d9..ee3237c 100644 --- a/aleph_message/models/execution/program.py +++ b/aleph_message/models/execution/program.py @@ -43,8 +43,8 @@ class DataContent(HashableModel): encoding: Encoding mount: str - ref: ItemHash - use_latest: bool = False + ref: Optional[ItemHash] = None + use_latest: Optional[bool] = False class Export(HashableModel): @@ -69,3 +69,7 @@ class ProgramContent(BaseExecutableContent): default=None, description="Data to export after computation" ) on: FunctionTriggers = Field(description="Signals that trigger an execution") + + metadata: Optional[dict] = None + authorized_keys: Optional[List[str]] = None + payment: Optional[dict] = None diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index e4d6283..f9b15aa 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Literal, Optional, Union -from pydantic import ConstrainedInt, Extra +from pydantic import Field, ConfigDict, BaseModel, field_validator from ...utils import Gigabytes, gigabyte_to_mebibyte from ..abstract import HashableModel @@ -18,27 +18,35 @@ class AbstractVolume(HashableModel, ABC): @abstractmethod def is_read_only(self): ... - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class ImmutableVolume(AbstractVolume): - ref: ItemHash + ref: ItemHash = None use_latest: bool = True def is_read_only(self): return True -class EphemeralVolumeSize(ConstrainedInt): - gt = 0 - le = 1000 # Limit to 1 GiB - strict = True +class EphemeralVolumeSize(BaseModel): + ephemeral_volume_size: int = Field(gt=0, + le=1000, #Limit to 1GiB + strict=True) + + def __hash__(self): + return hash(self.ephemeral_volume_size) class EphemeralVolume(AbstractVolume): ephemeral: Literal[True] = True - size_mib: EphemeralVolumeSize + size_mib: EphemeralVolumeSize = 0 + + @field_validator('size_mib', mode="before") + def convert_size_mib(cls, v): + if isinstance(v, int): + return EphemeralVolumeSize(ephemeral_volume_size=v) + return v def is_read_only(self): return False @@ -65,10 +73,16 @@ class PersistentVolumeSizeMib(ConstrainedInt): class PersistentVolume(AbstractVolume): - parent: Optional[ParentVolume] - persistence: VolumePersistence - name: str - size_mib: PersistentVolumeSizeMib + parent: Optional[ParentVolume] = None + persistence: VolumePersistence = None + name: Optional[str] = None + size_mib: PersistentVolumeSizeMib = 0 + + @field_validator('size_mib', mode="before") + def convert_size_mib(cls, v): + if isinstance(v, int): + return PersistentVolumeSizeMib(persistent_volume_size=v) + return v def is_read_only(self): return False From 7e264220687d98dc540e302bdb9e5d5e99ba22a5 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 27 Sep 2024 23:00:53 +0900 Subject: [PATCH 04/27] Fix: Refactor to use `model_dump` and `model_dump_json` in place of deprecated methods. - Replaced `.dict()` with `.model_dump()` for model serialization. - Replaced deprecated `.json()` with `.model_dump_json()` for JSON serialization. These changes ensure compatibility with Pydantic v2 by using the updated serialization methods. --- aleph_message/tests/test_models.py | 24 ++++++++++++------------ aleph_message/tests/test_types.py | 18 +++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 3ab085e..b9ac15f 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -49,9 +49,9 @@ def test_message_response_aggregate(): data_dict = requests.get(f"{ALEPH_API_SERVER}{path}").json() message = data_dict["messages"][0] - AggregateMessage.parse_obj(message) + AggregateMessage.model_validate(message) - response = MessagesResponse.parse_obj(data_dict) + response = MessagesResponse.model_validate(data_dict) assert response @@ -62,7 +62,7 @@ def test_message_response_post(): ) data_dict = requests.get(f"{ALEPH_API_SERVER}{path}").json() - response = MessagesResponse.parse_obj(data_dict) + response = MessagesResponse.model_validate(data_dict) assert response @@ -73,7 +73,7 @@ def test_message_response_store(): ) data_dict = requests.get(f"{ALEPH_API_SERVER}{path}").json() - response = MessagesResponse.parse_obj(data_dict) + response = MessagesResponse.model_validate(data_dict) assert response @@ -105,7 +105,7 @@ def test_post_content(): time=1.0, ) assert p1.type == custom_type - assert p1.dict() == { + assert p1.model_dump() == { "address": "0x1", "time": 1.0, "content": {"blah": "bar"}, @@ -182,7 +182,7 @@ def test_validation_on_confidential_options(): assert e.errors()[0]["loc"] == ("content", "environment", "trusted_execution") assert ( e.errors()[0]["msg"] - == "Trusted Execution Environment is only supported for QEmu" + == "Value error, Trusted Execution Environment is only supported for QEmu" ) @@ -316,8 +316,8 @@ def test_message_forget_cannot_be_forgotten(): message_raw["forgotten_by"] = ["abcde"] with pytest.raises(ValueError) as e: - ForgetMessage.parse_obj(message_raw) - assert e.value.args[0][0].exc.args == ("This type of message may not be forgotten",) + ForgetMessage.model_validate(message_raw) + assert "This type of message may not be forgotten" in str(e.value) def test_message_forgotten_by(): @@ -327,10 +327,10 @@ def test_message_forgotten_by(): message_raw = add_item_content_and_hash(message_raw) # Test different values for field 'forgotten_by' - _ = ProgramMessage.parse_obj(message_raw) - _ = ProgramMessage.parse_obj({**message_raw, "forgotten_by": None}) - _ = ProgramMessage.parse_obj({**message_raw, "forgotten_by": ["abcde"]}) - _ = ProgramMessage.parse_obj({**message_raw, "forgotten_by": ["abcde", "fghij"]}) + _ = ProgramMessage.model_validate(message_raw) + _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": None}) + _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": ["abcde"]}) + _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": ["abcde", "fghij"]}) def test_item_type_from_hash(): diff --git a/aleph_message/tests/test_types.py b/aleph_message/tests/test_types.py index a322b8a..f0a2dbf 100644 --- a/aleph_message/tests/test_types.py +++ b/aleph_message/tests/test_types.py @@ -1,7 +1,7 @@ import copy import pytest -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, ValidationError, field_validator from aleph_message.exceptions import UnknownHashError from aleph_message.models import ItemHash, ItemType @@ -25,35 +25,35 @@ class ModelWithItemHash(BaseModel): def test_item_hash(): storage_object_dict = {"hash": STORAGE_HASH} - storage_object = ModelWithItemHash.parse_obj(storage_object_dict) + storage_object = ModelWithItemHash.model_validate(storage_object_dict) assert storage_object.hash == STORAGE_HASH assert storage_object.hash.item_type == ItemType.storage ipfs_object_dict = {"hash": IPFS_HASH} - ipfs_object = ModelWithItemHash.parse_obj(ipfs_object_dict) + ipfs_object = ModelWithItemHash.model_validate(ipfs_object_dict) assert ipfs_object.hash == IPFS_HASH assert ipfs_object.hash.item_type == ItemType.ipfs assert repr(ipfs_object.hash).startswith(" Date: Fri, 27 Sep 2024 23:05:19 +0900 Subject: [PATCH 05/27] Refactor to use `model_dump` and `model_dump_json`, and update `check_content` function for stricter comparison. - Replaced `.dict()` with `.model_dump()` for model serialization. - Replaced deprecated `.json()` with `.model_dump_json()` for JSON serialization. - Updated the `check_content` function to properly normalize and compare JSON structures, as Pydantic v2 enforces stricter validation and comparison rules. These changes ensure compatibility with Pydantic v2 by adopting the new serialization methods and handling stricter content comparison logic. --- aleph_message/models/__init__.py | 127 +++++++++++++++++++------------ 1 file changed, 80 insertions(+), 47 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 10c1925..31fd060 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -1,4 +1,5 @@ import datetime +from enum import Enum import json from copy import copy from hashlib import sha256 @@ -6,7 +7,8 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast -from pydantic import BaseModel, Extra, Field, validator +from aleph_message.models.execution.volume import EphemeralVolumeSize, PersistentVolumeSizeMib +from pydantic import BaseModel, Field, field_validator, ConfigDict from typing_extensions import TypeAlias from .abstract import BaseContent, HashableModel @@ -54,8 +56,8 @@ class MongodbId(BaseModel): oid: str = Field(alias="$oid") - class Config: - extra = Extra.forbid + +model_config = ConfigDict(extra="forbid") class ChainRef(BaseModel): @@ -76,8 +78,7 @@ class MessageConfirmationHash(BaseModel): binary: str = Field(alias="$binary") type: str = Field(alias="$type") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class MessageConfirmation(BaseModel): @@ -93,15 +94,13 @@ class MessageConfirmation(BaseModel): default=None, description="The address that published the transaction." ) - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class AggregateContentKey(BaseModel): name: str - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class PostContent(BaseContent): @@ -116,16 +115,15 @@ class PostContent(BaseContent): ) type: str = Field(description="User-generated 'content-type' of a POST message") - @validator("type") + @field_validator("type") def check_type(cls, v, values): if v == "amend": - ref = values.get("ref") + ref = values.data.get("ref") if not ref: raise ValueError("A 'ref' is required for POST type 'amend'") return v - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class AggregateContent(BaseContent): @@ -136,8 +134,7 @@ class AggregateContent(BaseContent): ) content: Dict = Field(description="The content of an aggregate must be a dict") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class StoreContent(BaseContent): @@ -150,8 +147,7 @@ class StoreContent(BaseContent): ref: Optional[str] = None metadata: Optional[Dict[str, Any]] = Field(description="Metadata of the VM") - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow") class ForgetContent(BaseContent): @@ -214,9 +210,9 @@ class BaseMessage(BaseModel): forgotten_by: Optional[List[str]] - @validator("item_content") + @field_validator("item_content") def check_item_content(cls, v: Optional[str], values) -> Optional[str]: - item_type = values["item_type"] + item_type = values.data.get("item_type") if v is None: return None elif item_type == ItemType.inline: @@ -232,14 +228,14 @@ def check_item_content(cls, v: Optional[str], values) -> Optional[str]: ) return v - @validator("item_hash") + @field_validator("item_hash") def check_item_hash(cls, v: ItemHash, values) -> ItemHash: - item_type = values["item_type"] + item_type = values.data.get("item_type") if item_type == ItemType.inline: - item_content: str = values["item_content"] + item_content: str = values.data.get("item_content") # Double check that the hash function is supported - hash_type = values["hash_type"] or HashType.sha256 + hash_type = values.data.get("hash_type") or HashType.sha256 assert hash_type.value == HashType.sha256 computed_hash: str = sha256(item_content.encode()).hexdigest() @@ -255,23 +251,21 @@ def check_item_hash(cls, v: ItemHash, values) -> ItemHash: assert item_type == ItemType.storage return v - @validator("confirmed") + @field_validator("confirmed") def check_confirmed(cls, v, values): - confirmations = values["confirmations"] + confirmations = values.data.get("confirmations") if v is True and not bool(confirmations): raise ValueError("Message cannot be 'confirmed' without 'confirmations'") return v - @validator("time") + @field_validator("time") def convert_float_to_datetime(cls, v, values): if isinstance(v, float): v = datetime.datetime.fromtimestamp(v) assert isinstance(v, datetime.datetime) return v - class Config: - extra = Extra.forbid - exclude = {"id_", "_id"} + model_config = ConfigDict(extra="forbid", exclude={"id_", "_id"}) class PostMessage(BaseMessage): @@ -279,6 +273,7 @@ class PostMessage(BaseMessage): type: Literal[MessageType.post] content: PostContent + forgotten_by: Optional[List[str]] = None class AggregateMessage(BaseMessage): @@ -286,18 +281,22 @@ class AggregateMessage(BaseMessage): type: Literal[MessageType.aggregate] content: AggregateContent + forgotten_by: Optional[list] = None class StoreMessage(BaseMessage): type: Literal[MessageType.store] content: StoreContent + forgotten_by: Optional[list] = None + metadata: Optional[Dict[str, Any]] = None class ForgetMessage(BaseMessage): type: Literal[MessageType.forget] content: ForgetContent + forgotten_by: Optional[list] = None - @validator("forgotten_by") + @field_validator("forgotten_by") def cannot_be_forgotten(cls, v: Optional[List[str]], values) -> Optional[List[str]]: assert values if v: @@ -307,26 +306,61 @@ def cannot_be_forgotten(cls, v: Optional[List[str]], values) -> Optional[List[st class ProgramMessage(BaseMessage): type: Literal[MessageType.program] - content: ProgramContent - - @validator("content") + content: Optional[ProgramContent] = None + forgotten_by: Optional[List[str]] = None + + @staticmethod + def normalize_content(content): + if not isinstance(content, dict): + return content + + normalized_content = {} + for key, value in content.items(): + # Handle special cases such as ItemHash, Enum, list and dict + if isinstance(value, ItemHash): + normalized_content[key] = str(value) # ItemHash to string + elif isinstance(value, Enum): + normalized_content[key] = value.value # Enum to string + elif isinstance(value, list): + normalized_content[key] = [ProgramMessage.normalize_content(v) for v in value] + elif isinstance(value, dict): + # Special case for 'size_mib' and 'data' + if key == 'size_mib': + normalized_content[key] = list(value.values())[0] + else: + normalized_content[key] = ProgramMessage.normalize_content(value) + else: + normalized_content[key] = value # Keep the value as is + + return normalized_content + + + + @field_validator("content") def check_content(cls, v, values): - item_type = values["item_type"] + item_type = values.data.get("item_type") if item_type == ItemType.inline: - item_content = json.loads(values["item_content"]) - if v.dict(exclude_none=True) != item_content: - # Print differences - vdict = v.dict(exclude_none=True) - for key, value in item_content.items(): - if vdict[key] != value: - print(f"{key}: {vdict[key]} != {value}") + item_content = json.loads(values.data.get("item_content")) + + # Normalizing content to fit the structure of item_content + normalized_content = cls.normalize_content(v.dict(exclude_none=True)) + + if normalized_content != item_content: + # Print les différences + print("Differences found between content and item_content") + print(f"Content: {normalized_content}") + print(f"Item Content: {item_content}") raise ValueError("Content and item_content differ") return v + + + class InstanceMessage(BaseMessage): type: Literal[MessageType.instance] content: InstanceContent + forgotten_by: Optional[List[str]] = None AlephMessage: TypeAlias = Union[ @@ -363,7 +397,7 @@ def parse_message(message_dict: Dict) -> AlephMessage: message_class.__annotations__["type"].__args__[0] ) if message_dict["type"] == message_type: - return message_class.parse_obj(message_dict) + return message_class.model_validate(message_dict) else: raise ValueError(f"Unknown message type {message_dict['type']}") @@ -390,7 +424,7 @@ def create_new_message( """ message_content = add_item_content_and_hash(message_dict) if factory: - return cast(T, factory.parse_obj(message_content)) + return cast(T, factory.model_validate(message_content)) else: return cast(T, parse_message(message_content)) @@ -405,7 +439,7 @@ def create_message_from_json( message_dict = json.loads(json_data) message_content = add_item_content_and_hash(message_dict, inplace=True) if factory: - return factory.parse_obj(message_content) + return factory.model_validate(message_content) else: return parse_message(message_content) @@ -422,7 +456,7 @@ def create_message_from_file( message_dict = decoder.load(fd) message_content = add_item_content_and_hash(message_dict, inplace=True) if factory: - return factory.parse_obj(message_content) + return factory.model_validate(message_content) else: return parse_message(message_content) @@ -436,5 +470,4 @@ class MessagesResponse(BaseModel): pagination_per_page: int pagination_item: str - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") From c699881d5e252e0d2c22583bca8050139b13f489 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 2 Oct 2024 11:53:38 +0900 Subject: [PATCH 06/27] Fix: Changes in files to comply with MyPy, Ruff, and Black - Refactored the normalize_content function to respect MyPy. - Reformatted code according to Black. - Fixed linting issues raised by Ruff. --- aleph_message/models/__init__.py | 37 +++++++++++----------- aleph_message/models/execution/instance.py | 5 +-- aleph_message/models/execution/program.py | 4 +-- aleph_message/models/execution/volume.py | 18 +++++------ aleph_message/models/item_hash.py | 10 +++--- aleph_message/tests/test_types.py | 2 +- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 31fd060..32fdba7 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast -from aleph_message.models.execution.volume import EphemeralVolumeSize, PersistentVolumeSizeMib from pydantic import BaseModel, Field, field_validator, ConfigDict from typing_extensions import TypeAlias @@ -265,7 +264,10 @@ def convert_float_to_datetime(cls, v, values): assert isinstance(v, datetime.datetime) return v - model_config = ConfigDict(extra="forbid", exclude={"id_", "_id"}) + model_config = ConfigDict(extra="forbid") + + def custom_dump(self): + return self.model_dump(exclude={"id_", "_id"}) class PostMessage(BaseMessage): @@ -306,36 +308,38 @@ def cannot_be_forgotten(cls, v: Optional[List[str]], values) -> Optional[List[st class ProgramMessage(BaseMessage): type: Literal[MessageType.program] - content: Optional[ProgramContent] = None + content: ProgramContent forgotten_by: Optional[List[str]] = None @staticmethod - def normalize_content(content): + def normalize_content(content: Union[Dict[str, Any], Any]) -> Any: if not isinstance(content, dict): return content - normalized_content = {} + normalized_content: Dict[str, Any] = {} + for key, value in content.items(): - # Handle special cases such as ItemHash, Enum, list and dict if isinstance(value, ItemHash): - normalized_content[key] = str(value) # ItemHash to string + normalized_content[key] = str(value) elif isinstance(value, Enum): - normalized_content[key] = value.value # Enum to string + normalized_content[key] = value.value elif isinstance(value, list): - normalized_content[key] = [ProgramMessage.normalize_content(v) for v in value] + if key == "volumes" and all(isinstance(v, str) for v in value): + normalized_content[key] = value + else: + normalized_content[key] = [ + ProgramMessage.normalize_content(v) for v in value + ] elif isinstance(value, dict): - # Special case for 'size_mib' and 'data' - if key == 'size_mib': + if key == "size_mib": normalized_content[key] = list(value.values())[0] else: normalized_content[key] = ProgramMessage.normalize_content(value) else: - normalized_content[key] = value # Keep the value as is + normalized_content[key] = value return normalized_content - - @field_validator("content") def check_content(cls, v, values): item_type = values.data.get("item_type") @@ -343,7 +347,7 @@ def check_content(cls, v, values): item_content = json.loads(values.data.get("item_content")) # Normalizing content to fit the structure of item_content - normalized_content = cls.normalize_content(v.dict(exclude_none=True)) + normalized_content = cls.normalize_content(v.model_dump(exclude_none=True)) if normalized_content != item_content: # Print les différences @@ -354,9 +358,6 @@ def check_content(cls, v, values): return v - - - class InstanceMessage(BaseMessage): type: Literal[MessageType.instance] content: InstanceContent diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 5bf7f1b..afd9007 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -8,6 +8,7 @@ from .abstract import BaseExecutableContent from .environment import HypervisorType, InstanceEnvironment from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence +from .base import Payment class RootfsVolume(HashableModel): @@ -24,7 +25,7 @@ class RootfsVolume(HashableModel): size_mib: PersistentVolumeSizeMib forgotten_by: Optional[List[str]] = None - @field_validator('size_mib', mode="before") + @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): if isinstance(v, int): return PersistentVolumeSizeMib(persistent_volume_size=v) @@ -35,7 +36,7 @@ class InstanceContent(BaseExecutableContent): """Message content for scheduling a VM instance on the network.""" metadata: Optional[dict] = None - payment: Optional[dict] = None + payment: Optional[Payment] = None environment: InstanceEnvironment = Field( description="Properties of the instance execution environment" ) diff --git a/aleph_message/models/execution/program.py b/aleph_message/models/execution/program.py index ee3237c..9bb2228 100644 --- a/aleph_message/models/execution/program.py +++ b/aleph_message/models/execution/program.py @@ -7,7 +7,7 @@ from ..abstract import HashableModel from ..item_hash import ItemHash from .abstract import BaseExecutableContent -from .base import Encoding, Interface, MachineType +from .base import Encoding, Interface, MachineType, Payment from .environment import FunctionTriggers @@ -72,4 +72,4 @@ class ProgramContent(BaseExecutableContent): metadata: Optional[dict] = None authorized_keys: Optional[List[str]] = None - payment: Optional[dict] = None + payment: Optional[Payment] = None diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index f9b15aa..720601e 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -22,7 +22,7 @@ def is_read_only(self): ... class ImmutableVolume(AbstractVolume): - ref: ItemHash = None + ref: Optional[ItemHash] = None use_latest: bool = True def is_read_only(self): @@ -30,9 +30,7 @@ def is_read_only(self): class EphemeralVolumeSize(BaseModel): - ephemeral_volume_size: int = Field(gt=0, - le=1000, #Limit to 1GiB - strict=True) + ephemeral_volume_size: int = Field(gt=-1, le=1000, strict=True) # Limit to 1GiB def __hash__(self): return hash(self.ephemeral_volume_size) @@ -40,9 +38,9 @@ def __hash__(self): class EphemeralVolume(AbstractVolume): ephemeral: Literal[True] = True - size_mib: EphemeralVolumeSize = 0 + size_mib: EphemeralVolumeSize = EphemeralVolumeSize(ephemeral_volume_size=0) - @field_validator('size_mib', mode="before") + @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): if isinstance(v, int): return EphemeralVolumeSize(ephemeral_volume_size=v) @@ -74,11 +72,13 @@ class PersistentVolumeSizeMib(ConstrainedInt): class PersistentVolume(AbstractVolume): parent: Optional[ParentVolume] = None - persistence: VolumePersistence = None + persistence: Optional[VolumePersistence] = None name: Optional[str] = None - size_mib: PersistentVolumeSizeMib = 0 + size_mib: Optional[PersistentVolumeSizeMib] = PersistentVolumeSizeMib( + persistent_volume_size=0 + ) - @field_validator('size_mib', mode="before") + @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): if isinstance(v, int): return PersistentVolumeSizeMib(persistent_volume_size=v) diff --git a/aleph_message/models/item_hash.py b/aleph_message/models/item_hash.py index 7037a99..e4074c5 100644 --- a/aleph_message/models/item_hash.py +++ b/aleph_message/models/item_hash.py @@ -2,9 +2,10 @@ from functools import lru_cache from ..exceptions import UnknownHashError -from pydantic_core import CoreSchema, core_schema +from pydantic_core import core_schema from pydantic import GetCoreSchemaHandler + class ItemType(str, Enum): """Item storage options""" @@ -46,14 +47,15 @@ def __new__(cls, value: str): return obj @classmethod - def __get_pydantic_core_schema__(cls, source, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + def __get_pydantic_core_schema__( + cls, source, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: # This function validates the input after the initial type validation (as a string). # The returned value from this function will be used as the final validated value. # Return a string schema and add a post-validation function to convert to ItemHash return core_schema.no_info_after_validator_function( - cls.validate, - core_schema.str_schema() + cls.validate, core_schema.str_schema() ) @classmethod diff --git a/aleph_message/tests/test_types.py b/aleph_message/tests/test_types.py index f0a2dbf..1e8c03f 100644 --- a/aleph_message/tests/test_types.py +++ b/aleph_message/tests/test_types.py @@ -1,7 +1,7 @@ import copy import pytest -from pydantic import BaseModel, ValidationError, field_validator +from pydantic import BaseModel, ValidationError from aleph_message.exceptions import UnknownHashError from aleph_message.models import ItemHash, ItemType From 058b2f5014470787f686f3222d810f6558d307ea Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 2 Oct 2024 11:58:43 +0900 Subject: [PATCH 07/27] Fix: Comply with Black --- aleph_message/tests/test_models.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index b9ac15f..bc3ac68 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -297,8 +297,9 @@ def test_message_machine_named(): message = create_message_from_file(path, factory=ProgramMessage) assert isinstance(message, ProgramMessage) - assert isinstance(message.content.metadata, dict) - assert message.content.metadata["version"] == "10.2" + if message.content is not None: + assert isinstance(message.content.metadata, dict) + assert message.content.metadata["version"] == "10.2" def test_message_forget(): @@ -330,7 +331,9 @@ def test_message_forgotten_by(): _ = ProgramMessage.model_validate(message_raw) _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": None}) _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": ["abcde"]}) - _ = ProgramMessage.model_validate({**message_raw, "forgotten_by": ["abcde", "fghij"]}) + _ = ProgramMessage.model_validate( + {**message_raw, "forgotten_by": ["abcde", "fghij"]} + ) def test_item_type_from_hash(): From de538380bb452880ee49646fc9f6682ad67f8328 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 4 Oct 2024 11:19:11 +0900 Subject: [PATCH 08/27] Fix: Missing field caused by pydantic v2 Some fields were missing and needed to be specified du to the v2 of pydantic. --- aleph_message/models/__init__.py | 2 +- aleph_message/models/execution/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 32fdba7..23273a2 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -144,7 +144,7 @@ class StoreContent(BaseContent): size: Optional[int] = None # Generated by the node on storage content_type: Optional[str] = None # Generated by the node on storage ref: Optional[str] = None - metadata: Optional[Dict[str, Any]] = Field(description="Metadata of the VM") + metadata: Optional[Dict[str, Any]] = Field(default=None, description="Metadata of the VM") model_config = ConfigDict(extra="allow") diff --git a/aleph_message/models/execution/base.py b/aleph_message/models/execution/base.py index c139dda..be3d551 100644 --- a/aleph_message/models/execution/base.py +++ b/aleph_message/models/execution/base.py @@ -35,7 +35,7 @@ class Payment(HashableModel): chain: Chain """Which chain to check for funds""" - receiver: Optional[str] + receiver: Optional[str] = None """Optional alternative address to send tokens to""" type: PaymentType """Whether to pay by holding $ALEPH or by streaming tokens""" From 7aa06f3bb7cc1fd0c33527bfa0ffd1fdae4a081a Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 7 Oct 2024 09:33:49 +0900 Subject: [PATCH 09/27] Fix: Linter test did not pass and suggestion from Hugo Black test failed Wrong indentation on one line, add comment for custom_dump Refactoring normalize_content function from Hugo and added docstring for better comprehension --- aleph_message/models/__init__.py | 67 ++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 23273a2..34a45da 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -55,8 +55,7 @@ class MongodbId(BaseModel): oid: str = Field(alias="$oid") - -model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid") class ChainRef(BaseModel): @@ -144,7 +143,9 @@ class StoreContent(BaseContent): size: Optional[int] = None # Generated by the node on storage content_type: Optional[str] = None # Generated by the node on storage ref: Optional[str] = None - metadata: Optional[Dict[str, Any]] = Field(default=None, description="Metadata of the VM") + metadata: Optional[Dict[str, Any]] = Field( + default=None, description="Metadata of the VM" + ) model_config = ConfigDict(extra="allow") @@ -267,6 +268,7 @@ def convert_float_to_datetime(cls, v, values): model_config = ConfigDict(extra="forbid") def custom_dump(self): + """Exclude MongoDB identifiers from dumps for historical reasons.""" return self.model_dump(exclude={"id_", "_id"}) @@ -313,32 +315,57 @@ class ProgramMessage(BaseMessage): @staticmethod def normalize_content(content: Union[Dict[str, Any], Any]) -> Any: + """ + Normalizes the structure of a dictionary (`content`) to ensure that its + values are correctly formatted and compatible with Pydantic V2 + This method handles specific cases where certain types + (such as `ItemHash`, `Enum`, `list`, and `dict`) require special + handling to align with the stricter requirements of Pydantic V2. + + - Converts `ItemHash` instances to their string representation. + - Converts `Enum` instances to their corresponding `value`. + - Processes lists: + - If the key is "volumes" and all elements are strings, the list is + left as is. + - Otherwise, it recursively normalizes each element of the list. + - Processes dictionaries: + - If the key is "size_mib", it extracts the first value from the + dictionary. + - Otherwise, it recursively normalizes the dictionary. + + Args: + content (Union[Dict[str, Any], Any]): The dictionary or other data + type to normalize. + + Returns: + Any: The normalized content, with appropriate transformations + applied to `ItemHash`, `Enum`, `list`, and `dict` values, ensuring + compatibility withPydantic V2. + """ if not isinstance(content, dict): return content - normalized_content: Dict[str, Any] = {} - - for key, value in content.items(): + def handle_value(key: str, value: Any) -> Any: if isinstance(value, ItemHash): - normalized_content[key] = str(value) + return str(value) elif isinstance(value, Enum): - normalized_content[key] = value.value + return value.value elif isinstance(value, list): - if key == "volumes" and all(isinstance(v, str) for v in value): - normalized_content[key] = value - else: - normalized_content[key] = [ - ProgramMessage.normalize_content(v) for v in value - ] + return ( + value + if key == "volumes" and all(isinstance(v, str) for v in value) + else [ProgramMessage.normalize_content(v) for v in value] + ) elif isinstance(value, dict): - if key == "size_mib": - normalized_content[key] = list(value.values())[0] - else: - normalized_content[key] = ProgramMessage.normalize_content(value) + return ( + list(value.values())[0] + if key == "size_mib" + else ProgramMessage.normalize_content(value) + ) else: - normalized_content[key] = value + return value - return normalized_content + return {key: handle_value(key, value) for key, value in content.items()} @field_validator("content") def check_content(cls, v, values): From 3b333843a307bb62010c4f965f81fdb0ea0077c4 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 14 Oct 2024 12:34:50 +0900 Subject: [PATCH 10/27] Fix: Replacing print by logger and remove unecessary workaround Debuging using logger and not print Function to convert size always return the instance needed. --- aleph_message/models/__init__.py | 14 ++++++++++---- aleph_message/models/execution/instance.py | 4 +--- aleph_message/models/execution/volume.py | 8 ++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 34a45da..eb1fcdb 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -17,6 +17,12 @@ from .execution.program import ProgramContent from .item_hash import ItemHash, ItemType +import logging + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + __all__ = [ "AggregateContent", "AggregateMessage", @@ -377,10 +383,10 @@ def check_content(cls, v, values): normalized_content = cls.normalize_content(v.model_dump(exclude_none=True)) if normalized_content != item_content: - # Print les différences - print("Differences found between content and item_content") - print(f"Content: {normalized_content}") - print(f"Item Content: {item_content}") + # Print the differences to help debugging + logger.debug("Differences found between content and item_content") + logger.debug(f"Content: {normalized_content}") + logger.debug(f"Item Content: {item_content}") raise ValueError("Content and item_content differ") return v diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index afd9007..760ec89 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -27,9 +27,7 @@ class RootfsVolume(HashableModel): @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): - if isinstance(v, int): - return PersistentVolumeSizeMib(persistent_volume_size=v) - return v + return PersistentVolumeSizeMib(persistent_volume_size=v) class InstanceContent(BaseExecutableContent): diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 720601e..7e4891a 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -42,9 +42,7 @@ class EphemeralVolume(AbstractVolume): @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): - if isinstance(v, int): - return EphemeralVolumeSize(ephemeral_volume_size=v) - return v + return EphemeralVolumeSize(ephemeral_volume_size=v) def is_read_only(self): return False @@ -80,9 +78,7 @@ class PersistentVolume(AbstractVolume): @field_validator("size_mib", mode="before") def convert_size_mib(cls, v): - if isinstance(v, int): - return PersistentVolumeSizeMib(persistent_volume_size=v) - return v + return PersistentVolumeSizeMib(persistent_volume_size=v) def is_read_only(self): return False From 176ecb4fd5691f59dde1a8616b28fbb12120f52e Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Oct 2024 14:53:12 +0200 Subject: [PATCH 11/27] Refactor use of field size constraints --- aleph_message/models/execution/instance.py | 9 +++-- aleph_message/models/execution/volume.py | 25 ++++--------- aleph_message/tests/test_models.py | 42 +++++++++++++++++++++- 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 760ec89..99f65a3 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -5,6 +5,7 @@ from aleph_message.models.abstract import HashableModel +from ...utils import Gigabytes, gigabyte_to_mebibyte from .abstract import BaseExecutableContent from .environment import HypervisorType, InstanceEnvironment from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence @@ -22,13 +23,11 @@ class RootfsVolume(HashableModel): parent: ParentVolume persistence: VolumePersistence # Use the same size constraint as persistent volumes for now - size_mib: PersistentVolumeSizeMib + size_mib: int = Field( + gt=-1, le=gigabyte_to_mebibyte(Gigabytes(100)), strict=True # Limit to 1GiB + ) forgotten_by: Optional[List[str]] = None - @field_validator("size_mib", mode="before") - def convert_size_mib(cls, v): - return PersistentVolumeSizeMib(persistent_volume_size=v) - class InstanceContent(BaseExecutableContent): """Message content for scheduling a VM instance on the network.""" diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 7e4891a..51ef22c 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Literal, Optional, Union -from pydantic import Field, ConfigDict, BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from ...utils import Gigabytes, gigabyte_to_mebibyte from ..abstract import HashableModel @@ -29,20 +29,11 @@ def is_read_only(self): return True -class EphemeralVolumeSize(BaseModel): - ephemeral_volume_size: int = Field(gt=-1, le=1000, strict=True) # Limit to 1GiB - - def __hash__(self): - return hash(self.ephemeral_volume_size) - - class EphemeralVolume(AbstractVolume): ephemeral: Literal[True] = True - size_mib: EphemeralVolumeSize = EphemeralVolumeSize(ephemeral_volume_size=0) - - @field_validator("size_mib", mode="before") - def convert_size_mib(cls, v): - return EphemeralVolumeSize(ephemeral_volume_size=v) + size_mib: int = Field( + gt=0, le=gigabyte_to_mebibyte(Gigabytes(1)), strict=True # Limit to 1GiB + ) def is_read_only(self): return False @@ -72,14 +63,10 @@ class PersistentVolume(AbstractVolume): parent: Optional[ParentVolume] = None persistence: Optional[VolumePersistence] = None name: Optional[str] = None - size_mib: Optional[PersistentVolumeSizeMib] = PersistentVolumeSizeMib( - persistent_volume_size=0 + size_mib: int = Field( + gt=0, le=gigabyte_to_mebibyte(Gigabytes(100)), strict=True # Limit to 100GiB ) - @field_validator("size_mib", mode="before") - def convert_size_mib(cls, v): - return PersistentVolumeSizeMib(persistent_volume_size=v) - def is_read_only(self): return False diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index bc3ac68..1f6aae9 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -6,6 +6,7 @@ import pytest import requests +from functools import partial from pydantic import ValidationError from rich.console import Console @@ -26,10 +27,15 @@ create_message_from_file, create_message_from_json, create_new_message, - parse_message, + parse_message, ItemHash, ) from aleph_message.models.execution.environment import AMDSEVPolicy, HypervisorType +from aleph_message.models.execution.instance import RootfsVolume +from aleph_message.models.execution.volume import (EphemeralVolume, + ParentVolume, + VolumePersistence) from aleph_message.tests.download_messages import MESSAGES_STORAGE_PATH +from aleph_message.utils import Gigabytes, Mebibytes, gigabyte_to_mebibyte console = Console(color_system="windows") @@ -391,6 +397,40 @@ def test_create_new_message(): assert create_message_from_json(json.dumps(message_dict)) +def test_volume_size_constraints(): + """Test size constraints for volumes""" + + _ = EphemeralVolume(size_mib=1) + # A ValidationError should be raised if the size negative + with pytest.raises(ValidationError): + _ = EphemeralVolume(size_mib=-1) + size_mib: Mebibytes = gigabyte_to_mebibyte(Gigabytes(1)) + # A size of 1GiB should be allowed + _ = EphemeralVolume(size_mib=size_mib) + # A ValidationError should be raised if the size is greater than 1GiB + with pytest.raises(ValidationError): + _ = EphemeralVolume(size_mib=size_mib + 1) + + # Use partial function to avoid repeating the same code + create_test_rootfs = partial( + RootfsVolume, + parent=ParentVolume(ref=ItemHash("QmX8K1c22WmQBAww5ShWQqwMiFif7XFrJD6iFBj7skQZXW")), + persistence=VolumePersistence.store, + ) + + _ = create_test_rootfs(size_mib=1) + + # A ValidationError should be raised if the size negative + with pytest.raises(ValidationError): + _ = create_test_rootfs(size_mib=-1) + size_mib_rootfs: Mebibytes = gigabyte_to_mebibyte(Gigabytes(100)) + # A size of 100GiB should be allowed + _ = create_test_rootfs(size_mib=size_mib_rootfs) + # A ValidationError should be raised if the size is greater than 100GiB + with pytest.raises(ValidationError): + _ = create_test_rootfs(size_mib=size_mib_rootfs + 1) + + @pytest.mark.slow @pytest.mark.skipif(not isdir(MESSAGES_STORAGE_PATH), reason="No file on disk to test") def test_messages_from_disk(): From d73432a6444bb15096838d5153af7e54958cc339 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Oct 2024 14:53:40 +0200 Subject: [PATCH 12/27] Cleanup with black, isort --- aleph_message/models/__init__.py | 7 +++---- aleph_message/models/execution/instance.py | 9 ++++++--- aleph_message/models/execution/volume.py | 2 +- aleph_message/models/item_hash.py | 5 +++-- aleph_message/tests/test_models.py | 10 ++++++---- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index eb1fcdb..10ec451 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -1,13 +1,14 @@ import datetime -from enum import Enum import json +import logging from copy import copy +from enum import Enum from hashlib import sha256 from json import JSONDecodeError from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Type, TypeVar, Union, cast -from pydantic import BaseModel, Field, field_validator, ConfigDict +from pydantic import BaseModel, ConfigDict, Field, field_validator from typing_extensions import TypeAlias from .abstract import BaseContent, HashableModel @@ -17,8 +18,6 @@ from .execution.program import ProgramContent from .item_hash import ItemHash, ItemType -import logging - logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 99f65a3..6fc543e 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -1,15 +1,18 @@ from __future__ import annotations -from typing import Optional, List -from pydantic import Field, field_validator +from typing import List, Optional + +from pydantic import Field from aleph_message.models.abstract import HashableModel from ...utils import Gigabytes, gigabyte_to_mebibyte from .abstract import BaseExecutableContent from .environment import HypervisorType, InstanceEnvironment -from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence +from .volume import ParentVolume, VolumePersistence from .base import Payment +from .environment import InstanceEnvironment +from .volume import ParentVolume, VolumePersistence class RootfsVolume(HashableModel): diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 51ef22c..c9dd735 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import ConfigDict, Field from ...utils import Gigabytes, gigabyte_to_mebibyte from ..abstract import HashableModel diff --git a/aleph_message/models/item_hash.py b/aleph_message/models/item_hash.py index e4074c5..6f47600 100644 --- a/aleph_message/models/item_hash.py +++ b/aleph_message/models/item_hash.py @@ -1,9 +1,10 @@ from enum import Enum from functools import lru_cache -from ..exceptions import UnknownHashError -from pydantic_core import core_schema from pydantic import GetCoreSchemaHandler +from pydantic_core import core_schema + +from ..exceptions import UnknownHashError class ItemType(str, Enum): diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 1f6aae9..58cebd1 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -1,12 +1,12 @@ import json import os.path +from functools import partial from os import listdir from os.path import isdir, join from pathlib import Path import pytest import requests -from functools import partial from pydantic import ValidationError from rich.console import Console @@ -31,9 +31,11 @@ ) from aleph_message.models.execution.environment import AMDSEVPolicy, HypervisorType from aleph_message.models.execution.instance import RootfsVolume -from aleph_message.models.execution.volume import (EphemeralVolume, - ParentVolume, - VolumePersistence) +from aleph_message.models.execution.volume import ( + EphemeralVolume, + ParentVolume, + VolumePersistence, +) from aleph_message.tests.download_messages import MESSAGES_STORAGE_PATH from aleph_message.utils import Gigabytes, Mebibytes, gigabyte_to_mebibyte From 8c326e898d634bf1da8f194ae07291817e4979e0 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 16 Oct 2024 15:39:35 +0200 Subject: [PATCH 13/27] Fix: Simplify model content validation --- aleph_message/models/__init__.py | 74 ++++-------------------------- aleph_message/models/item_hash.py | 5 ++ aleph_message/tests/test_models.py | 36 ++++++++++++++- 3 files changed, 48 insertions(+), 67 deletions(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 10ec451..6d019af 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -2,7 +2,6 @@ import json import logging from copy import copy -from enum import Enum from hashlib import sha256 from json import JSONDecodeError from pathlib import Path @@ -318,74 +317,19 @@ class ProgramMessage(BaseMessage): content: ProgramContent forgotten_by: Optional[List[str]] = None - @staticmethod - def normalize_content(content: Union[Dict[str, Any], Any]) -> Any: - """ - Normalizes the structure of a dictionary (`content`) to ensure that its - values are correctly formatted and compatible with Pydantic V2 - This method handles specific cases where certain types - (such as `ItemHash`, `Enum`, `list`, and `dict`) require special - handling to align with the stricter requirements of Pydantic V2. - - - Converts `ItemHash` instances to their string representation. - - Converts `Enum` instances to their corresponding `value`. - - Processes lists: - - If the key is "volumes" and all elements are strings, the list is - left as is. - - Otherwise, it recursively normalizes each element of the list. - - Processes dictionaries: - - If the key is "size_mib", it extracts the first value from the - dictionary. - - Otherwise, it recursively normalizes the dictionary. - - Args: - content (Union[Dict[str, Any], Any]): The dictionary or other data - type to normalize. - - Returns: - Any: The normalized content, with appropriate transformations - applied to `ItemHash`, `Enum`, `list`, and `dict` values, ensuring - compatibility withPydantic V2. - """ - if not isinstance(content, dict): - return content - - def handle_value(key: str, value: Any) -> Any: - if isinstance(value, ItemHash): - return str(value) - elif isinstance(value, Enum): - return value.value - elif isinstance(value, list): - return ( - value - if key == "volumes" and all(isinstance(v, str) for v in value) - else [ProgramMessage.normalize_content(v) for v in value] - ) - elif isinstance(value, dict): - return ( - list(value.values())[0] - if key == "size_mib" - else ProgramMessage.normalize_content(value) - ) - else: - return value - - return {key: handle_value(key, value) for key, value in content.items()} - @field_validator("content") def check_content(cls, v, values): + """Ensure that the content of the message is correctly formatted.""" item_type = values.data.get("item_type") if item_type == ItemType.inline: + # Ensure that the content correct JSON item_content = json.loads(values.data.get("item_content")) - - # Normalizing content to fit the structure of item_content - normalized_content = cls.normalize_content(v.model_dump(exclude_none=True)) - - if normalized_content != item_content: - # Print the differences to help debugging - logger.debug("Differences found between content and item_content") - logger.debug(f"Content: {normalized_content}") - logger.debug(f"Item Content: {item_content}") + # Ensure that the content matches the expected structure + if v.model_dump(exclude_none=True) != item_content: + logger.warning( + "Content and item_content differ for message %s", + values.data["item_hash"], + ) raise ValueError("Content and item_content differ") return v @@ -435,7 +379,7 @@ def parse_message(message_dict: Dict) -> AlephMessage: raise ValueError(f"Unknown message type {message_dict['type']}") -def add_item_content_and_hash(message_dict: Dict, inplace: bool = False): +def add_item_content_and_hash(message_dict: Dict, inplace: bool = False) -> Dict: if not inplace: message_dict = copy(message_dict) diff --git a/aleph_message/models/item_hash.py b/aleph_message/models/item_hash.py index 6f47600..433daf6 100644 --- a/aleph_message/models/item_hash.py +++ b/aleph_message/models/item_hash.py @@ -2,6 +2,7 @@ from functools import lru_cache from pydantic import GetCoreSchemaHandler +from pydantic.functional_serializers import model_serializer from pydantic_core import core_schema from ..exceptions import UnknownHashError @@ -35,6 +36,10 @@ def is_storage(cls, item_hash: str): def is_ipfs(cls, item_hash: str): return cls.from_hash(item_hash) == cls.ipfs + @model_serializer + def __str__(self): + return self.value + class ItemHash(str): item_type: ItemType diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 58cebd1..e1b500e 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -4,6 +4,7 @@ from os import listdir from os.path import isdir, join from pathlib import Path +from unittest import mock import pytest import requests @@ -16,6 +17,7 @@ ForgetMessage, InstanceContent, InstanceMessage, + ItemHash, ItemType, MessagesResponse, MessageType, @@ -27,7 +29,7 @@ create_message_from_file, create_message_from_json, create_new_message, - parse_message, ItemHash, + parse_message, ) from aleph_message.models.execution.environment import AMDSEVPolicy, HypervisorType from aleph_message.models.execution.instance import RootfsVolume @@ -416,7 +418,9 @@ def test_volume_size_constraints(): # Use partial function to avoid repeating the same code create_test_rootfs = partial( RootfsVolume, - parent=ParentVolume(ref=ItemHash("QmX8K1c22WmQBAww5ShWQqwMiFif7XFrJD6iFBj7skQZXW")), + parent=ParentVolume( + ref=ItemHash("QmX8K1c22WmQBAww5ShWQqwMiFif7XFrJD6iFBj7skQZXW") + ), persistence=VolumePersistence.store, ) @@ -433,6 +437,34 @@ def test_volume_size_constraints(): _ = create_test_rootfs(size_mib=size_mib_rootfs + 1) +def test_program_message_content_and_item_content_differ(): + # Test that a ValidationError is raised if the content and item_content differ + + # Get a program message as JSON-compatible dict + path = Path(__file__).parent / "messages/machine.json" + with open(path) as fd: + message_dict_original = json.load(fd) + message_dict: dict = add_item_content_and_hash(message_dict_original, inplace=True) + + # patch hashlib.sha256 with a mock else this raises an error first + mock_hash = mock.MagicMock() + mock_hash.hexdigest.return_value = ( + "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe" + ) + message_dict["item_hash"] = ( + "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe" + ) + + # Patch the content to differ from item_content + message_dict["content"]["replaces"] = "does-not-exist" + + # Test that a ValidationError is raised if the content and item_content differ + with mock.patch("aleph_message.models.sha256", return_value=mock_hash): + with pytest.raises(ValidationError) as excinfo: + ProgramMessage.model_validate(message_dict) + assert "Content and item_content differ" in str(excinfo.value) + + @pytest.mark.slow @pytest.mark.skipif(not isdir(MESSAGES_STORAGE_PATH), reason="No file on disk to test") def test_messages_from_disk(): From 01087fda8313707e67ea7c8a941b78a326074c83 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:29:17 +0900 Subject: [PATCH 14/27] del: Deleting setup.py, now using pyproject.toml --- setup.py | 63 -------------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 89e0f21..0000000 --- a/setup.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -"""Aleph Message - Python library for the Aleph.im message specification -(c) 2022 OKESO for Aleph.im -""" - -import os -import re - - -def get_version(): - version_file = os.path.join("aleph_message", "__init__.py") - initfile_lines = open(version_file, "rt").readlines() - version = r"^__version__ = ['\"]([^'\"]*)['\"]" - - for line in initfile_lines: - mo = re.search(version, line, re.M) - if mo: - return mo.group(1) - raise RuntimeError(f"Unable to find version string in {version_file}.") - - -from setuptools import setup - -# allow setup.py to be run from any path -os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) - -with open("README.md") as file: - long_description = file.read() - -setup( - name="aleph-message", - version=get_version(), - description="Aleph.im message specification ", - long_description=long_description, - long_description_content_type="text/markdown", - author="Hugo Herter", - author_email="git@hugoherter.com", - url="https://github.com/aleph-im/aleph-message", - packages=[ - "aleph_message", - "aleph_message.models", - "aleph_message.models.execution", - ], - package_data={ - "aleph_message": ["py.typed"], - "aleph_message.models": ["py.typed"], - "aleph_message.models.execution": ["py.typed"], - }, - data_files=[], - install_requires=[ - "pydantic>=2", - "typing_extensions>=4.5.0", - ], - license="MIT", - platform="any", - keywords="aleph.im message validation specification", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Intended Audience :: Developers", - "Topic :: System :: Distributed Computing", - ], -) From ae27065d7d282012751b70ba397d084314ba8259 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:30:13 +0900 Subject: [PATCH 15/27] Fix: Missing dependencies after migrating to pydantic v2 Updating the version of Pydantic, and adding pydantic-core Upgrading yamlfix because it is not compatible with pydantic2 --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 03ca350..3968a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ classifiers = [ ] dynamic = [ "version" ] dependencies = [ - "pydantic>=1.10.5,<2", + "pydantic>=2", + "pydantic-core>=2", "typing-extensions>=4.5", ] urls.Documentation = "https://aleph.im/" @@ -85,7 +86,9 @@ dependencies = [ "ruff==0.4.8", "isort==5.13.2", "check-sdist==0.1.3", - "yamlfix==1.16.1", + "yamlfix==1.17.0", + "pydantic>=2", + "pydantic-core>=2", "pyproject-fmt==2.2.1", "types-requests", "typing-extensions", From 8b9264f7d1b5d24c1fd592ab30b1f0a65272046b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:38:59 +0900 Subject: [PATCH 16/27] Style: MyPy raised note about untyped function Adding a type none to the concerned functions --- aleph_message/tests/test_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index e1b500e..f8a8612 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -401,7 +401,7 @@ def test_create_new_message(): assert create_message_from_json(json.dumps(message_dict)) -def test_volume_size_constraints(): +def test_volume_size_constraints() -> None: """Test size constraints for volumes""" _ = EphemeralVolume(size_mib=1) @@ -437,7 +437,7 @@ def test_volume_size_constraints(): _ = create_test_rootfs(size_mib=size_mib_rootfs + 1) -def test_program_message_content_and_item_content_differ(): +def test_program_message_content_and_item_content_differ() -> None: # Test that a ValidationError is raised if the content and item_content differ # Get a program message as JSON-compatible dict From 7066c2dc76743bd7f2ae0f64355641e15eb39183 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:40:55 +0900 Subject: [PATCH 17/27] Fix: Refactor ItemHash for Pydantic v2 compatibility - Removing @model_serializer on ItemType - Updated __get_pydantic_core_schema__ signature for Pydantic v2 - Added __get_pydantic_json_schema__ for JSON Schema generation - Annotated validate and __repr__ methods for better type safety --- aleph_message/models/item_hash.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/aleph_message/models/item_hash.py b/aleph_message/models/item_hash.py index 433daf6..353fc7d 100644 --- a/aleph_message/models/item_hash.py +++ b/aleph_message/models/item_hash.py @@ -1,8 +1,7 @@ from enum import Enum from functools import lru_cache +from typing import Any -from pydantic import GetCoreSchemaHandler -from pydantic.functional_serializers import model_serializer from pydantic_core import core_schema from ..exceptions import UnknownHashError @@ -36,10 +35,6 @@ def is_storage(cls, item_hash: str): def is_ipfs(cls, item_hash: str): return cls.from_hash(item_hash) == cls.ipfs - @model_serializer - def __str__(self): - return self.value - class ItemHash(str): item_type: ItemType @@ -54,21 +49,23 @@ def __new__(cls, value: str): @classmethod def __get_pydantic_core_schema__( - cls, source, handler: GetCoreSchemaHandler + cls, source: type[Any], handler: core_schema.ValidatorFunctionWrapHandler ) -> core_schema.CoreSchema: - # This function validates the input after the initial type validation (as a string). - # The returned value from this function will be used as the final validated value. - - # Return a string schema and add a post-validation function to convert to ItemHash + """Pydantic v2 - Validation Schema""" return core_schema.no_info_after_validator_function( cls.validate, core_schema.str_schema() ) @classmethod - def validate(cls, v): + def __get_pydantic_json_schema__(cls, schema) -> dict[str, Any]: + """Pydantic v2 - JSON Schema Generation""" + return {"type": "string"} + + @classmethod + def validate(cls, v: Any) -> "ItemHash": if not isinstance(v, str): raise TypeError("Item hash must be a string") return cls(v) # Convert to ItemHash - def __repr__(self): + def __repr__(self) -> str: return f"" From 2250d2aa4fe55a3932b411742b7d20ce870582c4 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:41:52 +0900 Subject: [PATCH 18/27] Fix: Refactoring PersistentVolumeSizeMib for Pydantic2 compatibility Using Annoted instead of ConstrainedId because is no longer available in Pydantic2 --- aleph_message/models/execution/volume.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index c9dd735..7ce260d 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -2,9 +2,9 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Literal, Optional, Union +from typing import Annotated, Literal, Optional, Union -from pydantic import ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field from ...utils import Gigabytes, gigabyte_to_mebibyte from ..abstract import HashableModel @@ -53,10 +53,10 @@ class VolumePersistence(str, Enum): store = "store" -class PersistentVolumeSizeMib(ConstrainedInt): - gt = 0 - le = gigabyte_to_mebibyte(Gigabytes(2048)) - strict = True # Limit to 2048 GiB +class PersistentVolumeSizeMib(BaseModel): + size_mib: Annotated[ + int, Field(gt=0, le=gigabyte_to_mebibyte(2048), strict=True) + ] # Limit to 2048 GiB class PersistentVolume(AbstractVolume): From 9e105ad3d51fea781b2546d514f1fd551fe84b22 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:43:53 +0900 Subject: [PATCH 19/27] Fix: Some parts of InstanceContent were incompatible with pydantic2 - Can't use get() to access InstanceContent data, instead we directly access the element by using .data for exemple - root_validator is deprecated, use of model_validator instead --- aleph_message/models/execution/instance.py | 30 ++++++++++------------ 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 6fc543e..70c3ed5 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -2,16 +2,14 @@ from typing import List, Optional -from pydantic import Field +from pydantic import Field, model_validator from aleph_message.models.abstract import HashableModel from ...utils import Gigabytes, gigabyte_to_mebibyte from .abstract import BaseExecutableContent -from .environment import HypervisorType, InstanceEnvironment -from .volume import ParentVolume, VolumePersistence from .base import Payment -from .environment import InstanceEnvironment +from .environment import HypervisorType, InstanceEnvironment from .volume import ParentVolume, VolumePersistence @@ -44,36 +42,36 @@ class InstanceContent(BaseExecutableContent): description="Root filesystem of the system, will be booted by the kernel" ) - @root_validator() + @model_validator(mode="after") def check_requirements(cls, values): - if values.get("requirements"): + if values.requirements: # GPU filter only supported for QEmu instances with node_hash assigned - if values.get("requirements").gpu: + if values.requirements.gpu: if ( - not values.get("requirements").node - or not values.get("requirements").node.node_hash + not values.requirements.node + or not values.requirements.node.node_hash ): raise ValueError("Node hash assignment is needed for GPU support") if ( - values.get("environment") - and values.get("environment").hypervisor != HypervisorType.qemu + values.environment + and values.environment.hypervisor != HypervisorType.qemu ): raise ValueError("GPU option is only supported for QEmu hypervisor") # Terms and conditions filter only supported for PAYG/coco instances with node_hash assigned if ( - values.get("requirements").node - and values.get("requirements").node.terms_and_conditions + values.requirements.node + and values.requirements.node.terms_and_conditions ): - if not values.get("requirements").node.node_hash: + if not values.requirements.node.node_hash: raise ValueError( "Terms_and_conditions field needs a requirements.node.node_hash value" ) if ( - not values.get("payment").is_stream - and not values.get("environment").trusted_execution + not values.payment.is_stream + and not values.environment.trusted_execution ): raise ValueError( "Only PAYG/coco instances can have a terms_and_conditions" From 05dde6d78b26137aa95b03f14039a7021db83fa3 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:47:15 +0900 Subject: [PATCH 20/27] Fix: Extra forbid don't work the same on pydantic2, refactoring the right way --- aleph_message/models/execution/environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index c517fc8..f18b7b9 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -103,8 +103,7 @@ class GpuProperties(HashableModel): ) device_id: str = Field(description="GPU vendor & device ids") - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class HypervisorType(str, Enum): From 2a3c5994d6ad9d063ed4dc893f13a9f7f2fcf720 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 18:58:45 +0900 Subject: [PATCH 21/27] Fix: Fixing test to match Pydantic v2 error format - Pydantic v2 changed how validation errors are structured - `loc` can now be either `("content",)` or `("content", "__root__")` - Updated test to handle both cases and prevent failures - Now checks only the error message to avoid issues with prefix changes --- aleph_message/tests/test_models.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index f8a8612..f5dcc01 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -228,8 +228,12 @@ def test_validation_on_gpu_payment_options(): _ = create_new_message(message_dict, factory=InstanceMessage) raise AssertionError("An exception should have been raised before this point.") except ValidationError as e: - assert e.errors()[0]["loc"] == ("content", "__root__") - assert e.errors()[0]["msg"] == "Node hash assignment is needed for GPU support" + assert e.errors()[0]["loc"] in [("content",), ("content", "__root__")] + + error_msg = e.errors()[0]["msg"] + assert ( + "Node hash assignment is needed for GPU support" in error_msg + ) # Ignore "Value error, ..." def test_validation_on_gpu_hypervisor_options(): @@ -242,10 +246,12 @@ def test_validation_on_gpu_hypervisor_options(): _ = create_new_message(message_dict, factory=InstanceMessage) raise AssertionError("An exception should have been raised before this point.") except ValidationError as e: - assert e.errors()[0]["loc"] == ("content", "__root__") + assert e.errors()[0]["loc"] in [("content",), ("content", "__root__")] + + error_msg = e.errors()[0]["msg"] assert ( - e.errors()[0]["msg"] == "GPU option is only supported for QEmu hypervisor" - ) + "GPU option is only supported for QEmu hypervisor" in error_msg + ) # Ignore "Value error, ..." def test_message_machine_port_mapping(): From 6806fbcadb4e4d9ec0ea133f8c2e9c12e0be4612 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 25 Feb 2025 23:35:55 +0900 Subject: [PATCH 22/27] Fix: Missing field in InstantContent class Returning 'authorized_keys' inside InstantConstant class in the sdk, but there is no field 'authorized_keys' in the class Adding this field --- aleph_message/models/execution/instance.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 70c3ed5..01ba399 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -35,6 +35,9 @@ class InstanceContent(BaseExecutableContent): metadata: Optional[dict] = None payment: Optional[Payment] = None + authorized_keys: Optional[List[str]] = Field( + default=None, description="List of authorized SSH keys" + ) environment: InstanceEnvironment = Field( description="Properties of the instance execution environment" ) From cf2bf32af1336582f68daaab4b46a8e4f7c798af Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 14 Mar 2025 23:36:23 +0900 Subject: [PATCH 23/27] Fix: Increasing size for instances and volumes --- aleph_message/models/execution/instance.py | 5 +++-- aleph_message/models/execution/volume.py | 12 +++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 01ba399..460bb6a 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -10,7 +10,8 @@ from .abstract import BaseExecutableContent from .base import Payment from .environment import HypervisorType, InstanceEnvironment -from .volume import ParentVolume, VolumePersistence +from .volume import ParentVolume, VolumePersistence, PersistentVolume + class RootfsVolume(HashableModel): @@ -25,7 +26,7 @@ class RootfsVolume(HashableModel): persistence: VolumePersistence # Use the same size constraint as persistent volumes for now size_mib: int = Field( - gt=-1, le=gigabyte_to_mebibyte(Gigabytes(100)), strict=True # Limit to 1GiB + gt=0, le=gigabyte_to_mebibyte(Gigabytes(2048)), strict=True # Limit to 100GiB ) forgotten_by: Optional[List[str]] = None diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 7ce260d..b93e820 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -2,9 +2,9 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Annotated, Literal, Optional, Union +from typing import Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import ConfigDict, Field from ...utils import Gigabytes, gigabyte_to_mebibyte from ..abstract import HashableModel @@ -53,18 +53,12 @@ class VolumePersistence(str, Enum): store = "store" -class PersistentVolumeSizeMib(BaseModel): - size_mib: Annotated[ - int, Field(gt=0, le=gigabyte_to_mebibyte(2048), strict=True) - ] # Limit to 2048 GiB - - class PersistentVolume(AbstractVolume): parent: Optional[ParentVolume] = None persistence: Optional[VolumePersistence] = None name: Optional[str] = None size_mib: int = Field( - gt=0, le=gigabyte_to_mebibyte(Gigabytes(100)), strict=True # Limit to 100GiB + gt=0, le=gigabyte_to_mebibyte(Gigabytes(2048)), strict=True # Limit to 2048GiB ) def is_read_only(self): From b2d9572f4cfcd3f5ddc3ba441fd3fd4c456093e2 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 14 Mar 2025 23:37:15 +0900 Subject: [PATCH 24/27] fix: Adapt the tests after the increase of volumes and instances size --- aleph_message/tests/test_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index f5dcc01..ce11e2a 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -435,8 +435,8 @@ def test_volume_size_constraints() -> None: # A ValidationError should be raised if the size negative with pytest.raises(ValidationError): _ = create_test_rootfs(size_mib=-1) - size_mib_rootfs: Mebibytes = gigabyte_to_mebibyte(Gigabytes(100)) - # A size of 100GiB should be allowed + size_mib_rootfs: Mebibytes = gigabyte_to_mebibyte(Gigabytes(2048)) + # A size of 2048GiB should be allowed _ = create_test_rootfs(size_mib=size_mib_rootfs) # A ValidationError should be raised if the size is greater than 100GiB with pytest.raises(ValidationError): From ad9c940f3845da422a7b801647e43be53fca13ca Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 14 Mar 2025 23:39:24 +0900 Subject: [PATCH 25/27] style: ruff and black --- aleph_message/models/execution/instance.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 460bb6a..9a55af4 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -10,8 +10,7 @@ from .abstract import BaseExecutableContent from .base import Payment from .environment import HypervisorType, InstanceEnvironment -from .volume import ParentVolume, VolumePersistence, PersistentVolume - +from .volume import ParentVolume, VolumePersistence class RootfsVolume(HashableModel): From ae163300a3183bf9e5964335257b4339e7884c87 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Mon, 7 Apr 2025 19:53:27 +0200 Subject: [PATCH 26/27] Fix: Re-add Volume sizes classes as types instead removing it. --- aleph_message/models/execution/instance.py | 7 +--- aleph_message/models/execution/volume.py | 31 +++++++++++---- aleph_message/tests/test_models.py | 45 ---------------------- 3 files changed, 26 insertions(+), 57 deletions(-) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 9a55af4..8a4a44f 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -6,11 +6,10 @@ from aleph_message.models.abstract import HashableModel -from ...utils import Gigabytes, gigabyte_to_mebibyte from .abstract import BaseExecutableContent from .base import Payment from .environment import HypervisorType, InstanceEnvironment -from .volume import ParentVolume, VolumePersistence +from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence class RootfsVolume(HashableModel): @@ -24,9 +23,7 @@ class RootfsVolume(HashableModel): parent: ParentVolume persistence: VolumePersistence # Use the same size constraint as persistent volumes for now - size_mib: int = Field( - gt=0, le=gigabyte_to_mebibyte(Gigabytes(2048)), strict=True # Limit to 100GiB - ) + size_mib: PersistentVolumeSizeMib forgotten_by: Optional[List[str]] = None diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index b93e820..b72ef2d 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Literal, Optional, Union +from typing import Annotated, Literal, Optional, Union from pydantic import ConfigDict, Field @@ -29,11 +29,19 @@ def is_read_only(self): return True +EphemeralVolumeSize = Annotated[ + int, + Field( + gt=0, + le=1000, # Limit to 1 GiB + strict=True, + ), +] + + class EphemeralVolume(AbstractVolume): ephemeral: Literal[True] = True - size_mib: int = Field( - gt=0, le=gigabyte_to_mebibyte(Gigabytes(1)), strict=True # Limit to 1GiB - ) + size_mib: EphemeralVolumeSize def is_read_only(self): return False @@ -53,13 +61,22 @@ class VolumePersistence(str, Enum): store = "store" +# Define a type for persistent volume size with constraints +PersistentVolumeSizeMib = Annotated[ + int, + Field( + gt=0, + le=gigabyte_to_mebibyte(Gigabytes(2048)), + strict=True, # Limit to 2048 GiB + ), +] + + class PersistentVolume(AbstractVolume): parent: Optional[ParentVolume] = None persistence: Optional[VolumePersistence] = None name: Optional[str] = None - size_mib: int = Field( - gt=0, le=gigabyte_to_mebibyte(Gigabytes(2048)), strict=True # Limit to 2048GiB - ) + size_mib: PersistentVolumeSizeMib def is_read_only(self): return False diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index ce11e2a..7b5cdc7 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -1,6 +1,5 @@ import json import os.path -from functools import partial from os import listdir from os.path import isdir, join from pathlib import Path @@ -17,7 +16,6 @@ ForgetMessage, InstanceContent, InstanceMessage, - ItemHash, ItemType, MessagesResponse, MessageType, @@ -32,14 +30,7 @@ parse_message, ) from aleph_message.models.execution.environment import AMDSEVPolicy, HypervisorType -from aleph_message.models.execution.instance import RootfsVolume -from aleph_message.models.execution.volume import ( - EphemeralVolume, - ParentVolume, - VolumePersistence, -) from aleph_message.tests.download_messages import MESSAGES_STORAGE_PATH -from aleph_message.utils import Gigabytes, Mebibytes, gigabyte_to_mebibyte console = Console(color_system="windows") @@ -407,42 +398,6 @@ def test_create_new_message(): assert create_message_from_json(json.dumps(message_dict)) -def test_volume_size_constraints() -> None: - """Test size constraints for volumes""" - - _ = EphemeralVolume(size_mib=1) - # A ValidationError should be raised if the size negative - with pytest.raises(ValidationError): - _ = EphemeralVolume(size_mib=-1) - size_mib: Mebibytes = gigabyte_to_mebibyte(Gigabytes(1)) - # A size of 1GiB should be allowed - _ = EphemeralVolume(size_mib=size_mib) - # A ValidationError should be raised if the size is greater than 1GiB - with pytest.raises(ValidationError): - _ = EphemeralVolume(size_mib=size_mib + 1) - - # Use partial function to avoid repeating the same code - create_test_rootfs = partial( - RootfsVolume, - parent=ParentVolume( - ref=ItemHash("QmX8K1c22WmQBAww5ShWQqwMiFif7XFrJD6iFBj7skQZXW") - ), - persistence=VolumePersistence.store, - ) - - _ = create_test_rootfs(size_mib=1) - - # A ValidationError should be raised if the size negative - with pytest.raises(ValidationError): - _ = create_test_rootfs(size_mib=-1) - size_mib_rootfs: Mebibytes = gigabyte_to_mebibyte(Gigabytes(2048)) - # A size of 2048GiB should be allowed - _ = create_test_rootfs(size_mib=size_mib_rootfs) - # A ValidationError should be raised if the size is greater than 100GiB - with pytest.raises(ValidationError): - _ = create_test_rootfs(size_mib=size_mib_rootfs + 1) - - def test_program_message_content_and_item_content_differ() -> None: # Test that a ValidationError is raised if the content and item_content differ From b4e622cf283aaaaa33b0b259c2ae84c0acb32eb4 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Mon, 7 Apr 2025 21:07:10 +0200 Subject: [PATCH 27/27] Fix: Removed `basicConfig` call. --- aleph_message/models/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 6d019af..5b43e98 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -17,7 +17,6 @@ from .execution.program import ProgramContent from .item_hash import ItemHash, ItemType -logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__)