diff --git a/CHANGELOG.md b/CHANGELOG.md index 6819cde90..288db6e9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - Series values DTO conversion reworked with protocol buffer support ([#1738](https://github.com/neptune-ai/neptune-client/pull/1738)) - Series values fetching reworked with protocol buffer support ([#1744](https://github.com/neptune-ai/neptune-client/pull/1744)) - Added support for enhanced field definitions querying ([#1751](https://github.com/neptune-ai/neptune-client/pull/1751)) +- Added initial operations to the `core.operations` package ([#1759](https://github.com/neptune-ai/neptune-client/pull/1759)) ### Fixes - Fixed `tqdm.notebook` import only in Notebook environment ([#1716](https://github.com/neptune-ai/neptune-client/pull/1716)) diff --git a/src/neptune/core/operations/__init__.py b/src/neptune/core/operations/__init__.py new file mode 100644 index 000000000..b402fcf37 --- /dev/null +++ b/src/neptune/core/operations/__init__.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2024, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__all__ = [ + "AssignInt", + "AssignFloat", + "AssignBool", + "AssignString", + "AssignDatetime", + "LogFloats", +] + +from neptune.core.operations.operation import ( + AssignBool, + AssignDatetime, + AssignFloat, + AssignInt, + AssignString, + LogFloats, +) diff --git a/src/neptune/core/operations/operation.py b/src/neptune/core/operations/operation.py new file mode 100644 index 000000000..7363aaf0d --- /dev/null +++ b/src/neptune/core/operations/operation.py @@ -0,0 +1,207 @@ +# +# Copyright (c) 2024, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__all__ = [ + "AssignInt", + "AssignFloat", + "AssignBool", + "AssignString", + "AssignDatetime", + "LogFloats", + "Operation", +] + +import abc +from dataclasses import dataclass +from datetime import datetime +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + Generic, + List, + Optional, + Type, + TypeVar, + Union, +) + +from neptune.core.components.operation_storage import OperationStorage +from neptune.exceptions import MalformedOperation + +if TYPE_CHECKING: + from neptune.core.operations.operation_visitor import OperationVisitor + +Ret = TypeVar("Ret") +T = TypeVar("T") + + +@dataclass +class Operation(abc.ABC): + path: List[str] + _registry: ClassVar[Dict[str, Type["Operation"]]] = {} + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + cls._registry[cls.__name__] = cls + + @abc.abstractmethod + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + pass + + def clean(self, operation_storage: OperationStorage) -> None: + pass + + def to_dict(self) -> Dict[str, Any]: + return {"type": self.__class__.__name__, "path": self.path} + + @staticmethod + def from_dict(data: dict) -> "Operation": + if "type" not in data: + raise MalformedOperation("Malformed operation {} - type is missing".format(data)) + operation_type = data["type"] + if operation_type not in Operation._registry: + raise MalformedOperation("Malformed operation {} - unknown type {}".format(data, operation_type)) + return Operation._registry[operation_type].from_dict(data) + + +@dataclass +class AssignFloat(Operation): + value: float + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_assign_float(self) + + def to_dict(self) -> dict: + ret = super().to_dict() + ret["value"] = self.value + return ret + + @staticmethod + def from_dict(data: dict) -> "AssignFloat": + return AssignFloat(data["path"], data["value"]) + + +@dataclass +class AssignInt(Operation): + value: int + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_assign_int(self) + + def to_dict(self) -> dict: + ret = super().to_dict() + ret["value"] = self.value + return ret + + @staticmethod + def from_dict(data: dict) -> "AssignInt": + return AssignInt(data["path"], data["value"]) + + +@dataclass +class AssignBool(Operation): + value: bool + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_assign_bool(self) + + def to_dict(self) -> dict: + ret = super().to_dict() + ret["value"] = self.value + return ret + + @staticmethod + def from_dict(data: dict) -> "AssignBool": + return AssignBool(data["path"], data["value"]) + + +@dataclass +class AssignString(Operation): + value: str + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_assign_string(self) + + def to_dict(self) -> dict: + ret = super().to_dict() + ret["value"] = self.value + return ret + + @staticmethod + def from_dict(data: dict) -> "AssignString": + return AssignString(data["path"], data["value"]) + + +@dataclass +class AssignDatetime(Operation): + value: datetime + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_assign_datetime(self) + + def to_dict(self) -> dict: + ret = super().to_dict() + ret["value"] = int(1000 * self.value.timestamp()) + return ret + + @staticmethod + def from_dict(data: dict) -> "AssignDatetime": + return AssignDatetime(data["path"], datetime.fromtimestamp(data["value"] / 1000)) + + +class LogOperation(Operation, abc.ABC): + pass + + +@dataclass +class LogSeriesValue(Generic[T]): + value: T + step: Optional[float] + ts: float + + def to_dict( + self, + value_serializer: Callable[[T], Any] = lambda x: x, + ) -> Dict[str, Union[T, Optional[float], float]]: + return {"value": value_serializer(self.value), "step": self.step, "ts": self.ts} + + @staticmethod + def from_dict(data: Dict[str, Any], value_deserializer: Callable[[T], Any] = lambda x: x) -> "LogSeriesValue[T]": + return LogSeriesValue[T](value_deserializer(data["value"]), data.get("step"), data["ts"]) + + +@dataclass +class LogFloats(LogOperation): + ValueType = LogSeriesValue[float] + + values: List[ValueType] + + def accept(self, visitor: "OperationVisitor[Ret]") -> Ret: + return visitor.visit_log_floats(self) + + def to_dict(self) -> Dict[str, Any]: + ret = super().to_dict() + ret["values"] = [value.to_dict() for value in self.values] + return ret + + @staticmethod + def from_dict(data: dict) -> "LogFloats": + return LogFloats( + data["path"], + [LogFloats.ValueType.from_dict(value) for value in data["values"]], # type: ignore[misc] + ) diff --git a/src/neptune/core/operations/operation_visitor.py b/src/neptune/core/operations/operation_visitor.py new file mode 100644 index 000000000..2127b284f --- /dev/null +++ b/src/neptune/core/operations/operation_visitor.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2024, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__all__ = ["OperationVisitor"] + +import abc +from typing import ( + Generic, + TypeVar, +) + +from neptune.core.operations.operation import ( + AssignBool, + AssignDatetime, + AssignFloat, + AssignInt, + AssignString, + LogFloats, + Operation, +) + +Ret = TypeVar("Ret") + + +class OperationVisitor(Generic[Ret]): + def visit(self, op: Operation) -> Ret: + return op.accept(self) + + @abc.abstractmethod + def visit_assign_float(self, op: AssignFloat) -> Ret: + pass + + @abc.abstractmethod + def visit_assign_int(self, op: AssignInt) -> Ret: + pass + + @abc.abstractmethod + def visit_assign_bool(self, op: AssignBool) -> Ret: + pass + + @abc.abstractmethod + def visit_assign_string(self, op: AssignString) -> Ret: + pass + + @abc.abstractmethod + def visit_assign_datetime(self, op: AssignDatetime) -> Ret: + pass + + @abc.abstractmethod + def visit_log_floats(self, op: LogFloats) -> Ret: + pass diff --git a/tests/unit/neptune/new/core/operations/__init__.py b/tests/unit/neptune/new/core/operations/__init__.py new file mode 100644 index 000000000..665b8500e --- /dev/null +++ b/tests/unit/neptune/new/core/operations/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2024, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/neptune/new/core/operations/test_operations.py b/tests/unit/neptune/new/core/operations/test_operations.py new file mode 100644 index 000000000..6e84592d0 --- /dev/null +++ b/tests/unit/neptune/new/core/operations/test_operations.py @@ -0,0 +1,224 @@ +# +# Copyright (c) 2024, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import random +import string + +import pytest + +from neptune.core.operations import ( + AssignBool, + AssignDatetime, + AssignFloat, + AssignInt, + AssignString, + LogFloats, +) +from neptune.core.operations.operation import Operation +from neptune.exceptions import MalformedOperation + + +class TestAssignInt: + def test__assign_int_operation__to_dict(self): + # given + value = random.randint(int(-1e8), int(1e8)) + assign_int = AssignInt(["test", "path"], value) + + # when + data = assign_int.to_dict() + + # then + assert data == {"type": "AssignInt", "path": ["test", "path"], "value": value} + + def test__assign_int_operation__from_dict(self): + # given + value = random.randint(int(-1e8), int(1e8)) + data = {"type": "AssignInt", "path": ["test", "path"], "value": value} + + # when + assign_int = AssignInt.from_dict(data) + + # then + assert assign_int == AssignInt(["test", "path"], value) + + +class TestAssignString: + def test__assign_string_operation__to_dict(self): + # given + value = "".join(random.choices(string.ascii_lowercase, k=20)) + assign_string = AssignString(["test", "path"], value) + + # when + data = assign_string.to_dict() + + # then + assert data == {"type": "AssignString", "path": ["test", "path"], "value": value} + + def test__assign_string_operation__from_dict(self): + # given + value = "".join(random.choices(string.ascii_lowercase, k=20)) + data = {"type": "AssignString", "path": ["test", "path"], "value": value} + assign_string = AssignString.from_dict(data) + + # then + assert assign_string == AssignString(["test", "path"], value) + + +class TestAssignBool: + def test__assign_bool_operation__to_dict(self): + # given + value = random.choice([True, False]) + assign_bool = AssignBool(["test", "path"], value) + + # when + data = assign_bool.to_dict() + + # then + assert data == {"type": "AssignBool", "path": ["test", "path"], "value": value} + + def test__assign_bool_operation__from_dict(self): + # given + value = random.choice([True, False]) + data = {"type": "AssignBool", "path": ["test", "path"], "value": value} + + # when + assign_bool = AssignBool.from_dict(data) + + # then + assert assign_bool == AssignBool(["test", "path"], value) + + +class TestAssignFloat: + def test__assign_float_operation__to_dict(self): + # given + value = random.uniform(-1e8, 1e8) + assign_float = AssignFloat(["test", "path"], value) + + # when + data = assign_float.to_dict() + + # then + assert data == {"type": "AssignFloat", "path": ["test", "path"], "value": value} + + def test__assign_float_operation__from_dict(self): + # given + value = random.uniform(-1e8, 1e8) + data = {"type": "AssignFloat", "path": ["test", "path"], "value": value} + assign_float = AssignFloat.from_dict(data) + + # then + assert assign_float == AssignFloat(["test", "path"], value) + + +class TestAssignDatetime: + def test__assign_datetime_operation__to_dict(self): + # given + value = datetime.datetime.utcnow() + assign_datetime = AssignDatetime(["test", "path"], value) + + # when + data = assign_datetime.to_dict() + + # then + assert data == {"type": "AssignDatetime", "path": ["test", "path"], "value": int(value.timestamp() * 1000)} + + def test__assign_datetime_operation__from_dict(self): + # given + value = datetime.datetime.now().replace(microsecond=0) + value_as_int = int(value.timestamp() * 1000) + data = { + "type": "AssignDatetime", + "path": ["test", "path"], + "value": value_as_int, + } + assign_datetime = AssignDatetime.from_dict(data) + + # then + assert assign_datetime == AssignDatetime(["test", "path"], value) + + +class TestLogFloats: + def test__log_floats_operation__to_dict(self): + # given + values = [ + LogFloats.ValueType( + value=random.uniform(-1e8, 1e8), + step=random.uniform(-1e8, 1e8), + ts=random.uniform(-1e8, 1e8), + ) + for _ in range(5) + ] + + expected_values = [value.to_dict() for value in values] + log_floats = LogFloats(["test", "path"], values) + + # when + data = log_floats.to_dict() + + # then + assert data == {"type": "LogFloats", "path": ["test", "path"], "values": expected_values} + + def test__log_floats_operation__from_dict(self): + # given + values = [ + LogFloats.ValueType( + value=random.uniform(-1e8, 1e8), + step=random.uniform(-1e8, 1e8), + ts=random.uniform(-1e8, 1e8), + ) + for _ in range(5) + ] + + dict_values = [value.to_dict() for value in values] + data = {"type": "LogFloats", "path": ["test", "path"], "values": dict_values} + + # when + log_floats = LogFloats.from_dict(data) + + # then + assert log_floats == LogFloats(["test", "path"], values) + + +@pytest.mark.parametrize( + "operation", + [ + AssignInt(["test", "path"], 1), + AssignString(["test", "path"], "test"), + AssignBool(["test", "path"], True), + AssignDatetime(["test", "path"], datetime.datetime.now().replace(microsecond=0)), + AssignFloat(["test", "path"], 1.0), + LogFloats(["test", "path"], [LogFloats.ValueType(1.0, 1.0, 1.0)]), + ], +) +def test_is_serialisation_consistent(operation): + assert operation.from_dict(operation.to_dict()) == operation + + +def test__operation__from_dict(): + with pytest.raises(MalformedOperation): + Operation.from_dict({"path": ["test", "path"], "value": 1}) + + with pytest.raises(MalformedOperation): + Operation.from_dict({"type": "IncorrectType", "path": ["test", "path"], "value": 1}) + + assert Operation.from_dict({"type": "AssignInt", "path": ["test", "path"], "value": 1}) == AssignInt( + ["test", "path"], 1 + ) + + assert Operation.from_dict( + {"type": "LogFloats", "path": ["test", "path"], "values": [{"value": 1.0, "step": 1.0, "ts": 1.0}]} + ) == LogFloats(["test", "path"], [LogFloats.ValueType(1.0, 1.0, 1.0)])