Skip to content

Commit

Permalink
Remove .from_annot.
Browse files Browse the repository at this point in the history
  • Loading branch information
tobiasraabe committed Sep 7, 2023
1 parent 0373e6a commit 2ff2865
Show file tree
Hide file tree
Showing 12 changed files with 27 additions and 155 deletions.
1 change: 1 addition & 0 deletions docs/source/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
- {pull}`412` adds protocols for tasks.
- {pull}`413` removes scripts to generate `.svg`s.
- {pull}`414` allow more ruff rules.
- {pull}`416` removes `.from_annot` again.

## 0.3.2 - 2023-06-07

Expand Down
11 changes: 2 additions & 9 deletions src/_pytask/collect_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
from _pytask.pluginmanager import get_plugin_manager
from _pytask.session import Session
from _pytask.tree_util import tree_leaves
from _pytask.typing import no_value
from rich.text import Text
from rich.tree import Tree

Expand Down Expand Up @@ -129,14 +128,10 @@ def _find_common_ancestor_of_all_nodes(
all_paths.append(task.path)
if show_nodes:
all_paths.extend(
x.path
for x in tree_leaves(task.depends_on)
if isinstance(x, PPathNode) and x.path is not no_value
x.path for x in tree_leaves(task.depends_on) if isinstance(x, PPathNode)
)
all_paths.extend(
x.path
for x in tree_leaves(task.produces)
if isinstance(x, PPathNode) and x.path is not no_value
x.path for x in tree_leaves(task.produces) if isinstance(x, PPathNode)
)

return find_common_ancestor(*all_paths, *paths)
Expand Down Expand Up @@ -208,7 +203,6 @@ def _print_collected_tasks(
sorted_nodes = sorted(nodes, key=lambda x: x.name)
for node in sorted_nodes:
if isinstance(node, PPathNode):
assert node.path is not no_value
if node.path.as_posix() in node.name:
reduced_node_name = str(
relative_to(node.path, common_ancestor)
Expand All @@ -228,7 +222,6 @@ def _print_collected_tasks(
tree_leaves(task.produces), key=lambda x: getattr(x, "path", x.name)
):
if isinstance(node, PPathNode):
assert node.path is not no_value
reduced_node_name = str(relative_to(node.path, common_ancestor))
url_style = create_url_style_for_path(
node.path, editor_url_scheme
Expand Down
39 changes: 8 additions & 31 deletions src/_pytask/collect_utils.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""Contains utility functions for :mod:`_pytask.collect`."""
from __future__ import annotations

import functools
import itertools
import uuid
import warnings
Expand Down Expand Up @@ -275,17 +274,12 @@ def parse_dependencies_from_task_function(
if parameter_name == "depends_on":
continue

partialed_evolve = functools.partial(
_evolve_instance,
instance_from_annot=parameters_with_node_annot.get(parameter_name),
)

nodes = tree_map_with_path(
lambda p, x: _collect_dependency(
session,
path,
name,
NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023
NodeInfo(parameter_name, p, x), # noqa: B023
),
value,
)
Expand Down Expand Up @@ -402,22 +396,21 @@ def parse_products_from_task_function(

if parameters_with_product_annot:
has_annotation = True
for parameter_name in parameters_with_product_annot:
if parameter_name in kwargs:
partialed_evolve = functools.partial(
_evolve_instance,
instance_from_annot=parameters_with_node_annot.get(parameter_name),
)

for parameter_name in parameters_with_product_annot:
value = kwargs.get(parameter_name) or parameters_with_node_annot.get(
parameter_name
)
if value:
collected_products = tree_map_with_path(
lambda p, x: _collect_product(
session,
path,
name,
NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023
NodeInfo(parameter_name, p, x), # noqa: B023
is_string_allowed=False,
),
kwargs[parameter_name],
value,
)
out = {parameter_name: collected_products}

Expand Down Expand Up @@ -602,19 +595,3 @@ def _collect_product(
raise NodeNotCollectedError(msg)

return collected_node


def _evolve_instance(x: Any, instance_from_annot: Node | None) -> Any:
"""Evolve a value to a node if it is given by annotations."""
if not instance_from_annot:
return x

if not hasattr(instance_from_annot, "from_annot"):
msg = (
f"The node {instance_from_annot!r} does not define '.from_annot' which is "
f"necessary to complete the node with the value {x!r}."
)
raise AttributeError(msg)

instance_from_annot.from_annot(x) # type: ignore[attr-defined]
return instance_from_annot
2 changes: 0 additions & 2 deletions src/_pytask/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
from _pytask.tree_util import tree_leaves
from _pytask.tree_util import tree_map
from _pytask.tree_util import tree_structure
from _pytask.typing import no_value
from rich.text import Text

if TYPE_CHECKING:
Expand Down Expand Up @@ -136,7 +135,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None:
for product in session.dag.successors(task.name):
node = session.dag.nodes[product]["node"]
if isinstance(node, PPathNode):
assert node.path is not no_value
node.path.parent.mkdir(parents=True, exist_ok=True)

would_be_executed = has_mark(task, "would_be_executed")
Expand Down
5 changes: 2 additions & 3 deletions src/_pytask/node_protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@


if TYPE_CHECKING:
from _pytask.typing import NoValue
from _pytask.tree_util import PyTree
from pathlib import Path
from _pytask.mark import Mark
Expand Down Expand Up @@ -54,7 +53,7 @@ class PPathNode(Node, Protocol):
"""

path: Path | NoValue
path: Path


@runtime_checkable
Expand Down Expand Up @@ -82,4 +81,4 @@ class PTaskWithPath(PTask, Protocol):
"""

path: Path | NoValue
path: Path
63 changes: 4 additions & 59 deletions src/_pytask/nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,14 @@

import functools
import hashlib
from pathlib import Path
from pathlib import Path # noqa: TCH003
from typing import Any
from typing import Callable
from typing import TYPE_CHECKING

from _pytask.node_protocols import MetaNode
from _pytask.node_protocols import Node
from _pytask.node_protocols import PPathNode
from _pytask.typing import no_value
from _pytask.typing import NoValue
from attrs import define
from attrs import field

Expand Down Expand Up @@ -76,33 +74,11 @@ def execute(self, **kwargs: Any) -> None:
class PathNode(PPathNode):
"""The class for a node which is a path."""

name: str = ""
name: str
"""Name of the node which makes it identifiable in the DAG."""
path: Path | NoValue = no_value
path: Path
"""The path to the file."""

def from_annot(self, value: Path) -> None:
"""Set path and if other attributes are not set, set sensible defaults.
Use it, if you want to control the name of the node.
.. codeblock: python
def task_example(value: Annotated[Any, PathNode(name="value")]):
...
"""
if self.path is not no_value:
msg = "'path' is already defined."
raise ValueError(msg)
if not isinstance(value, Path):
msg = "'value' must be a 'pathlib.Path'."
raise TypeError(msg)
if not self.name:
self.name = value.as_posix()
self.path = value

@classmethod
@functools.lru_cache
def from_path(cls, path: Path) -> PathNode:
Expand All @@ -122,24 +98,16 @@ def state(self) -> str | None:
The state is given by the modification timestamp.
"""
if self.path is no_value:
return None
if self.path.exists():
return str(self.path.stat().st_mtime)
return None

def load(self) -> Path:
"""Load the value."""
if self.path is no_value:
msg = "PathNode has no path defined."
raise ValueError(msg)
return self.path

def save(self, value: bytes | str) -> None:
"""Save strings or bytes to file."""
if self.path is no_value:
msg = "'path' must be a 'pathlib.Path' to store data."
raise TypeError(msg)
if isinstance(value, str):
self.path.write_text(value)
elif isinstance(value, bytes):
Expand All @@ -155,40 +123,19 @@ class PythonNode(Node):

name: str = ""
"""Name of the node."""
value: Any | NoValue = no_value
value: Any | None = None
"""Value of the node."""
hash: bool | Callable[[Any], bool] = False # noqa: A003
"""Whether the value should be hashed to determine the state."""

def load(self) -> Any:
"""Load the value."""
if self.value is no_value:
msg = "PythonNode has no value defined."
raise ValueError(msg)
return self.value

def save(self, value: Any) -> None:
"""Save the value."""
self.value = value

def from_annot(self, value: Any) -> None:
"""Set the value from a function annotation.
Use it, if you want to add information on how a node handles an argument while
keeping the type of the value unrelated to pytask. For example, the node could
be hashed.
.. codeblock: python
def task_example(value: Annotated[Any, PythonNode(hash=True)]):
...
"""
if self.value is not no_value:
msg = "'value' is already defined."
raise ValueError(msg)
self.value = value

def state(self) -> str | None:
"""Calculate state of the node.
Expand All @@ -204,8 +151,6 @@ def state(self) -> str | None:
random integer and it would confuse users.
"""
if self.value is no_value:
return None
if self.hash:
if callable(self.hash):
return str(self.hash(self.value))
Expand Down
2 changes: 0 additions & 2 deletions src/_pytask/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
from _pytask.pluginmanager import get_plugin_manager
from _pytask.session import Session
from _pytask.traceback import render_exc_info
from _pytask.typing import no_value
from rich.table import Table
from sqlalchemy import Column
from sqlalchemy import Float
Expand Down Expand Up @@ -230,7 +229,6 @@ def pytask_profile_add_info_on_task(
node = session.dag.nodes[successor]["node"]
if isinstance(node, PPathNode):
with suppress(FileNotFoundError):
assert node.path is not no_value
sum_bytes += node.path.stat().st_size

profile[task.name]["Size of Products"] = _to_human_readable_size(
Expand Down
29 changes: 1 addition & 28 deletions src/_pytask/typing.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,9 @@
from __future__ import annotations

from enum import Enum
from typing import Literal

from attr import define


__all__ = ["no_value", "NoValue"]


class _NoValue(Enum):
"""Sentinel value when no value was provided.
For nodes, we need a different default value than :obj:`None` to signal that the
user provided no value.
We make this an Enum
- because it round-trips through pickle correctly (see GH#pandas/40397)
- because mypy does not understand singletons
"""

no_value = "NO_VALUE"

def __repr__(self) -> str:
return "<no_value>"


# Note: no_value is exported to the public API in pytask
no_value = _NoValue.no_value
NoValue = Literal[_NoValue.no_value]
__all__ = ["Product", "ProductType"]


@define(frozen=True)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ def test_setting_name_for_path_node_via_annotation(tmp_path):
from typing import Any
def task_example(
path: Annotated[Path, Product, PathNode(name="product")] = Path("out.txt"),
path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")],
) -> None:
path.write_text("text")
"""
Expand Down
9 changes: 2 additions & 7 deletions tests/test_collect_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,6 @@ def state(self):
def load(self): ...
def save(self, value): ...
def from_annot(self, value): ...
def task_example(
data = CustomNode("custom", "text"),
Expand All @@ -556,7 +555,6 @@ def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path):
class PickleFile:
name: str
path: Path
value: Path
def state(self):
return str(self.path.stat().st_mtime)
Expand All @@ -570,13 +568,10 @@ def save(self, value):
with self.path.open("wb") as f:
pickle.dump(value, f)
def from_annot(self, value): ...
_PATH = Path(__file__).parent.joinpath("in.pkl")
def task_example(
data = PickleFile(_PATH.as_posix(), _PATH, _PATH),
data = PickleFile(_PATH.as_posix(), _PATH),
out: Annotated[Path, Product] = Path("out.txt"),
) -> None:
out.write_text(data)
Expand All @@ -598,7 +593,7 @@ def test_setting_name_for_python_node_via_annotation(runner, tmp_path):
from typing import Any
def task_example(
input: Annotated[str, PythonNode(name="node-name")] = "text",
input: Annotated[str, PythonNode(name="node-name", value="text")],
path: Annotated[Path, Product] = Path("out.txt"),
) -> None:
path.write_text(input)
Expand Down

0 comments on commit 2ff2865

Please sign in to comment.