diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 7d5769f081..8950c68847 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -32,9 +32,16 @@ Renku Python API .. automodule:: renku.ui.api.models.dataset +.. _api-plan: + +``Plan, CompositePlan`` +----------------------- + +.. automodule:: renku.ui.api.models.plan + .. _api-run: ``Inputs, Outputs, and Parameters`` ----------------------------------- -.. automodule:: renku.ui.api.models.run +.. automodule:: renku.ui.api.models.parameter diff --git a/renku/command/workflow.py b/renku/command/workflow.py index ac087cd4e9..928c2fdece 100644 --- a/renku/command/workflow.py +++ b/renku/command/workflow.py @@ -131,9 +131,7 @@ def _remove_workflow(name: str, force: bool, plan_gateway: IPlanGateway): communication.confirm(prompt_text, abort=True, warning=True) plan = plan or workflows[name] - plan.unfreeze() - plan.invalidated_at = local_now() - plan.freeze() + plan.delete() def remove_workflow_command(): diff --git a/renku/domain_model/workflow/parameter.py b/renku/domain_model/workflow/parameter.py index ed98aeb104..dd20d54d1a 100644 --- a/renku/domain_model/workflow/parameter.py +++ b/renku/domain_model/workflow/parameter.py @@ -179,6 +179,12 @@ def generate_id(plan_id: str, position: Optional[int] = None, postfix: str = Non plan_id, parameter_type="parameters", position=position, postfix=postfix ) + def __repr__(self): + return ( + f"" + ) + def _get_default_name(self) -> str: return self._generate_name(base="parameter") diff --git a/renku/domain_model/workflow/plan.py b/renku/domain_model/workflow/plan.py index 7dc31d90c1..f698565a65 100644 --- a/renku/domain_model/workflow/plan.py +++ b/renku/domain_model/workflow/plan.py @@ -86,7 +86,7 @@ def validate_name(name: str): """Check a name for invalid characters.""" if not re.match("[a-zA-Z0-9-_]+", name): raise errors.ParameterError( - f"Name {name} contains illegal characters. Only characters, numbers, _ and - are allowed." + f"Name {name} contains illegal characters. Only English letters, numbers, _ and - are allowed." ) def assign_new_id(self) -> str: @@ -183,6 +183,16 @@ def __init__( duplicates_string = ", ".join(duplicates) raise errors.ParameterError(f"Duplicate input, output or parameter names found: {duplicates_string}") + @property + def keywords_csv(self) -> str: + """Comma-separated list of keywords associated with workflow.""" + return ", ".join(self.keywords) + + @property + def deleted(self) -> bool: + """True if plan is deleted.""" + return self.invalidated_at is not None + def is_similar_to(self, other: "Plan") -> bool: """Return true if plan has the same inputs/outputs/arguments as another plan.""" @@ -288,11 +298,6 @@ def is_derivation(self) -> bool: """Return if an ``Plan`` has correct derived_from.""" return self.derived_from is not None and self.id != self.derived_from - @property - def keywords_csv(self): - """Comma-separated list of keywords associated with workflow.""" - return ", ".join(self.keywords) - def to_argv(self, with_streams: bool = False) -> List[Any]: """Convert a Plan into argv list.""" arguments = itertools.chain(self.inputs, self.outputs, self.parameters) @@ -336,6 +341,12 @@ def copy(self): """ return copy.deepcopy(self) + def delete(self, when: datetime = local_now()): + """Mark a plan as deleted.""" + self.unfreeze() + self.invalidated_at = when + self.freeze() + class PlanDetailsJson(marshmallow.Schema): """Serialize a plan to a response object.""" diff --git a/renku/ui/api/__init__.py b/renku/ui/api/__init__.py index bdc62316cf..c04535d67b 100644 --- a/renku/ui/api/__init__.py +++ b/renku/ui/api/__init__.py @@ -18,7 +18,8 @@ """Renku API.""" from renku.ui.api.models.dataset import Dataset +from renku.ui.api.models.parameter import Input, Link, Mapping, Output, Parameter +from renku.ui.api.models.plan import CompositePlan, Plan from renku.ui.api.models.project import Project -from renku.ui.api.models.run import Input, Output, Parameter -__all__ = ("Dataset", "Input", "Output", "Parameter", "Project") +__all__ = ("CompositePlan", "Dataset", "Input", "Link", "Mapping", "Output", "Parameter", "Plan", "Project") diff --git a/renku/ui/api/models/dataset.py b/renku/ui/api/models/dataset.py index 2c493887db..3a19c01587 100644 --- a/renku/ui/api/models/dataset.py +++ b/renku/ui/api/models/dataset.py @@ -41,7 +41,7 @@ from operator import attrgetter from pathlib import Path -from typing import Optional +from typing import List, Optional from renku.command.command_builder.database_dispatcher import DatabaseDispatcher from renku.domain_model import dataset as core_dataset @@ -73,14 +73,14 @@ def __init__(self): self._files = [] @classmethod - def _from_dataset(cls, dataset: core_dataset.Dataset): + def _from_dataset(cls, dataset: core_dataset.Dataset) -> "Dataset": """Create an instance from Dataset metadata. Args: dataset(core_dataset.Dataset): The core dataset to wrap. Returns: - An API ``Dataset`` wrapping a core dataset. + Dataset: An API ``Dataset`` wrapping a core dataset. """ self = cls() self._dataset = dataset @@ -90,14 +90,14 @@ def _from_dataset(cls, dataset: core_dataset.Dataset): @staticmethod @ensure_project_context - def list(project): + def list(project) -> List["Dataset"]: """List all datasets in a project. Args: project: The current project Returns: - A list of all datasets in the supplied project. + List["Dataset"]: A list of all datasets in the supplied project. """ client = project.client if not client or not client.has_graph_files(): diff --git a/renku/ui/api/models/parameter.py b/renku/ui/api/models/parameter.py new file mode 100644 index 0000000000..48045e8960 --- /dev/null +++ b/renku/ui/api/models/parameter.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2022 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Renku API Workflow Models. + +Input and Output classes can be used to define inputs and outputs of a script +within the same script. Paths defined with these classes are added to explicit +inputs and outputs in the workflow's metadata. For example, the following +mark a ``data/data.csv`` as an input with name ``my-input`` to the script: + +.. code-block:: python + + from renku.ui.api import Input + + with open(Input("my-input", "data/data.csv")) as input_data: + for line in input_data: + print(line) + + +Users can track parameters' values in a workflow by defining them using +``Parameter`` function. + +.. code-block:: python + + from renku.ui.api import Parameter + + nc = Parameter(name="n_components", value=10) + + print(nc.value) # 10 + +Once a Parameter is tracked like this, it can be set normally in commands like +``renku workflow execute`` with the ``--set`` option to override the value. + +""" + +from os import PathLike, environ +from pathlib import Path +from typing import Any, List, Optional, Union + +from renku.core import errors +from renku.core.plugin.provider import RENKU_ENV_PREFIX +from renku.core.workflow.plan_factory import ( + add_indirect_parameter, + add_to_files_list, + get_indirect_inputs_path, + get_indirect_outputs_path, +) +from renku.domain_model.workflow import parameter as core_parameter +from renku.domain_model.workflow import plan as core_plan +from renku.ui.api.models.project import ensure_project_context + + +def _validate_name(name: str): + if not name: + raise errors.ParameterError("'name' must be set.") + + core_plan.Plan.validate_name(name) + + +class _PathBase(PathLike): + """Base class of API input/output parameters.""" + + @ensure_project_context + def __init__(self, name: str, path: Union[str, Path], project=None, skip_addition: bool = False): + env_value = environ.get(f"{RENKU_ENV_PREFIX}{name}", None) + if env_value: + self._path = Path(env_value) + else: + self._path = Path(path) + + if not skip_addition: + indirect_list_path = self._get_indirect_list_path(project.path) + add_to_files_list(indirect_list_path, name, self._path) + + @staticmethod + def _get_indirect_list_path(project_path): + raise NotImplementedError + + def __fspath__(self): + """Abstract method of PathLike.""" + return str(self._path) + + @property + def path(self) -> Path: + """Return path of file.""" + return self._path + + +class Parameter: + """API Parameter model.""" + + @ensure_project_context + def __init__(self, name: str, value: Union[Path, str, bool, int, float], project, skip_addition: bool = False): + _validate_name(name) + + value = self._get_parameter_value(name=name, value=value) + + self.name: str = name + self.value: Union[Path, str, bool, int, float] = value + self.default_value: Union[Path, str, bool, int, float] = value + self.description: Optional[str] = None + self.position: Optional[int] = None + self.prefix: Optional[str] = None + + if not skip_addition: + add_indirect_parameter(project.path, name=name, value=value) + + @classmethod + def from_parameter(cls, parameter: core_parameter.CommandParameter) -> "Parameter": + """Create an instance from a core CommandParameterBase.""" + value = parameter.actual_value + default_value = parameter.default_value + value = value if value is not None else default_value + + self = cls(name=parameter.name, value=value, skip_addition=True) + self.default_value = default_value if default_value is not None else value + self.description = parameter.description + self.position = parameter.position + self.prefix = parameter.prefix + + return self + + @staticmethod + def _get_parameter_value(name: str, value: Any) -> Union[str, bool, int, float]: + """Get parameter's actual value from env variables. + + Args: + name (str): The name of the parameter. + value (Any): The value of the parameter. + + Returns: + The supplied value or a value set on workflow execution. + """ + env_value = environ.get(f"{RENKU_ENV_PREFIX}{name}", None) + + if env_value: + if isinstance(value, str): + value = env_value + elif isinstance(value, bool): + value = bool(env_value) + elif isinstance(value, int): + value = int(env_value) + elif isinstance(value, float): + value = float(env_value) + else: + raise errors.ParameterError( + f"Can't convert value '{env_value}' to type '{type(value)}' for parameter '{name}'. Only " + "str, bool, int and float are supported." + ) + + return value + + +class Input(_PathBase, Parameter): + """API Input model.""" + + @staticmethod + def _get_indirect_list_path(project_path): + return get_indirect_inputs_path(project_path) + + def __init__(self, name: str, path: Union[str, Path], skip_addition: bool = False): + _PathBase.__init__(self, name=name, path=path, skip_addition=skip_addition) + Parameter.__init__(self, name=name, value=path, skip_addition=True) + + self.mapped_stream: Optional[str] = None + + @classmethod + def from_parameter(cls, input: core_parameter.CommandInput) -> "Input": # type: ignore + """Create an instance from a CommandInput.""" + assert isinstance(input, core_parameter.CommandInput) + + self = cls(name=input.name, path=input.default_value, skip_addition=True) + self.description = input.description + self.prefix = input.prefix + self.position = input.position + self.mapped_stream = input.mapped_to.stream_type if input.mapped_to else None + + return self + + +class Output(_PathBase, Parameter): + """API Output model.""" + + @staticmethod + def _get_indirect_list_path(project_path): + return get_indirect_outputs_path(project_path) + + def __init__(self, name: str, path: Union[str, Path], skip_addition: bool = False): + _PathBase.__init__(self, name=name, path=path, skip_addition=skip_addition) + Parameter.__init__(self, name=name, value=path, skip_addition=True) + + self.mapped_stream: Optional[str] = None + + @classmethod + def from_parameter(cls, output: core_parameter.CommandOutput) -> "Output": # type: ignore + """Create an instance from a CommandOutput.""" + assert isinstance(output, core_parameter.CommandOutput) + + self = cls(name=output.name, path=output.default_value, skip_addition=True) + self.description = output.description + self.prefix = output.prefix + self.position = output.position + self.mapped_stream = output.mapped_to.stream_type if output.mapped_to else None + + return self + + +class Link: + """Parameter Link API model.""" + + def __init__(self, source: Parameter, sinks: List[Parameter]): + self.source: Parameter = source + self.sinks: List[Parameter] = sinks + + @classmethod + def from_link(cls, link: core_parameter.ParameterLink) -> "Link": + """Create an instance from a ParameterLink.""" + return cls(source=convert_parameter(link.source), sinks=[convert_parameter(p) for p in link.sinks]) + + +class Mapping(Parameter): + """Parameter Mapping API model.""" + + def __init__( + self, + name: str, + value: Union[Path, str, bool, int, float], + default_value: Union[Path, str, bool, int, float] = None, + description: Optional[str] = None, + parameters: List[Parameter] = None, + ): + super().__init__(name=name, value=value) + self.default_value: Union[Path, str, bool, int, float] = default_value if default_value is not None else value + self.description: Optional[str] = description + self.parameters: List[Parameter] = parameters or [] + + @classmethod + def from_parameter(cls, mapping: core_parameter.CommandParameterBase) -> "Mapping": + """Create an instance from a ParameterMapping.""" + assert isinstance(mapping, core_parameter.ParameterMapping) + + return cls( + name=mapping.name, + value=mapping.actual_value, + default_value=mapping.default_value, + description=mapping.description, + parameters=[convert_parameter(p) for p in mapping.mapped_parameters], + ) + + +def convert_parameter( + parameter: Union[core_parameter.CommandParameterBase], +) -> Union[Input, Output, Parameter, Mapping]: + """Convert a core CommandParameterBase subclass to its equivalent API class.""" + if isinstance(parameter, core_parameter.CommandInput): + return Input.from_parameter(parameter) + elif isinstance(parameter, core_parameter.CommandOutput): + return Output.from_parameter(parameter) + elif isinstance(parameter, core_parameter.CommandParameter): + return Parameter.from_parameter(parameter) + elif isinstance(parameter, core_parameter.ParameterMapping): + return Mapping.from_parameter(parameter) + + raise errors.ParameterError(f"Invalid parameter type: '{type(parameter)}'") diff --git a/renku/ui/api/models/plan.py b/renku/ui/api/models/plan.py new file mode 100644 index 0000000000..2c5c96821b --- /dev/null +++ b/renku/ui/api/models/plan.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2022 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Renku API Plan. + +``Plan`` and ``CompositePlan`` classes represent Renku workflow plans executed +in a Project. Each of these classes has a static ``list`` method that returns a +list of all active plans/composite-plans in a project: + +.. code-block:: python + + from renku.ui.api import Plan + + plans = Plan.list() + + composite_plans = CompositePlan.list() + +""" + +from datetime import datetime +from typing import List, Optional, Type, Union + +from renku.command.command_builder.database_dispatcher import DatabaseDispatcher +from renku.core import errors +from renku.domain_model.workflow import composite_plan as core_composite_plan +from renku.domain_model.workflow import plan as core_plan +from renku.infrastructure.gateway.plan_gateway import PlanGateway +from renku.ui.api.models.parameter import Input, Link, Mapping, Output, Parameter +from renku.ui.api.models.project import ensure_project_context + + +class Plan: + """API Plan.""" + + def __init__( + self, + command: str, + date_created: Optional[datetime] = None, + deleted: bool = False, + description: Optional[str] = None, + inputs: List[Input] = None, + keywords: Optional[List[str]] = None, + name: Optional[str] = None, + outputs: List[Output] = None, + parameters: List[Parameter] = None, + success_codes: Optional[List[int]] = None, + ): + self.command: str = command + self.date_created: Optional[datetime] = date_created + self.deleted: bool = deleted + self.description: Optional[str] = description + self.inputs: List[Input] = inputs or [] + self.keywords: List[str] = keywords or [] + self.name: Optional[str] = name + self.outputs: List[Output] = outputs or [] + self.parameters: List[Parameter] = parameters or [] + self.success_codes: List[int] = success_codes or [] + + @classmethod + def from_plan(cls, plan: core_plan.Plan) -> "Plan": + """Create an instance from a core Plan model. + + Args: + plan(core_plan.Plan): The core plan. + + Returns: + Plan: An API Plan model. + """ + return cls( + command=plan.command, + date_created=plan.date_created, + deleted=plan.invalidated_at is not None, + description=plan.description, + inputs=[Input.from_parameter(i) for i in plan.inputs], + keywords=plan.keywords, + name=plan.name, + outputs=[Output.from_parameter(o) for o in plan.outputs], + parameters=[Parameter.from_parameter(p) for p in plan.parameters], + success_codes=plan.success_codes, + ) + + @staticmethod + def list(include_deleted: bool = False) -> List[Union["Plan", "CompositePlan"]]: + """List all plans in a project. + + Args: + include_deleted(bool): Whether to include deleted plans. + + Returns: + A list of all plans in the supplied project. + """ + return _list_plans(include_deleted=include_deleted, type=core_plan.Plan) + + +class CompositePlan: + """API CompositePlan.""" + + def __init__( + self, + date_created: Optional[datetime] = None, + deleted: bool = False, + description: Optional[str] = None, + keywords: Optional[List[str]] = None, + links: List[Link] = None, + mappings: List[Mapping] = None, + name: Optional[str] = None, + plans: List[Union["CompositePlan", "Plan"]] = None, + ): + self.date_created: Optional[datetime] = date_created + self.deleted: bool = deleted + self.description: Optional[str] = description + self.keywords: List[str] = keywords or [] + self.links: List[Link] = links or [] + self.mappings: List[Mapping] = mappings or [] + self.name: Optional[str] = name + self.plans: List[Union["CompositePlan", "Plan"]] = plans or [] + + @classmethod + def from_composite_plan(cls, composite_plan: core_composite_plan.CompositePlan) -> "CompositePlan": + """Create an instance from a core CompositePlan model. + + Args: + composite_plan(core_composite_plan.CompositePlan): The core composite plan. + + Returns: + CompositePlan: An API CompositePlan instance. + """ + return cls( + date_created=composite_plan.date_created, + deleted=composite_plan.invalidated_at is not None, + description=composite_plan.description, + keywords=composite_plan.keywords, + links=[Link.from_link(link) for link in composite_plan.links], + mappings=[Mapping.from_parameter(m) for m in composite_plan.mappings], + name=composite_plan.name, + plans=_convert_plans(composite_plan.plans), + ) + + @staticmethod + def list(include_deleted: bool = False) -> List[Union["Plan", "CompositePlan"]]: + """List all plans in a project. + + Args: + include_deleted(bool): Whether to include deleted plans. + + Returns: + A list of all plans in the supplied project. + """ + return _list_plans(include_deleted=include_deleted, type=core_composite_plan.CompositePlan) + + +def _convert_plans(plans: List[Union[core_plan.AbstractPlan]]) -> List[Union[Plan, CompositePlan]]: + """Convert a list of core Plans/CompositePlans to API Plans/CompositePlans.""" + + def convert_plan(plan): + if isinstance(plan, core_plan.Plan): + return Plan.from_plan(plan) + elif isinstance(plan, core_composite_plan.CompositePlan): + return CompositePlan.from_composite_plan(plan) + + raise errors.ParameterError(f"Invalid plan type: {type(plan)}") + + return [convert_plan(p) for p in plans] + + +@ensure_project_context +def _list_plans( + include_deleted: bool, type: Type[Union[core_plan.Plan, core_composite_plan.CompositePlan]], project +) -> List[Union[Plan, CompositePlan]]: + """List all plans in a project. + + Args: + include_deleted(bool): Whether to include deleted plans. + project: The current project + + Returns: + A list of all plans in the supplied project. + """ + client = project.client + if not client: + return [] + + database_dispatcher = DatabaseDispatcher() + database_dispatcher.push_database_to_stack(client.database_path) + plan_gateway = PlanGateway() + plan_gateway.database_dispatcher = database_dispatcher + + plans = plan_gateway.get_all_plans() + + if not include_deleted: + plans = [p for p in plans if p.invalidated_at is None] + + plans = [p for p in plans if isinstance(p, type)] + + return _convert_plans(plans) diff --git a/renku/ui/api/models/project.py b/renku/ui/api/models/project.py index 6318633f0f..0431a6ff6f 100644 --- a/renku/ui/api/models/project.py +++ b/renku/ui/api/models/project.py @@ -45,7 +45,7 @@ """ from functools import wraps from pathlib import Path -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union from werkzeug.local import LocalStack @@ -53,6 +53,9 @@ from renku.core import errors from renku.core.workflow.run import StatusResult +if TYPE_CHECKING: + from renku.core.management.client import LocalClient + class Project: """API Project context class.""" @@ -60,7 +63,7 @@ class Project: _project_contexts = LocalStack() def __init__(self): - self._client = _get_local_client() + self._client: "LocalClient" = _get_local_client() def __enter__(self): self._project_contexts.push(self) @@ -73,9 +76,9 @@ def __exit__(self, type, value, traceback): raise RuntimeError("Project context was changed.") @property - def client(self): + def client(self) -> Optional["LocalClient"]: """Return the LocalClient instance.""" - return self._client + return None if self._client.repository is None else self._client @property def path(self): @@ -125,14 +128,14 @@ def _get_current_project(): return Project._project_contexts.top if Project._project_contexts.top else None -def _get_local_client(): +def _get_local_client() -> "LocalClient": from renku.core.management.client import LocalClient from renku.infrastructure.repository import Repository try: repository = Repository(".", search_parent_directories=True) except errors.GitError: - path = "." + path = Path(".") else: path = repository.path diff --git a/renku/ui/api/models/run.py b/renku/ui/api/models/run.py deleted file mode 100644 index 55d9897523..0000000000 --- a/renku/ui/api/models/run.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2017-2022 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Renku API Workflow Models. - -Input and Output classes can be used to define inputs and outputs of a script -within the same script. Paths defined with these classes are added to explicit -inputs and outputs in the workflow's metadata. For example, the following -mark a ``data/data.csv`` as an input with name ``my-input`` to the script: - -.. code-block:: python - - from renku.ui.api import Input - - with open(Input("my-input", "data/data.csv")) as input_data: - for line in input_data: - print(line) - - -Users can track parameters' values in a workflow by defining them using -``Parameter`` function. - -.. code-block:: python - - from renku.ui.api import Parameter - - nc = Parameter(name="n_components", value=10) - -Once a Parameter is tracked like this, it can be set normally in commands like -``renku workflow execute`` with the ``--set`` option to override the value. - -""" - -import re -from os import PathLike, environ -from pathlib import Path -from typing import Union - -from renku.core import errors -from renku.core.plugin.provider import RENKU_ENV_PREFIX -from renku.core.workflow.plan_factory import ( - add_indirect_parameter, - add_to_files_list, - get_indirect_inputs_path, - get_indirect_outputs_path, -) -from renku.ui.api.models.project import ensure_project_context - -name_pattern = re.compile("[a-zA-Z0-9-_]+") - - -class _PathBase(PathLike): - @ensure_project_context - def __init__(self, name: str, path: Union[str, Path], project=None): - if not name: - raise errors.ParameterError("'name' must be set.") - - if name and not name_pattern.match(name): - raise errors.ParameterError( - f"Name {name} contains illegal characters. Only characters, numbers, _ and - are allowed." - ) - self.name = name - - env_value = environ.get(f"{RENKU_ENV_PREFIX}{name}", None) - - if env_value: - self._path = Path(env_value) - else: - self._path = Path(path) - - indirect_list_path = self._get_indirect_list_path(project.path) - - add_to_files_list(indirect_list_path, name, self._path) - - @staticmethod - def _get_indirect_list_path(project_path): - raise NotImplementedError - - def __fspath__(self): - """Abstract method of PathLike.""" - return str(self._path) - - @property - def path(self): - """Return path of file.""" - return self._path - - -class Input(_PathBase): - """API Input model.""" - - @staticmethod - def _get_indirect_list_path(project_path): - return get_indirect_inputs_path(project_path) - - -class Output(_PathBase): - """API Output model.""" - - @staticmethod - def _get_indirect_list_path(project_path): - return get_indirect_outputs_path(project_path) - - -@ensure_project_context -def parameter(name, value, project): - """Store parameter's name and value. - - Args: - name (str): The name of the parameter. - value (Any): The value of the parameter. - project: The current project. - - Returns: - The supplied value or a value set on workflow execution. - """ - if not name: - raise errors.ParameterError("'name' must be set.") - - if not name_pattern.match(name): - raise errors.ParameterError( - f"Name {name} contains illegal characters. Only characters, numbers, _ and - are allowed." - ) - env_value = environ.get(f"{RENKU_ENV_PREFIX}{name}", None) - - if env_value: - if isinstance(value, str): - value = env_value - elif isinstance(value, bool): - value = bool(env_value) - elif isinstance(value, int): - value = int(env_value) - elif isinstance(value, float): - value = float(env_value) - else: - raise errors.ParameterError( - f"Can't convert value '{env_value}' to type '{type(value)}' for parameter '{name}'. Only " - "str, bool, int and float are supported." - ) - - add_indirect_parameter(project.path, name=name, value=value) - - return value - - -Parameter = parameter diff --git a/tests/api/test_plan.py b/tests/api/test_plan.py new file mode 100644 index 0000000000..8145712f9b --- /dev/null +++ b/tests/api/test_plan.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2022 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for Plan API.""" + +import os + +from renku.ui.api import CompositePlan, Input, Output, Parameter, Plan +from renku.ui.cli import cli +from tests.utils import format_result_exception + + +def test_list_plans(client_with_runs): + """Test listing plans.""" + plans = Plan.list() + + assert {"plan-1", "plan-2"} == {p.name for p in plans} + assert isinstance(plans[0], Plan) + assert isinstance(plans[1], Plan) + + +def test_list_deleted_plans(client_with_runs, runner): + """Test listing deleted plans.""" + result = runner.invoke(cli, ["workflow", "remove", "plan-1"]) + assert 0 == result.exit_code, format_result_exception(result) + + plans = Plan.list() + + assert {"plan-2"} == {p.name for p in plans} + + plans = Plan.list(include_deleted=True) + + assert {"plan-1", "plan-2"} == {p.name for p in plans} + + +def test_list_datasets_outside_a_renku_project(directory_tree): + """Test listing plans in a non-renku directory.""" + os.chdir(directory_tree) + + assert [] == Plan.list() + + +def test_get_plan_parameters(client_with_runs): + """Test getting parameters of a plan.""" + plan = next(p for p in Plan.list() if p.name == "plan-1") + + assert {"n-1"} == {p.name for p in plan.parameters} + parameter = plan.parameters[0] + + assert isinstance(parameter, Parameter) + assert "2" == parameter.default_value + assert "2" == parameter.value + assert "-n" == parameter.prefix.strip() + assert 1 == parameter.position + + +def test_get_plan_inputs(client_with_runs): + """Test getting inputs of a plan.""" + plan = next(p for p in Plan.list() if p.name == "plan-1") + + assert {"input-2"} == {p.name for p in plan.inputs} + input = plan.inputs[0] + + assert isinstance(input, Input) + assert "input" == input.default_value + assert "input" == input.value + assert input.prefix is None + assert 2 == input.position + assert input.mapped_stream is None + + +def test_get_plan_outputs(client_with_runs): + """Test getting outputs of a plan.""" + plan = next(p for p in Plan.list() if p.name == "plan-1") + + assert {"output-3"} == {p.name for p in plan.outputs} + output = plan.outputs[0] + + assert isinstance(output, Output) + assert "intermediate" == output.default_value + assert "intermediate" == output.value + assert output.prefix is None + assert 3 == output.position + assert "stdout" == output.mapped_stream + + +def test_list_composite_plans(client_with_runs, runner): + """Test listing plans.""" + result = runner.invoke( + cli, + [ + "workflow", + "compose", + "--map", + "input_file=@step1.@input1", + "--link", + "@step1.@output1=@step2.@input1", + "--description", + "Composite plan", + "composite-plan", + "plan-1", + "plan-2", + "--set", + "input_file=composite-input-file", + ], + ) + assert 0 == result.exit_code, format_result_exception(result) + + plans = Plan.list() + + assert {"plan-1", "plan-2"} == {p.name for p in plans} + + plans = CompositePlan.list() + + assert {"composite-plan"} == {p.name for p in plans} + + plan = next(p for p in plans if p.name == "composite-plan") + assert isinstance(plan, CompositePlan) + assert "Composite plan" == plan.description + assert {"plan-1", "plan-2"} == {p.name for p in plan.plans} + + assert {"input_file"} == {m.name for m in plan.mappings} + mapping = plan.mappings[0] + assert "composite-input-file" == mapping.value + mapping_parameter = mapping.parameters[0] + assert isinstance(mapping_parameter, Input) + assert "input-2" == mapping_parameter.name + + assert 1 == len(plan.links) + link = plan.links[0] + assert isinstance(link.source, Output) + assert "output-3" == link.source.name + assert ["input-2"] == [s.name for s in link.sinks] diff --git a/tests/api/test_project.py b/tests/api/test_project.py index 09a74d5219..97c14b10ba 100644 --- a/tests/api/test_project.py +++ b/tests/api/test_project.py @@ -63,7 +63,7 @@ def test_get_project_outside_a_renku_project(directory_tree): os.chdir(directory_tree) with Project() as project: - assert project.client is not None + assert project.client is None def test_status(runner, client): diff --git a/tests/api/test_run.py b/tests/api/test_run.py index d39ac20f03..559b1046f3 100644 --- a/tests/api/test_run.py +++ b/tests/api/test_run.py @@ -120,7 +120,7 @@ def test_parameters(client): p3 = Parameter("parameter_3 ", 42.42) - assert (42, "42", 42.42) == (p1, p2, p3) + assert (42, "42", 42.42) == (p1.value, p2.value, p3.value) data = read_indirect_parameters(client.path) diff --git a/tests/cli/test_workflow.py b/tests/cli/test_workflow.py index a76fdd91a1..9ccfe1f654 100644 --- a/tests/cli/test_workflow.py +++ b/tests/cli/test_workflow.py @@ -619,9 +619,9 @@ def test_workflow_execute_command_with_api_parameter_set(runner, run_shell, proj output = client.path / "output" with client.commit(): - script.write_text("from renku.ui.api import Parameter\n" 'print(Parameter("test", "hello world"))\n') + script.write_text("from renku.ui.api import Parameter\n" 'print(Parameter("test", "hello world").value)\n') - result = run_shell(f"renku run --name run1 -- python {script} > {output}") + result = run_shell(f"renku run --name run1 -- python3 {script} > {output}") # Assert expected empty stdout. assert b"" == result[0] @@ -657,7 +657,7 @@ def test_workflow_execute_command_with_api_input_set(runner, run_shell, project, " print(f.read())" ) - result = run_shell(f"renku run --name run1 -- python {script.name} > {output.name}") + result = run_shell(f"renku run --name run1 -- python3 {script.name} > {output.name}") # Assert expected empty stdout. assert b"" == result[0] @@ -689,7 +689,7 @@ def test_workflow_execute_command_with_api_output_set(runner, run_shell, project " f.write('test')" ) - result = run_shell(f"renku run --name run1 -- python {script.name}") + result = run_shell(f"renku run --name run1 -- python3 {script.name}") # Assert expected empty stdout. assert b"" == result[0] @@ -737,7 +737,7 @@ def test_workflow_execute_command_with_api_valid_duplicate_output(runner, run_sh f"open(Output('my-output', '{output.name}'), 'w')" ) - result = run_shell(f"renku run --name run1 -- python {script.name}") + result = run_shell(f"renku run --name run1 -- python3 {script.name}") # Assert expected empty stdout. assert b"" == result[0] diff --git a/tests/core/fixtures/core_workflow.py b/tests/core/fixtures/core_workflow.py index 9de07ed37f..6a5a00e1ab 100644 --- a/tests/core/fixtures/core_workflow.py +++ b/tests/core/fixtures/core_workflow.py @@ -22,6 +22,7 @@ from renku.domain_model.workflow.composite_plan import CompositePlan from renku.domain_model.workflow.parameter import CommandInput, CommandOutput, CommandParameter from renku.domain_model.workflow.plan import Plan +from tests.utils import write_and_commit_file def _create_run(name: str) -> Plan: @@ -85,3 +86,14 @@ def composite_plan(): grouped = CompositePlan(id=CompositePlan.generate_id(), plans=[run1, run2], name="grouped1") return grouped, run1, run2 + + +@pytest.fixture +def client_with_runs(client, run, client_database_injection_manager): + """A client with runs.""" + write_and_commit_file(client.repository, "input", "some\ninput\nfile") + + assert 0 == run(["run", "--name", "plan-1", "head", "-n", "2", "input"], stdout="intermediate") + assert 0 == run(["run", "--name", "plan-2", "tail", "-n", "1", "intermediate"], stdout="output") + + yield client