Add Step abstract class and new Pipeline

[gabrielmbmb]

Description

This PR includes the new Step abstract base class and the new Pipeline to build more complex distilabel pipelines defined as a DAG. This PR, just adds the needed logic to add and connect steps, as well as validating that the resulting pipeline is valid and can be executed.

Some more validations are still missing, like checking that non-optional runtime parameters have been provided in the pipeline run method, but I'll add those in a different PR, as this one is quite big already.

from typing import Any, Dict, Generator, List

from distilabel.pipeline.local import Pipeline
from distilabel.step.base import GeneratorStep, Step, StepInput


class LoadHubDataset(GeneratorStep):
    def process(
        self, repo_id: str, split: str
    ) -> Generator[List[Dict[str, Any]], None, None]:
        yield []

    @property
    def outputs(self) -> List[str]:
        return ["instruction"]


class GenerateResponse(Step):
    @property
    def inputs(self) -> List[str]:
        return ["instruction"]

    def process(self, inputs: StepInput) -> Generator[List[Dict[str, Any]], None, None]:
        yield []

    @property
    def outputs(self) -> List[str]:
        return ["response"]


class EvolResponse(Step):
    @property
    def inputs(self) -> List[str]:
        return ["response"]

    def process(
        self, *inputs: StepInput
    ) -> Generator[List[Dict[str, Any]], None, None]:
        yield []

    @property
    def outputs(self) -> List[str]:
        return ["evol_response"]


with Pipeline() as pipeline:
    load_dataset = LoadHubDataset(name="load_dataset")
    generate_response = GenerateResponse(name="generate_response")
    evol_response = EvolResponse(name="evol_response")

    load_dataset.connect(generate_response)
    generate_response.connect(evol_response)


pipeline.run(
    configuration={
        "load_dataset": {
            "repo_id": "HuggingFaceH4/instruction-dataset",
            "split": "test",
        }
    }
)