practical-data-engineering/src/pipelines/real-estate/realestate/common/solids_jupyter.py at v1 · ssp-data/practical-data-engineering · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
from dagster import InputDefinition, Field, OutputDefinition, FileHandle, ModeDefinition
import dagstermill as dm
from dagster.utils import script_relative_path
import os


def _notebook_path(name):
    return os.path.join(
        os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'notebooks')), name
    )
    # os.path.join(os.path.dirname(os.path.abspath(__file__)), "notebooks", name)


def notebook_solid(name, notebook_path, input_defs, output_defs, required_resource_keys=None):
    return dm.define_dagstermill_solid(
        name,
        _notebook_path(notebook_path),
        input_defs,
        output_defs,
        required_resource_keys=required_resource_keys,
    )


# TODO: add spark as resource and use configs inside notebook.
# TODO: plot pdfs -> copy airline demo notebook
data_exploration = notebook_solid(
    "data_exploration",
    "comprehensive-real-estate-data-exploration.ipynb",
    input_defs=[
        InputDefinition("delta_path", str, description="s3 path to the property-delta-table"),
        InputDefinition("key", str, description="s3 key"),
        InputDefinition("secret", str, description="s3 secret"),
        InputDefinition("endpoint", str, description="s3 endpoint"),
        # InputDefinition("pyspark", ModeDefinition, description="pyspark resource"),
    ],
    output_defs=[
        OutputDefinition(
            dagster_type=FileHandle,
            # name='plots_pdf_path',
            description="The saved PDF plots.",
        )
    ],
    required_resource_keys={"pyspark"},
)

# data_exploration = dm.define_dagstermill_solid(
#     "data_exploration",
#     script_relative_path("../notebooks/comprehensive-real-estate-data-exploration.ipynb"),
#     input_defs=[
#         InputDefinition("delta_path", str, description="s3 path to the property-delta-table")
#     ],
#     # config_schema={
#     #     'delta_path': Field(
#     #         str,
#     #         default_value="s3a://real-estate/lake/bronze/property",
#     #         is_required=False,
#     #         description="s3 path to the property-delta-table",
#     #     )
#     # },
# )