Skip to content

Commit

Permalink
updating to reflect metadata section
Browse files Browse the repository at this point in the history
  • Loading branch information
tacastillo committed Jan 12, 2024
1 parent 5bb6fc2 commit 158702d
Show file tree
Hide file tree
Showing 6 changed files with 28 additions and 17 deletions.
2 changes: 1 addition & 1 deletion dagster_university/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
all_sensors = [adhoc_request_sensor]

defs = Definitions(
assets=[*trip_assets, *metric_assets, *requests_assets],
assets=trip_assets + metric_assets + requests_assets,
resources={
"database": database_resource,
},
Expand Down
15 changes: 9 additions & 6 deletions dagster_university/assets/constants.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
from datetime import date
import os

TAXI_ZONES_FILE_PATH = "data/raw/taxi_zones.csv"
TAXI_TRIPS_TEMPLATE_FILE_PATH = "data/raw/taxi_trips_{}.parquet"

TRIPS_BY_AIRPORT_FILE_PATH = "data/outputs/trips_by_airport.csv"
TRIPS_BY_WEEK_FILE_PATH = "data/outputs/trips_by_week.csv"
MANHATTAN_STATS_FILE_PATH = "data/staging/manhattan_stats.geojson"
MANHATTAN_MAP_FILE_PATH = "data/outputs/manhattan_map.png"
TAXI_ZONES_FILE_PATH = os.path.join("data", "raw", "taxi_zones.csv")
TAXI_TRIPS_TEMPLATE_FILE_PATH = os.path.join("data", "raw", "taxi_trips_{}.parquet")

REQUEST_DESTINATION_TEMPLATE_FILE_PATH = "data/outputs/{}.png"
TRIPS_BY_AIRPORT_FILE_PATH = os.path.join("data", "outputs", "trips_by_airport.csv")
TRIPS_BY_WEEK_FILE_PATH = os.path.join("data", "outputs", "trips_by_week.csv")
MANHATTAN_STATS_FILE_PATH = os.path.join("data", "staging", "manhattan_stats.geojson")
MANHATTAN_MAP_FILE_PATH = os.path.join("data", "outputs", "manhattan_map.png")

REQUEST_DESTINATION_TEMPLATE_FILE_PATH = os.path.join("data", "outputs", "{}.png")

DATE_FORMAT = "%Y-%m-%d"

Expand Down
9 changes: 6 additions & 3 deletions dagster_university/assets/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@

@asset(
deps=[AssetKey(["taxi_trips"])],
partitions_def=weekly_partition
partitions_def=weekly_partition,
compute_kind="DuckDB",
)
def trips_by_week(context, database: DuckDBResource):
"""
Expand Down Expand Up @@ -62,7 +63,8 @@ def trips_by_week(context, database: DuckDBResource):
## Lesson 4 (later part)
@asset(
deps=[AssetKey(["taxi_trips"]), AssetKey(["taxi_zones"])],
key_prefix="manhattan"
key_prefix="manhattan",
compute_kind="DuckDB",
)
def manhattan_stats(database: DuckDBResource):
"""
Expand Down Expand Up @@ -92,7 +94,8 @@ def manhattan_stats(database: DuckDBResource):

## Lesson 4 (later part)
@asset(
deps=[AssetKey(["manhattan", "manhattan_stats"])]
deps=[AssetKey(["manhattan", "manhattan_stats"])],
compute_kind="Python",
)
def manhattan_map(context):
"""
Expand Down
3 changes: 2 additions & 1 deletion dagster_university/assets/requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ class AdhocRequestConfig(Config):

## Lesson 9
@asset(
deps=["taxi_trips", "taxi_zones"]
deps=["taxi_trips", "taxi_zones"],
compute_kind="Python",
)
def adhoc_request(context, config: AdhocRequestConfig, database: DuckDBResource):
"""
Expand Down
12 changes: 8 additions & 4 deletions dagster_university/assets/trips.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

## Lesson 3 (change this to HW)
@asset(
group_name="raw_files"
group_name="raw_files",
compute_kind="Python",
)
def taxi_zones_file(context):
"""
Expand All @@ -28,7 +29,8 @@ def taxi_zones_file(context):
## Lesson 4 (HW) , 6
@asset(
deps=["taxi_zones_file"],
group_name="ingested"
group_name="ingested",
compute_kind="DuckDB",
)
def taxi_zones(database: DuckDBResource):
"""
Expand All @@ -52,7 +54,8 @@ def taxi_zones(database: DuckDBResource):
## Lesson 3, 8
@asset(
partitions_def=monthly_partition,
group_name="raw_files"
group_name="raw_files",
compute_kind="DuckDB",
)
def taxi_trips_file(context):
"""
Expand All @@ -76,7 +79,8 @@ def taxi_trips_file(context):
@asset(
deps=["taxi_trips_file"],
partitions_def=monthly_partition,
group_name="ingested"
group_name="ingested",
compute_kind="DuckDB",
)
def taxi_trips(context, database: DuckDBResource):
"""
Expand Down
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@
"plotly",
"shapely",
],
extras_require={"dev": ["dagit", "pytest"]},
)
extras_require={"dev": ["dagster-webserver", "pytest"]},
)

0 comments on commit 158702d

Please sign in to comment.