Skip to content

Commit

Permalink
add timezone property to pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
roelschr committed Sep 23, 2020
1 parent d1b5d2a commit 082d134
Showing 1 changed file with 18 additions and 0 deletions.
18 changes: 18 additions & 0 deletions butterfree/pipelines/feature_set_pipeline.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
"""FeatureSetPipeline entity."""
import os
import time
from typing import List

from butterfree.clients import SparkClient
Expand All @@ -16,6 +18,8 @@ class FeatureSetPipeline:
feature_set: feature set composed by features and context metadata.
sink: sink used to write the output dataframe in the desired locations.
spark_client: client used to access Spark connection.
timezone: timestamp feature transformations will assume this timezone
when they don't have a tz suffix.
Example:
This an example regarding the feature set pipeline definition. All
Expand Down Expand Up @@ -122,11 +126,25 @@ def __init__(
feature_set: FeatureSet,
sink: Sink,
spark_client: SparkClient = None,
timezone: str = "UTC",
):
self.source = source
self.feature_set = feature_set
self.sink = sink
self.spark_client = spark_client
self.timezone = timezone

@property
def timezone(self) -> str:
return self._timezone

@timezone.setter
def timezone(self, value: str):
if value:
self.spark_client.conn.conf.set("spark.sql.session.timeZone", value)
os.environ["TZ"] = value
time.tzset()
self._timezone = value

@property
def source(self) -> Source:
Expand Down

0 comments on commit 082d134

Please sign in to comment.