Skip to content

Commit

Permalink
Split up default data adapters
Browse files Browse the repository at this point in the history
With the change of applies_to -- it means
that single classes are not a good idea. So splitting
up the default ones appropriately.
  • Loading branch information
skrawcz committed Oct 4, 2023
1 parent 54f0639 commit f96fe39
Showing 1 changed file with 47 additions and 4 deletions.
51 changes: 47 additions & 4 deletions hamilton/io/default_data_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@


@dataclasses.dataclass
class JSONDataAdapter(DataLoader, DataSaver):
class JSONDataLoader(DataLoader):
path: str

@classmethod
Expand All @@ -24,14 +24,27 @@ def load_data(self, type_: Type) -> Tuple[dict, Dict[str, Any]]:
def name(cls) -> str:
return "json"


@dataclasses.dataclass
class JSONDataSaver(DataSaver):
path: str

@classmethod
def applicable_types(cls) -> Collection[Type]:
return [dict]

@classmethod
def name(cls) -> str:
return "json"

def save_data(self, data: Any) -> Dict[str, Any]:
with open(self.path, "w") as f:
json.dump(data, f)
return get_file_metadata(self.path)


@dataclasses.dataclass
class RawFileDataLoader(DataLoader, DataSaver):
class RawFileDataLoader(DataLoader):
path: str
encoding: str = "utf-8"

Expand All @@ -47,6 +60,20 @@ def applicable_types(cls) -> Collection[Type]:
def name(cls) -> str:
return "file"


@dataclasses.dataclass
class RawFileDataSaver(DataSaver):
path: str
encoding: str = "utf-8"

@classmethod
def applicable_types(cls) -> Collection[Type]:
return [str]

@classmethod
def name(cls) -> str:
return "file"

def save_data(self, data: Any) -> Dict[str, Any]:
with open(self.path, "w", encoding=self.encoding) as f:
f.write(data)
Expand All @@ -65,10 +92,23 @@ def applicable_types(cls) -> Collection[Type]:
def name(cls) -> str:
return "pickle"

def load_data(self, type_: Type[dict]) -> Tuple[str, Dict[str, Any]]:
def load_data(self, type_: Type[object]) -> Tuple[object, Dict[str, Any]]:
with open(self.path, "rb") as f:
return pickle.load(f), get_file_metadata(self.path)


@dataclasses.dataclass
class PickleSaver(DataSaver):
path: str

@classmethod
def applicable_types(cls) -> Collection[Type]:
return [object]

@classmethod
def name(cls) -> str:
return "pickle"

def save_data(self, data: Any) -> Dict[str, Any]:
with open(self.path, "wb") as f:
pickle.dump(data, f)
Expand Down Expand Up @@ -127,10 +167,13 @@ def name(cls) -> str:


DATA_ADAPTERS = [
JSONDataAdapter,
JSONDataSaver,
JSONDataLoader,
LiteralValueDataLoader,
RawFileDataLoader,
RawFileDataSaver,
PickleLoader,
PickleSaver,
EnvVarDataLoader,
InMemoryResult,
]

0 comments on commit f96fe39

Please sign in to comment.