Skip to content

Commit

Permalink
Improve serialization for jsonl format
Browse files Browse the repository at this point in the history
 * Imrpove logic for text and html tracking
 * Fix regressions
  • Loading branch information
polyaxon-ci committed Sep 29, 2023
1 parent 6aabcd3 commit a17ebe4
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 6 deletions.
23 changes: 20 additions & 3 deletions traceml/traceml/events/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,23 @@ def get_value(self, dump=True):
if self.dataframe is not None:
return self.dataframe.to_json() if dump else self.dataframe

def to_json(
self,
humanize_values: bool = False,
include_kind: bool = False,
include_version: bool = False,
kind: Optional[str] = None,
) -> str:
if not kind:
kind = self._IDENTIFIER
values = {
"timestamp": str(self.timestamp),
"step": self.step,
kind: self.get_value(dump=True),
}

return json.dumps(values)

def to_csv(self) -> str:
values = [
str(self.step) if self.step is not None else "",
Expand Down Expand Up @@ -417,8 +434,8 @@ def _read_jsonl(
# Pyarrow automatically converts timestamp fields
if "timestamp" in df.columns:
df["timestamp"] = df["timestamp"].astype(str)
if not hasattr(df, 'step'):
df['step'] = None
if not hasattr(df, "step"):
df["step"] = None
return df

@classmethod
Expand Down Expand Up @@ -515,7 +532,7 @@ def get_csv_events(self) -> str:
return "".join(events)

def get_jsonl_events(self) -> str:
events = ["\n{}".format(e.to_json()) for e in self.events]
events = ["\n{}".format(e.to_json(kind=self.kind)) for e in self.events]
return "".join(events)

def empty_events(self):
Expand Down
4 changes: 1 addition & 3 deletions traceml/traceml/tracking/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1225,9 +1225,7 @@ def log_html(
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event.make(
timestamp=timestamp, step=step, html=html.replace("\n", "")
),
event=V1Event.make(timestamp=timestamp, step=step, html=html),
)
self._add_event(logged_event)

Expand Down

0 comments on commit a17ebe4

Please sign in to comment.