Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix JSONDecodeError in JournalStorage #5195

Merged
merged 12 commits into from
Feb 5, 2024
16 changes: 14 additions & 2 deletions optuna/storages/_journal/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,28 +164,40 @@ def __init__(self, file_path: str, lock_obj: Optional[JournalFileBaseLock] = Non
def read_logs(self, log_number_from: int) -> List[Dict[str, Any]]:
logs = []
with open(self._file_path, "rb") as f:
remaining_log_size = os.stat(self._file_path).st_size
log_number_start = 0
if log_number_from in self._log_number_offset:
f.seek(self._log_number_offset[log_number_from])
log_number_start = log_number_from
remaining_log_size -= self._log_number_offset[log_number_from]

last_decode_error = None
for log_number, line in enumerate(f, start=log_number_start):
if last_decode_error is not None:
raise last_decode_error
byte_len = len(line)
remaining_log_size -= byte_len
if remaining_log_size < 0:
break
if log_number + 1 not in self._log_number_offset:
byte_len = len(line)
self._log_number_offset[log_number + 1] = (
self._log_number_offset[log_number] + byte_len
)
if log_number < log_number_from:
continue
if line.decode("utf-8") in [
"\n",
"\r",
"\r\n",
]: # to avoid json.loads(<line separator>)
y0z marked this conversation as resolved.
Show resolved Hide resolved
continue
try:
logs.append(json.loads(line))
except json.JSONDecodeError as err:
last_decode_error = err
del self._log_number_offset[log_number + 1]

if remaining_log_size == 0:
break
return logs

def append_logs(self, logs: List[Dict[str, Any]]) -> None:
Expand Down