Skip to content

Commit

Permalink
Merge pull request #35 from Textualize/headers
Browse files Browse the repository at this point in the history
Headers
  • Loading branch information
willmcgugan committed Feb 18, 2024
2 parents 54b455a + 01a3980 commit 00c29e2
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 12 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "toolong"
version = "1.2.0"
version = "1.2.1"
description = "A terminal log file viewer / tailer / analyzer"
authors = ["Will McGugan <will@textualize.io>"]
license = "MIT"
Expand Down
5 changes: 2 additions & 3 deletions src/toolong/log_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,16 +214,15 @@ def scan_timestamps(
scan_time = monotonic()
scan = self.timestamp_scanner.scan
line_no = 0
timestamp = self.get_create_time() or datetime.now()
position = 0
results: list[tuple[int, int, float]] = []
append = results.append
get_length = results.__len__
while line_bytes := log_mmap.readline():
line = line_bytes.decode("utf-8", errors="replace")
timestamp = scan(line) or timestamp
append((line_no, position, timestamp.timestamp()))
timestamp = scan(line)
position += len(line_bytes)
append((line_no, position, timestamp.timestamp() if timestamp else 0.0))
line_no += 1
if (
results
Expand Down
34 changes: 26 additions & 8 deletions src/toolong/log_lines.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,7 @@ def run_scan(self, save_merge: str | None = None) -> None:
def merge_log_files(self) -> None:
worker = get_current_worker()
self._merge_lines = []
merge_lines: list[tuple[float, int, LogFile]] = self._merge_lines
append_meta = merge_lines.append
merge_lines = self._merge_lines

for log_file in self.log_files:
try:
Expand All @@ -363,13 +362,17 @@ def merge_log_files(self) -> None:
for log_file in self.log_files:
if not log_file.is_open:
continue
append = self._line_breaks[log_file].append
line_breaks = self._line_breaks[log_file]
append = line_breaks.append
meta: list[tuple[float, int, LogFile]] = []
append_meta = meta.append
for timestamps in log_file.scan_timestamps():
break_position = 0

for line_no, break_position, timestamp in timestamps:
if break_position:
append_meta((timestamp, line_no, log_file))
append(break_position)
append_meta((timestamp, line_no, log_file))
append(break_position)
append(log_file.size)

self.post_message(
ScanProgress(
Expand All @@ -383,6 +386,20 @@ def merge_log_files(self) -> None:
)
return

# Header may be missing timestamp, so we will attempt to back fill timestamps
seconds = 0.0
for offset, (seconds, line_no, log_file) in enumerate(meta):
if seconds:
for index, (_seconds, line_no, log_file) in zip(
range(offset), meta
):
meta[index] = (seconds, line_no, log_file)
break
if offset > 10:
# May be pointless to scan the entire thing
break
self._merge_lines.extend(meta)

position += log_file.size

merge_lines.sort(key=itemgetter(0, 1))
Expand Down Expand Up @@ -447,11 +464,12 @@ def get_log_file_from_index(self, index: int) -> tuple[LogFile, int]:
def index_to_span(self, index: int) -> tuple[LogFile, int, int]:
log_file, index = self.get_log_file_from_index(index)
line_breaks = self._line_breaks.setdefault(log_file, [])
scan_start = 0 if self._merge_lines else self._scan_start
if not line_breaks:
return (log_file, self._scan_start, self._scan_start)
return (log_file, scan_start, self._scan_start)
index = clamp(index, 0, len(line_breaks))
if index == 0:
return (log_file, self._scan_start, line_breaks[0])
return (log_file, scan_start, line_breaks[0])
start = line_breaks[index - 1]
end = (
line_breaks[index]
Expand Down

0 comments on commit 00c29e2

Please sign in to comment.