From 8d373db4f51ecb2ef4e0a8a3f3106502edbb9e9b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 17:34:54 +0000 Subject: [PATCH 1/7] Buffer incomplete trailing row in resumable Reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On input not ending in the row-terminating empty line, Reader used to emit the incomplete trailing row at EOF — the non-resumable behavior, diverging from the Java/Rust/Scala readers which buffer it. Align with them: preserve line and row buffers across StopIteration so iteration resumes cleanly when more data arrives (tailing), including a cell split across the EOF boundary. Non-resumable load/loads keep emitting the incomplete tail. --- README.md | 2 ++ nsv/reader.py | 26 +++++++++++++++---------- tests/samples/basic.nsv | 3 +++ tests/test_incremental.py | 41 +++++++++++++++++++++++++++++++++++++-- tests/test_utils.py | 2 +- 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0725ba7..18b660c 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ with open('output.nsv', 'w') as f: writer.write_row(['row2cell1', 'row2cell2', 'row2cell3']) ``` +At EOF, `Reader` raises `StopIteration` without emitting an incomplete trailing row (one not yet terminated by an empty line); the row stays buffered, so iteration resumes correctly if the file grows (e.g. tailing). The non-resumable `load`/`loads` treat EOF as the definitive end of data and do emit the incomplete tail. + ## Vendor The core NSV format is frozen by-design. diff --git a/nsv/reader.py b/nsv/reader.py index 76e0345..fd26731 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -1,23 +1,29 @@ class Reader: def __init__(self, file_obj): self._file_obj = file_obj + self._line_buffer = '' # incomplete line at EOF, preserved for next call + self._row_buffer = [] # incomplete row at EOF, preserved for next call def __iter__(self): return self def __next__(self): - acc = [] for line in self._file_obj: + line = self._line_buffer + line + self._line_buffer = '' + if line[-1] != '\n': + # Incomplete line at EOF, preserve for next call + self._line_buffer = line + break if line == '\n': - return acc - if line[-1] == '\n': # so as not to chop if missing newline at EOF - line = line[:-1] - acc.append(Reader.unescape(line)) # bruh - # at the end of the file - if acc: - return acc - else: # an empty row would self-report in the cycle body - raise StopIteration + # Row complete, return + row = self._row_buffer + self._row_buffer = [] + return row + # Cell complete, keep reading + self._row_buffer.append(Reader.unescape(line[:-1])) + # Incomplete row at EOF, preserve row_buffer for next call + raise StopIteration @staticmethod def unescape(s: str) -> str: diff --git a/tests/samples/basic.nsv b/tests/samples/basic.nsv index c95e716..e95cd5c 100644 --- a/tests/samples/basic.nsv +++ b/tests/samples/basic.nsv @@ -6,3 +6,6 @@ d e f +last1 +last2 + diff --git a/tests/test_incremental.py b/tests/test_incremental.py index 8ee769d..c1a5901 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -1,4 +1,5 @@ import unittest +import io import os import tempfile import nsv @@ -17,13 +18,49 @@ def test_incremental_reading(self): second = next(reader) self.assertEqual(second, ["d", "e", "f"]) - # third = next(reader) - # self.assertEqual(third, ["last1", "last2"]) + third = next(reader) + self.assertEqual(third, ["last1", "last2"]) # Should be at end of the file with self.assertRaises(StopIteration): next(reader) + def test_incomplete_trailing_row_buffered(self): + """A resumable Reader buffers an incomplete trailing row instead of emitting it.""" + reader = nsv.Reader(io.StringIO('a\nb\n\nc\nd')) + + self.assertEqual(next(reader), ['a', 'b']) + # The trailing row is not terminated by an empty line: buffer, don't emit + with self.assertRaises(StopIteration): + next(reader) + # Same for a cell-terminated but row-unterminated tail + reader = nsv.Reader(io.StringIO('a\nb\n\nc\nd\n')) + self.assertEqual(next(reader), ['a', 'b']) + with self.assertRaises(StopIteration): + next(reader) + + def test_reading_resumes_after_eof(self): + """Reading continues across EOF once more data is appended (tailing).""" + with tempfile.TemporaryDirectory() as output_dir: + file_path = os.path.join(output_dir, 'tail.nsv') + with open(file_path, 'w') as f: + f.write('a\nb') + f.flush() + + with open(file_path, 'r') as rf: + reader = nsv.Reader(rf) + with self.assertRaises(StopIteration): + next(reader) + + # Complete the row, splitting a cell across the EOF boundary + f.write('c\nd\n\n') + f.flush() + self.assertEqual(next(reader), ['a', 'bc', 'd']) + + def test_load_emits_incomplete_trailing_row(self): + """Non-resumable loads keeps emitting the incomplete tail.""" + self.assertEqual(nsv.loads('a\nb\n\nc\nd'), [['a', 'b'], ['c', 'd']]) + def test_incremental_writing(self): """Test writing elements incrementally.""" data = [["field1", "field2"], ["value1", "value2"], ["last1", "last2"]] diff --git a/tests/test_utils.py b/tests/test_utils.py index 802f130..082d875 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,7 @@ 'empty_one': [[]], 'empty_two': [[], []], 'empty_three': [[], [], []], - 'basic': [["a", "b", "c"], ["d", "e", "f"]], + 'basic': [["a", "b", "c"], ["d", "e", "f"], ["last1", "last2"]], 'comments': [["# This is a comment", "// Another comment", "-- And another"], ["---"], ["r1c1", "r1c2"], ["r2c1", "r2c2"]], 'empty_fields': [["r1c1", "", "r1c3"], ["r2c1", "", "r2c3"]], 'empty_sequence': [["r1c1", "r1c2"], [], ["r3c1", "r3c2"]], From 39793210915bc1280aaa5405edafdcacafb85d22 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 17:57:08 +0000 Subject: [PATCH 2/7] Restructure Reader.__next__ to minimize diff vs previous logic Same semantics as the previous commit, but keeping the original check order and loop shape so the change reads as a delta: acc promoted to a persistent row buffer, a line buffer for the partial line at EOF, and the EOF tail emission removed. --- nsv/reader.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/nsv/reader.py b/nsv/reader.py index fd26731..eda209e 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -11,18 +11,14 @@ def __next__(self): for line in self._file_obj: line = self._line_buffer + line self._line_buffer = '' - if line[-1] != '\n': - # Incomplete line at EOF, preserve for next call - self._line_buffer = line - break if line == '\n': - # Row complete, return - row = self._row_buffer - self._row_buffer = [] + row, self._row_buffer = self._row_buffer, [] return row - # Cell complete, keep reading - self._row_buffer.append(Reader.unescape(line[:-1])) - # Incomplete row at EOF, preserve row_buffer for next call + if line[-1] == '\n': + self._row_buffer.append(Reader.unescape(line[:-1])) + else: # no trailing newline only happens at EOF: incomplete line, keep + self._line_buffer = line + # at the end of the file; incomplete row stays buffered for resumption raise StopIteration @staticmethod From 3129586ec8a14500691e1b5a61262953c550043c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 18:21:43 +0000 Subject: [PATCH 3/7] Accumulate partial-line pieces in a list, join on completion The previous prepend (line_buffer + line) re-copied the accumulated prefix on every resume, so a large cell arriving in small reads while tailing cost O(n^2). Keep the pieces in a list and join once when the terminating newline arrives. Lines that arrive complete skip the list entirely (two cheap checks, no append/clear). --- nsv/reader.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nsv/reader.py b/nsv/reader.py index eda209e..1230505 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -1,7 +1,7 @@ class Reader: def __init__(self, file_obj): self._file_obj = file_obj - self._line_buffer = '' # incomplete line at EOF, preserved for next call + self._line_parts = [] # pieces of an incomplete line at EOF, joined once on completion self._row_buffer = [] # incomplete row at EOF, preserved for next call def __iter__(self): @@ -9,16 +9,18 @@ def __iter__(self): def __next__(self): for line in self._file_obj: - line = self._line_buffer + line - self._line_buffer = '' + if line[-1] != '\n': # only happens at EOF: incomplete line, keep the piece + self._line_parts.append(line) + continue + if self._line_parts: + self._line_parts.append(line) + line = ''.join(self._line_parts) + self._line_parts.clear() if line == '\n': row, self._row_buffer = self._row_buffer, [] return row - if line[-1] == '\n': - self._row_buffer.append(Reader.unescape(line[:-1])) - else: # no trailing newline only happens at EOF: incomplete line, keep - self._line_buffer = line - # at the end of the file; incomplete row stays buffered for resumption + self._row_buffer.append(Reader.unescape(line[:-1])) + # at the end of the file; incomplete row and line stay buffered for resumption raise StopIteration @staticmethod From fd0d9f8afa9b22f90f8e344441ab6272d94a8111 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 18:46:10 +0000 Subject: [PATCH 4/7] Keep basic.nsv basic; trim comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert the third row added to the basic sample and its expected data; drop the stale commented-out assertion instead of restoring it — the EOF contract has its own tests. Reduce reader comments to the one non-obvious invariant. --- nsv/reader.py | 7 +++---- tests/samples/basic.nsv | 3 --- tests/test_incremental.py | 8 +------- tests/test_utils.py | 2 +- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/nsv/reader.py b/nsv/reader.py index 1230505..14f2370 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -1,15 +1,15 @@ class Reader: def __init__(self, file_obj): self._file_obj = file_obj - self._line_parts = [] # pieces of an incomplete line at EOF, joined once on completion - self._row_buffer = [] # incomplete row at EOF, preserved for next call + self._line_parts = [] + self._row_buffer = [] def __iter__(self): return self def __next__(self): for line in self._file_obj: - if line[-1] != '\n': # only happens at EOF: incomplete line, keep the piece + if line[-1] != '\n': # missing newline = EOF mid-line self._line_parts.append(line) continue if self._line_parts: @@ -20,7 +20,6 @@ def __next__(self): row, self._row_buffer = self._row_buffer, [] return row self._row_buffer.append(Reader.unescape(line[:-1])) - # at the end of the file; incomplete row and line stay buffered for resumption raise StopIteration @staticmethod diff --git a/tests/samples/basic.nsv b/tests/samples/basic.nsv index e95cd5c..c95e716 100644 --- a/tests/samples/basic.nsv +++ b/tests/samples/basic.nsv @@ -6,6 +6,3 @@ d e f -last1 -last2 - diff --git a/tests/test_incremental.py b/tests/test_incremental.py index c1a5901..ed3d9ab 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -18,9 +18,6 @@ def test_incremental_reading(self): second = next(reader) self.assertEqual(second, ["d", "e", "f"]) - third = next(reader) - self.assertEqual(third, ["last1", "last2"]) - # Should be at end of the file with self.assertRaises(StopIteration): next(reader) @@ -28,12 +25,10 @@ def test_incremental_reading(self): def test_incomplete_trailing_row_buffered(self): """A resumable Reader buffers an incomplete trailing row instead of emitting it.""" reader = nsv.Reader(io.StringIO('a\nb\n\nc\nd')) - self.assertEqual(next(reader), ['a', 'b']) - # The trailing row is not terminated by an empty line: buffer, don't emit with self.assertRaises(StopIteration): next(reader) - # Same for a cell-terminated but row-unterminated tail + reader = nsv.Reader(io.StringIO('a\nb\n\nc\nd\n')) self.assertEqual(next(reader), ['a', 'b']) with self.assertRaises(StopIteration): @@ -52,7 +47,6 @@ def test_reading_resumes_after_eof(self): with self.assertRaises(StopIteration): next(reader) - # Complete the row, splitting a cell across the EOF boundary f.write('c\nd\n\n') f.flush() self.assertEqual(next(reader), ['a', 'bc', 'd']) diff --git a/tests/test_utils.py b/tests/test_utils.py index 082d875..802f130 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,7 @@ 'empty_one': [[]], 'empty_two': [[], []], 'empty_three': [[], [], []], - 'basic': [["a", "b", "c"], ["d", "e", "f"], ["last1", "last2"]], + 'basic': [["a", "b", "c"], ["d", "e", "f"]], 'comments': [["# This is a comment", "// Another comment", "-- And another"], ["---"], ["r1c1", "r1c2"], ["r2c1", "r2c2"]], 'empty_fields': [["r1c1", "", "r1c3"], ["r2c1", "", "r2c3"]], 'empty_sequence': [["r1c1", "r1c2"], [], ["r3c1", "r3c2"]], From 6d0dd0eef5adacd7388c77c8d7acee7b07471fd0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 19:06:05 +0000 Subject: [PATCH 5/7] Replace clear() with a fresh list Reusing the object would silently wipe any outside view of the parts, e.g. if peeking is ever introduced. --- nsv/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nsv/reader.py b/nsv/reader.py index 14f2370..84358c1 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -15,7 +15,7 @@ def __next__(self): if self._line_parts: self._line_parts.append(line) line = ''.join(self._line_parts) - self._line_parts.clear() + self._line_parts = [] if line == '\n': row, self._row_buffer = self._row_buffer, [] return row From 80e2f5641d6af598126428d37eb264357d52306f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 20:10:43 +0000 Subject: [PATCH 6/7] Revert README addition --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 18b660c..0725ba7 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,6 @@ with open('output.nsv', 'w') as f: writer.write_row(['row2cell1', 'row2cell2', 'row2cell3']) ``` -At EOF, `Reader` raises `StopIteration` without emitting an incomplete trailing row (one not yet terminated by an empty line); the row stays buffered, so iteration resumes correctly if the file grows (e.g. tailing). The non-resumable `load`/`loads` treat EOF as the definitive end of data and do emit the incomplete tail. - ## Vendor The core NSV format is frozen by-design. From 9d29bafb7d317313349d238c53b4f4c2fefdae2b Mon Sep 17 00:00:00 2001 From: namingbe Date: Tue, 9 Jun 2026 22:29:45 +0200 Subject: [PATCH 7/7] bruh --- nsv/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nsv/reader.py b/nsv/reader.py index 84358c1..6f8d0d8 100644 --- a/nsv/reader.py +++ b/nsv/reader.py @@ -19,7 +19,7 @@ def __next__(self): if line == '\n': row, self._row_buffer = self._row_buffer, [] return row - self._row_buffer.append(Reader.unescape(line[:-1])) + self._row_buffer.append(Reader.unescape(line[:-1])) # bruh raise StopIteration @staticmethod