fix(json): ndjson with trailing newlines (#5787)

polars/polars-io/src/ndjson_core/ndjson.rs

@@ -349,10 +349,15 @@ pub(crate) fn get_line_stats_json(bytes: &[u8], n_lines: usize) -> Option<(f32,
 
     let mut n_read = 0;
 
+    let bytes_len = bytes.len();
+
     // sample from start and 75% in the file
-    for offset in [0, (bytes.len() as f32 * 0.75) as usize] {
+    for offset in [0, (bytes_len as f32 * 0.75) as usize] {
         bytes_trunc = &bytes[offset..];
         let pos = next_line_position_naive_json(bytes_trunc)?;
+        if pos >= bytes_len {
+            return None;
+        }
         bytes_trunc = &bytes_trunc[pos + 1..];
 
         for _ in offset..(offset + n_lines_per_iter) {

polars/tests/it/io/json.rs

@@ -108,3 +108,20 @@ fn read_unordered_json() {
     assert_eq!("d", df.get_columns()[3].name());
     assert_eq!((12, 4), df.shape());
 }
+
+#[test]
+fn read_ndjson_with_trailing_newline() {
+    let data = r#"{"Column1":"Value1"}\n"#;
+
+    let file = Cursor::new(data);
+    let df = JsonReader::new(file)
+        .with_json_format(JsonFormat::JsonLines)
+        .finish()
+        .unwrap();
+
+    let expected = df! {
+        "Column1" => ["Value1"]
+    }
+    .unwrap();
+    assert!(expected.frame_equal(&df));
+}

py-polars/tests/unit/io/test_json.py

@@ -73,3 +73,12 @@ def test_write_json2(df: pl.DataFrame) -> None:
     file.seek(0)
     out = pl.read_json(file)
     assert df.frame_equal(out, null_equal=True)
+
+
+def test_ndjson_with_trailing_newline() -> None:
+
+    input = """{"Column1":"Value1"}\n"""
+
+    df = pl.read_ndjson(io.StringIO(input))
+    expected = pl.DataFrame({"Column1": ["Value1"]})
+    assert df.frame_equal(expected)