Add RustyIceberg.with_pos_column!

iceberg_rust_ffi/Cargo.toml

@@ -12,14 +12,14 @@ default = ["julia"]
 julia = []
 
 [dependencies]
-iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "ae83309fd198ec30f052a7e9f983711c5f581aea" }
+iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "cd1daca8d45eb2a78bc90f1eec18435502c6bc04" }
 object_store_ffi = { git = "https://github.com/RelationalAI/object_store_ffi", rev = "79b08071c7a1642532b5891253280861eca9e44e", default-features = false }
 tokio = { version = "1.0", features = ["full"] }
 futures = "0.3"
 libc = "0.2"
 anyhow = "1.0"
-arrow-array = "56.2.0"
-arrow-ipc = "56.2.0"
+arrow-array = { git = "https://github.com/apache/arrow-rs", rev = "fea605cb16f7524cb69a197bfa581a1d4f5fe5d0" }
+arrow-ipc = { git = "https://github.com/apache/arrow-rs", rev = "fea605cb16f7524cb69a197bfa581a1d4f5fe5d0" }
 tracing-subscriber = "0.3"
 tracing = "0.1"
 once_cell = "1.19"

iceberg_rust_ffi/src/full.rs

@@ -54,6 +54,8 @@ impl_with_batch_size!(iceberg_scan_with_batch_size, IcebergScan);
 
 impl_scan_builder_method!(iceberg_scan_with_file_column, IcebergScan, with_file_column);
 
+impl_scan_builder_method!(iceberg_scan_with_pos_column, IcebergScan, with_pos_column);
+
 impl_scan_build!(iceberg_scan_build, IcebergScan);
 
 // Async function to initialize stream from a table scan

iceberg_rust_ffi/src/incremental.rs

@@ -130,6 +130,12 @@ impl_scan_builder_method!(
     with_file_column
 );
 
+impl_scan_builder_method!(
+    iceberg_incremental_scan_with_pos_column,
+    IcebergIncrementalScan,
+    with_pos_column
+);
+
 impl_scan_build!(iceberg_incremental_scan_build, IcebergIncrementalScan);
 
 // Get unzipped Arrow streams from incremental scan (async)

src/RustyIceberg.jl

@@ -11,9 +11,9 @@ export IcebergException
 export new_incremental_scan, free_incremental_scan!
 export table_open, free_table, new_scan, free_scan!
 export select_columns!, with_batch_size!, with_data_file_concurrency_limit!, with_manifest_entry_concurrency_limit!
-export with_file_column!
+export with_file_column!, with_pos_column!
 export scan!, next_batch, free_batch, free_stream
-export FILE_COLUMN
+export FILE_COLUMN, POS_COLUMN
 
 # Always use the JLL library - override via Preferences if needed for local development
 # To use a local build, set the preference:

src/full.jl

@@ -133,6 +133,32 @@ function with_file_column!(scan::Scan)
     return nothing
 end
 
+"""
+    with_pos_column!(scan::Scan)
+
+Add the _pos metadata column to the scan.
+
+The _pos column contains the position of each row within its data file, which can
+be useful for tracking row locations and debugging.
+
+# Example
+```julia
+scan = new_scan(table)
+with_pos_column!(scan)
+stream = scan!(scan)
+```
+"""
+function with_pos_column!(scan::Scan)
+    result = @ccall rust_lib.iceberg_scan_with_pos_column(
+        convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}
+    )::Cint
+
+    if result != 0
+        error("Failed to add pos column to scan")
+    end
+    return nothing
+end
+
 """
     build!(scan::Scan)