Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Fixed error writing dictionary extension to IPC #397

Merged
merged 1 commit into from
Sep 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 39 additions & 28 deletions src/io/ipc/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,29 @@ fn write_metadata<'a>(
}
}

fn write_extension<'a>(
fbb: &mut FlatBufferBuilder<'a>,
name: &str,
metadata: &Option<String>,
kv_vec: &mut Vec<WIPOffset<ipc::KeyValue<'a>>>,
) {
// metadata
if let Some(metadata) = metadata {
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:metadata")),
value: Some(fbb.create_string(metadata.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

// name
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:name")),
value: Some(fbb.create_string(name)),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

/// Create an IPC Field from an Arrow Field
pub(crate) fn build_field<'a>(
fbb: &mut FlatBufferBuilder<'a>,
Expand All @@ -341,39 +364,16 @@ pub(crate) fn build_field<'a>(
// custom metadata.
let mut kv_vec = vec![];
if let DataType::Extension(name, _, metadata) = field.data_type() {
// append extension information.

// metadata
if let Some(metadata) = metadata {
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:metadata")),
value: Some(fbb.create_string(metadata.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

// name
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:name")),
value: Some(fbb.create_string(name.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
write_extension(fbb, name, metadata, &mut kv_vec);
}
if let Some(metadata) = field.metadata() {
if !metadata.is_empty() {
write_metadata(fbb, metadata, &mut kv_vec);
}
};
let fb_metadata = if !kv_vec.is_empty() {
Some(fbb.create_vector(&kv_vec))
} else {
None
};

let fb_field_name = fbb.create_string(field.name().as_str());
let field_type = get_fb_field_type(field.data_type(), field.is_nullable(), fbb);

let fb_dictionary = if let Dictionary(index_type, _) = field.data_type() {
let fb_dictionary = if let Dictionary(index_type, inner) = field.data_type() {
if let DataType::Extension(name, _, metadata) = inner.as_ref() {
write_extension(fbb, name, metadata, &mut kv_vec);
}
Some(get_fb_dictionary(
index_type,
field
Expand All @@ -388,6 +388,17 @@ pub(crate) fn build_field<'a>(
None
};

if let Some(metadata) = field.metadata() {
if !metadata.is_empty() {
write_metadata(fbb, metadata, &mut kv_vec);
}
};
let fb_metadata = if !kv_vec.is_empty() {
Some(fbb.create_vector(&kv_vec))
} else {
None
};

let mut field_builder = ipc::FieldBuilder::new(fbb);
field_builder.add_name(fb_field_name);
if let Some(dictionary) = fb_dictionary {
Expand Down
6 changes: 6 additions & 0 deletions tests/it/io/ipc/write/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,12 @@ fn write_100_decimal() -> Result<()> {
test_file("1.0.0-bigendian", "generated_decimal")
}

#[test]
fn write_100_extension() -> Result<()> {
test_file("1.0.0-littleendian", "generated_extension")?;
test_file("1.0.0-bigendian", "generated_extension")
}

#[test]
fn write_100_union() -> Result<()> {
test_file("1.0.0-littleendian", "generated_union")?;
Expand Down