Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed error in writing dictionary with extensions (#397)
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Sep 12, 2021
1 parent f326fed commit 9fa21bd
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 28 deletions.
67 changes: 39 additions & 28 deletions src/io/ipc/convert.rs
Expand Up @@ -333,6 +333,29 @@ fn write_metadata<'a>(
}
}

fn write_extension<'a>(
fbb: &mut FlatBufferBuilder<'a>,
name: &str,
metadata: &Option<String>,
kv_vec: &mut Vec<WIPOffset<ipc::KeyValue<'a>>>,
) {
// metadata
if let Some(metadata) = metadata {
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:metadata")),
value: Some(fbb.create_string(metadata.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

// name
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:name")),
value: Some(fbb.create_string(name)),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

/// Create an IPC Field from an Arrow Field
pub(crate) fn build_field<'a>(
fbb: &mut FlatBufferBuilder<'a>,
Expand All @@ -341,39 +364,16 @@ pub(crate) fn build_field<'a>(
// custom metadata.
let mut kv_vec = vec![];
if let DataType::Extension(name, _, metadata) = field.data_type() {
// append extension information.

// metadata
if let Some(metadata) = metadata {
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:metadata")),
value: Some(fbb.create_string(metadata.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
}

// name
let kv_args = ipc::KeyValueArgs {
key: Some(fbb.create_string("ARROW:extension:name")),
value: Some(fbb.create_string(name.as_str())),
};
kv_vec.push(ipc::KeyValue::create(fbb, &kv_args));
write_extension(fbb, name, metadata, &mut kv_vec);
}
if let Some(metadata) = field.metadata() {
if !metadata.is_empty() {
write_metadata(fbb, metadata, &mut kv_vec);
}
};
let fb_metadata = if !kv_vec.is_empty() {
Some(fbb.create_vector(&kv_vec))
} else {
None
};

let fb_field_name = fbb.create_string(field.name().as_str());
let field_type = get_fb_field_type(field.data_type(), field.is_nullable(), fbb);

let fb_dictionary = if let Dictionary(index_type, _) = field.data_type() {
let fb_dictionary = if let Dictionary(index_type, inner) = field.data_type() {
if let DataType::Extension(name, _, metadata) = inner.as_ref() {
write_extension(fbb, name, metadata, &mut kv_vec);
}
Some(get_fb_dictionary(
index_type,
field
Expand All @@ -388,6 +388,17 @@ pub(crate) fn build_field<'a>(
None
};

if let Some(metadata) = field.metadata() {
if !metadata.is_empty() {
write_metadata(fbb, metadata, &mut kv_vec);
}
};
let fb_metadata = if !kv_vec.is_empty() {
Some(fbb.create_vector(&kv_vec))
} else {
None
};

let mut field_builder = ipc::FieldBuilder::new(fbb);
field_builder.add_name(fb_field_name);
if let Some(dictionary) = fb_dictionary {
Expand Down
6 changes: 6 additions & 0 deletions tests/it/io/ipc/write/file.rs
Expand Up @@ -162,6 +162,12 @@ fn write_100_decimal() -> Result<()> {
test_file("1.0.0-bigendian", "generated_decimal")
}

#[test]
fn write_100_extension() -> Result<()> {
test_file("1.0.0-littleendian", "generated_extension")?;
test_file("1.0.0-bigendian", "generated_extension")
}

#[test]
fn write_100_union() -> Result<()> {
test_file("1.0.0-littleendian", "generated_union")?;
Expand Down

0 comments on commit 9fa21bd

Please sign in to comment.