Support nested dictionaries in ipc serialization & deserialization

[helgikrs]

Ipc serialization and deserialization of a column containing a dictionary nested within a struct does not seem to work as expected right now. Here's a small code snippet exhibiting that behavior.

use arrow::{
    array::{self, ArrayRef, StructArray},
    datatypes::{self, Field, Schema},
    ipc::{reader::FileReader, writer::FileWriter},
    record_batch::RecordBatch,
};
use std::sync::Arc;

fn main() {
    // This works
    // let inner: array::StringArray = vec![Some("a"), Some("b"), Some("a")].into_iter().collect();

    // This does not
    let inner: array::DictionaryArray<datatypes::Int32Type> =
        vec!["a", "b", "a"].into_iter().collect();

    let array = Arc::new(inner) as ArrayRef;

    let dctfield = Field::new("dict", array.data_type().clone(), false);

    let s = StructArray::from(vec![(dctfield, array)]);
    let struct_array = Arc::new(s) as ArrayRef;

    let schema = Arc::new(Schema::new(vec![Field::new(
        "struct",
        struct_array.data_type().clone(),
        false,
    )]));

    let batch = RecordBatch::try_new(schema.clone(), vec![struct_array]).unwrap();

    let mut buf = Vec::new();
    let mut writer = FileWriter::try_new(&mut buf, &schema).unwrap();
    writer.write(&batch).unwrap();
    writer.finish().unwrap();
    drop(writer);

    let reader = FileReader::try_new(std::io::Cursor::new(buf)).unwrap();
    let batch2: Result<Vec<_>, _> = reader.collect();

    assert_eq!(batch, batch2.unwrap()[0]);
}

The code panics when the struct field is a dictionary, but works as expected for other types.

[alamb]

FWIW when I tried the reproducer on current apache/master I still see an error:

    Finished dev [unoptimized + debuginfo] target(s) in 18.38s
     Running `target/debug/rust_arrow_playground`
thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: InvalidArgumentError("dictionary id not found in schema")', src/main.rs:39:65
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

@helgikrs do you know what is left for this PR?

[helgikrs]

@alamb Currently the deserializer only considers dictionaries at the top-level of the schema, so when a dictionary id is found during loading that is not at the top-level of the schema you get the panic above. I have a working fix, I'll submit a PR that closes this probably tomorrow.

[alamb]

Thank you @helgikrs