Docs and a test for unnest / Proposed consistent API

datafusion/core/src/physical_plan/unnest.rs

@@ -44,6 +44,23 @@ use crate::physical_plan::{
 use super::DisplayAs;
 
 /// Unnest the given column by joining the row with each value in the nested type.
+///
+/// For example, calling unnest(c1) results in the following:
+///
+/// ```text
+/// ┌─────────┐ ┌─────┐                ┌─────────┐ ┌─────┐
+/// │ {1, 2}  │ │  A  │   Unnest       │    1    │ │  A  │
+/// ├─────────┤ ├─────┤                ├─────────┤ ├─────┤
+/// │  null   │ │  B  │                │    2    │ │  A  │
+/// ├─────────┤ ├─────┤ ────────────▶  ├─────────┤ ├─────┤
+/// │   {}    │ │  D  │                │  null   │ │  B  │
+/// ├─────────┤ ├─────┤                ├─────────┤ ├─────┤
+/// │   {3}   │ │  E  │                │  null   │ │  D  │
+/// └─────────┘ └─────┘                ├─────────┤ ├─────┤
+///   c1         c2                    │    3    │ │  E  │
+///                                    └─────────┘ └─────┘
+///                                        c1        c2
+/// ```
 #[derive(Debug)]
 pub struct UnnestExec {
     /// Input execution plan

datafusion/core/tests/dataframe/mod.rs

@@ -1114,6 +1114,71 @@ async fn unnest_fixed_list() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test]
+async fn unnest_fixed_list_nonull() -> Result<()> {
+    let mut shape_id_builder = UInt32Builder::new();
+    let mut tags_builder = FixedSizeListBuilder::new(StringBuilder::new(), 2);
+
+    for idx in 0..6 {
+        // Append shape id.
+        shape_id_builder.append_value(idx as u32 + 1);
+
+        tags_builder
+            .values()
+            .append_value(format!("tag{}1", idx + 1));
+        tags_builder
+            .values()
+            .append_value(format!("tag{}2", idx + 1));
+        tags_builder.append(true);
+    }
+
+    let batch = RecordBatch::try_from_iter(vec![
+        ("shape_id", Arc::new(shape_id_builder.finish()) as ArrayRef),
+        ("tags", Arc::new(tags_builder.finish()) as ArrayRef),
+    ])?;
+
+    let ctx = SessionContext::new();
+    ctx.register_batch("shapes", batch)?;
+    let df = ctx.table("shapes").await?;
+
+    let results = df.clone().collect().await?;
+    let expected = vec![
+        "+----------+----------------+",
+        "| shape_id | tags           |",
+        "+----------+----------------+",
+        "| 1        | [tag11, tag12] |",
+        "| 2        | [tag21, tag22] |",
+        "| 3        | [tag31, tag32] |",
+        "| 4        | [tag41, tag42] |",
+        "| 5        | [tag51, tag52] |",
+        "| 6        | [tag61, tag62] |",
+        "+----------+----------------+",
+    ];
+    assert_batches_sorted_eq!(expected, &results);
+
+    let results = df.unnest_column("tags")?.collect().await?;
+    let expected = vec![
+        "+----------+-------+",
+        "| shape_id | tags  |",
+        "+----------+-------+",
+        "| 1        | tag11 |",
+        "| 1        | tag12 |",
+        "| 2        | tag21 |",
+        "| 2        | tag22 |",
+        "| 3        | tag31 |",
+        "| 3        | tag32 |",
+        "| 4        | tag41 |",
+        "| 4        | tag42 |",
+        "| 5        | tag51 |",
+        "| 5        | tag52 |",
+        "| 6        | tag61 |",
+        "| 6        | tag62 |",
+        "+----------+-------+",
+    ];
+    assert_batches_sorted_eq!(expected, &results);
+
+    Ok(())
+}
 #[tokio::test]
 async fn unnest_aggregate_columns() -> Result<()> {
     const NUM_ROWS: usize = 5;

datafusion/expr/src/logical_plan/plan.rs

@@ -120,7 +120,8 @@ pub enum LogicalPlan {
     Ddl(DdlStatement),
     /// Describe the schema of table
     DescribeTable(DescribeTable),
-    /// Unnest a column that contains a nested list type.
+    /// Unnest a column that contains a nested list type. See
+    /// [`Unnest`] for more details.
     Unnest(Unnest),
 }
 
@@ -1753,6 +1754,23 @@ pub enum Partitioning {
 }
 
 /// Unnest a column that contains a nested list type.
+///
+/// For example, calling unnest(c1) results in the following:
+///
+/// ```text
+/// ┌─────────┐ ┌─────┐                ┌─────────┐ ┌─────┐
+/// │ {1, 2}  │ │  A  │   Unnest       │    1    │ │  A  │
+/// ├─────────┤ ├─────┤                ├─────────┤ ├─────┤
+/// │  null   │ │  B  │                │    2    │ │  A  │
+/// ├─────────┤ ├─────┤ ────────────▶  ├─────────┤ ├─────┤
+/// │   {}    │ │  D  │                │  null   │ │  B  │
+/// ├─────────┤ ├─────┤                ├─────────┤ ├─────┤
+/// │   {3}   │ │  E  │                │  null   │ │  D  │
+/// └─────────┘ └─────┘                ├─────────┤ ├─────┤
+///   c1         c2                    │    3    │ │  E  │
+///                                    └─────────┘ └─────┘
+///                                        c1        c2
+/// ```
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Unnest {
     /// The incoming logical plan