[WIP] Support deep schema pruning and projection

Cargo.toml

@@ -15,6 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
+[profile.idea]
+inherits = "dev"
+opt-level = 0
+debug = 2
+split-debuginfo = "packed"
+strip = "none"
+debug-assertions = true
+overflow-checks = false
+incremental = true
+codegen-units = 256
+lto = "off"
+
 [workspace]
 exclude = ["datafusion-cli", "dev/depcheck"]
 members = [

datafusion/catalog/src/table.rs

@@ -16,6 +16,7 @@
 // under the License.
 
 use std::any::Any;
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use crate::session::Session;
@@ -153,6 +154,28 @@ pub trait TableProvider: Sync + Send {
         limit: Option<usize>,
     ) -> Result<Arc<dyn ExecutionPlan>>;
 
+    /// Create an [`ExecutionPlan`] with an extra parameter
+    /// specifying the deep column projections
+    /// # Deep column projection
+    ///
+    /// If specified, a datasource such as Parquet can do deep projection pushdown.
+    /// In the case of deeply nested schemas (lists in structs etc), the
+    /// implementation can return a smaller schema that rewrites the entire file
+    /// schema to return only the necessary fields, no matter where they are (top-level
+    /// or deep)
+    ///
+    async fn scan_deep(
+           &self,
+            state: &dyn Session,
+            projection: Option<&Vec<usize>>,
+            _projection_deep: Option<&HashMap<usize, Vec<String>>>,
+            filters: &[Expr],
+            limit: Option<usize>,
+        ) -> Result<Arc<dyn ExecutionPlan>> {
+            self.scan(state, projection, filters, limit).await
+    }
+
+
     /// Specify if DataFusion should provide filter expressions to the
     /// TableProvider to apply *during* the scan.
     ///

datafusion/common/Cargo.toml

@@ -64,6 +64,7 @@ paste = "1.0.15"
 pyo3 = { version = "0.21.0", optional = true }
 sqlparser = { workspace = true }
 tokio = { workspace = true }
+log = { workspace = true }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
 instant = { version = "0.1", features = ["wasm-bindgen"] }