From c51b888875eb61d09c6269e39c635acea0e346c8 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 22 Nov 2025 15:48:58 -0500 Subject: [PATCH 1/7] Implement FFI_ExtensionOptions --- datafusion/common/src/config.rs | 33 +- .../ffi/src/config/extension_options.rs | 285 ++++++++++++++++++ datafusion/ffi/src/config/mod.rs | 75 +++++ datafusion/ffi/src/lib.rs | 1 + datafusion/ffi/src/session/config.rs | 36 +-- 5 files changed, 397 insertions(+), 33 deletions(-) create mode 100644 datafusion/ffi/src/config/extension_options.rs create mode 100644 datafusion/ffi/src/config/mod.rs diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index dad12c1c6bc91..3b9507b1acf1a 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1256,7 +1256,7 @@ impl<'a> TryInto> for &'a FormatOptions } /// A key value pair, with a corresponding description -#[derive(Debug, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ConfigEntry { /// A unique string to identify this config value pub key: String, @@ -1352,6 +1352,8 @@ impl ConfigField for ConfigOptions { } } +pub const DATAFUSION_FFI_CONFIG_NAMESPACE: &str = "datafusion_ffi"; + impl ConfigOptions { /// Creates a new [`ConfigOptions`] with default values pub fn new() -> Self { @@ -1366,12 +1368,12 @@ impl ConfigOptions { /// Set a configuration option pub fn set(&mut self, key: &str, value: &str) -> Result<()> { - let Some((prefix, key)) = key.split_once('.') else { + let Some((mut prefix, mut inner_key)) = key.split_once('.') else { return _config_err!("could not find config namespace for key \"{key}\""); }; if prefix == "datafusion" { - if key == "optimizer.enable_dynamic_filter_pushdown" { + if inner_key == "optimizer.enable_dynamic_filter_pushdown" { let bool_value = value.parse::().map_err(|e| { DataFusionError::Configuration(format!( "Failed to parse '{value}' as bool: {e}", @@ -1386,13 +1388,23 @@ impl ConfigOptions { } return Ok(()); } - return ConfigField::set(self, key, value); + return ConfigField::set(self, inner_key, value); + } + + if !self.extensions.0.contains_key(prefix) + && self + .extensions + .0 + .contains_key(DATAFUSION_FFI_CONFIG_NAMESPACE) + { + inner_key = key; + prefix = DATAFUSION_FFI_CONFIG_NAMESPACE; } let Some(e) = self.extensions.0.get_mut(prefix) else { return _config_err!("Could not find config namespace \"{prefix}\""); }; - e.0.set(key, value) + e.0.set(inner_key, value) } /// Create new [`ConfigOptions`], taking values from environment variables @@ -2157,7 +2169,7 @@ impl TableOptions { /// /// A result indicating success or failure in setting the configuration option. pub fn set(&mut self, key: &str, value: &str) -> Result<()> { - let Some((prefix, _)) = key.split_once('.') else { + let Some((mut prefix, _)) = key.split_once('.') else { return _config_err!("could not find config namespace for key \"{key}\""); }; @@ -2169,6 +2181,15 @@ impl TableOptions { return Ok(()); } + if !self.extensions.0.contains_key(prefix) + && self + .extensions + .0 + .contains_key(DATAFUSION_FFI_CONFIG_NAMESPACE) + { + prefix = DATAFUSION_FFI_CONFIG_NAMESPACE; + } + let Some(e) = self.extensions.0.get_mut(prefix) else { return _config_err!("Could not find config namespace \"{prefix}\""); }; diff --git a/datafusion/ffi/src/config/extension_options.rs b/datafusion/ffi/src/config/extension_options.rs new file mode 100644 index 0000000000000..09137dac8b27e --- /dev/null +++ b/datafusion/ffi/src/config/extension_options.rs @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; +use std::collections::HashMap; +use std::ffi::c_void; + +use abi_stable::StableAbi; +use abi_stable::std_types::{RResult, RStr, RString, RVec, Tuple2}; +use datafusion_common::config::{ConfigEntry, ConfigExtension, ExtensionOptions}; +use datafusion_common::{Result, exec_err}; + +use crate::df_result; + +/// A stable struct for sharing [`ExtensionOptions`] across FFI boundaries. +/// +/// Unlike other FFI structs in this crate, we do not construct a foreign +/// variant of this object. This is due to the typical method for interacting +/// with extension options is by creating a local struct of your concrete type. +/// To support this methodology use the `to_extension` method instead. +/// +/// When using [`FFI_ExtensionOptions`] with multiple extensions, all extension +/// values are stored on a single [`FFI_ExtensionOptions`] object. The keys +/// are stored with the full path prefix to avoid overwriting values when using +/// multiple extensions. +#[repr(C)] +#[derive(Debug, StableAbi)] +pub struct FFI_ExtensionOptions { + /// Return a deep clone of this [`ExtensionOptions`] + pub cloned: unsafe extern "C" fn(&Self) -> FFI_ExtensionOptions, + + /// Set the given `key`, `value` pair + pub set: + unsafe extern "C" fn(&mut Self, key: RStr, value: RStr) -> RResult<(), RString>, + + /// Returns the [`ConfigEntry`] stored in this [`ExtensionOptions`] + pub entries: unsafe extern "C" fn(&Self) -> RVec>, + + /// Release the memory of the private data when it is no longer being used. + pub release: unsafe extern "C" fn(&mut Self), + + /// Internal data. This is only to be accessed by the provider of the options. + pub private_data: *mut c_void, +} + +unsafe impl Send for FFI_ExtensionOptions {} +unsafe impl Sync for FFI_ExtensionOptions {} + +pub struct ExtensionOptionsPrivateData { + pub options: HashMap, +} + +impl FFI_ExtensionOptions { + #[inline] + fn inner_mut(&mut self) -> &mut HashMap { + let private_data = self.private_data as *mut ExtensionOptionsPrivateData; + unsafe { &mut (*private_data).options } + } + + #[inline] + fn inner(&self) -> &HashMap { + let private_data = self.private_data as *const ExtensionOptionsPrivateData; + unsafe { &(*private_data).options } + } +} + +unsafe extern "C" fn cloned_fn_wrapper( + options: &FFI_ExtensionOptions, +) -> FFI_ExtensionOptions { + options + .inner() + .iter() + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .collect::>() + .into() +} + +unsafe extern "C" fn set_fn_wrapper( + options: &mut FFI_ExtensionOptions, + key: RStr, + value: RStr, +) -> RResult<(), RString> { + let _ = options.inner_mut().insert(key.into(), value.into()); + RResult::ROk(()) +} + +unsafe extern "C" fn entries_fn_wrapper( + options: &FFI_ExtensionOptions, +) -> RVec> { + options + .inner() + .iter() + .map(|(key, value)| (key.to_owned().into(), value.to_owned().into()).into()) + .collect() +} + +unsafe extern "C" fn release_fn_wrapper(options: &mut FFI_ExtensionOptions) { + let private_data = unsafe { + Box::from_raw(options.private_data as *mut ExtensionOptionsPrivateData) + }; + drop(private_data); +} + +impl Default for FFI_ExtensionOptions { + fn default() -> Self { + HashMap::new().into() + } +} + +impl From> for FFI_ExtensionOptions { + fn from(options: HashMap) -> Self { + let private_data = ExtensionOptionsPrivateData { options }; + + Self { + cloned: cloned_fn_wrapper, + set: set_fn_wrapper, + entries: entries_fn_wrapper, + release: release_fn_wrapper, + private_data: Box::into_raw(Box::new(private_data)) as *mut c_void, + } + } +} + +impl Drop for FFI_ExtensionOptions { + fn drop(&mut self) { + unsafe { (self.release)(self) } + } +} + +impl Clone for FFI_ExtensionOptions { + fn clone(&self) -> Self { + unsafe { (self.cloned)(self) } + } +} + +impl ConfigExtension for FFI_ExtensionOptions { + const PREFIX: &'static str = + datafusion_common::config::DATAFUSION_FFI_CONFIG_NAMESPACE; +} + +impl ExtensionOptions for FFI_ExtensionOptions { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn cloned(&self) -> Box { + let ffi_options = unsafe { (self.cloned)(self) }; + Box::new(ffi_options) + } + + fn set(&mut self, key: &str, value: &str) -> Result<()> { + if key.split_once('.').is_none() { + return exec_err!("Unable to set FFI config value without namespace set"); + }; + + df_result!(unsafe { (self.set)(self, key.into(), value.into()) }) + } + + fn entries(&self) -> Vec { + unsafe { + (self.entries)(self) + .into_iter() + .map(|entry_tuple| ConfigEntry { + key: entry_tuple.0.into(), + value: Some(entry_tuple.1.into()), + description: "ffi_config_options", + }) + .collect() + } + } +} + +impl FFI_ExtensionOptions { + /// Add all of the values in a concrete configuration extension to the + /// FFI variant. This is safe to call on either side of the FFI + /// boundary. + pub fn add_config(&mut self, config: &C) -> Result<()> { + for entry in config.entries() { + if let Some(value) = entry.value { + let key = format!("{}.{}", C::PREFIX, entry.key); + self.set(key.as_str(), value.as_str())?; + } + } + + Ok(()) + } + + /// Merge another `FFI_ExtensionOptions` configurations into this one. + /// This is safe to call on either side of the FFI boundary. + pub fn merge(&mut self, other: &FFI_ExtensionOptions) -> Result<()> { + for entry in other.entries() { + if let Some(value) = entry.value { + self.set(entry.key.as_str(), value.as_str())?; + } + } + Ok(()) + } + + /// Create a concrete extension type from the FFI variant. + /// This is safe to call on either side of the FFI boundary. + pub fn to_extension(&self) -> Result { + let mut result = C::default(); + + unsafe { + for entry in (self.entries)(self) { + let key = entry.0.as_str(); + let value = entry.1.as_str(); + + if let Some((prefix, inner_key)) = key.split_once('.') + && prefix == C::PREFIX + { + result.set(inner_key, value)?; + } + } + } + + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use datafusion_common::config::{ConfigExtension, ConfigOptions}; + use datafusion_common::extensions_options; + + use crate::config::extension_options::FFI_ExtensionOptions; + + // Define a new configuration struct using the `extensions_options` macro + extensions_options! { + /// My own config options. + pub struct MyConfig { + /// Should "foo" be replaced by "bar"? + pub foo_to_bar: bool, default = true + + /// How many "baz" should be created? + pub baz_count: usize, default = 1337 + } + } + + impl ConfigExtension for MyConfig { + const PREFIX: &'static str = "my_config"; + } + + #[test] + fn round_trip_ffi_extension_options() { + // set up config struct and register extension + let mut config = ConfigOptions::default(); + let mut ffi_options = FFI_ExtensionOptions::default(); + ffi_options.add_config(&MyConfig::default()).unwrap(); + + config.extensions.insert(ffi_options); + + // overwrite config default + config.set("my_config.baz_count", "42").unwrap(); + + // check config state + let returned_ffi_config = + config.extensions.get::().unwrap(); + let my_config: MyConfig = returned_ffi_config.to_extension().unwrap(); + + // check default value + assert!(my_config.foo_to_bar); + + // check overwritten value + assert_eq!(my_config.baz_count, 42); + } +} diff --git a/datafusion/ffi/src/config/mod.rs b/datafusion/ffi/src/config/mod.rs new file mode 100644 index 0000000000000..ead1eed8b654d --- /dev/null +++ b/datafusion/ffi/src/config/mod.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod extension_options; + +use abi_stable::StableAbi; +use abi_stable::std_types::{RHashMap, RString}; +use datafusion_common::config::{ConfigOptions, ExtensionOptions}; +use datafusion_common::{DataFusionError, Result}; + +use crate::config::extension_options::FFI_ExtensionOptions; + +// TODO(tsaucer) add text about how extension options will require user to convert to concrete type. +#[repr(C)] +#[derive(Debug, Clone, StableAbi)] +pub struct FFI_ConfigOptions { + base_options: RHashMap, + + extensions: FFI_ExtensionOptions, +} + +impl From<&ConfigOptions> for FFI_ConfigOptions { + fn from(options: &ConfigOptions) -> Self { + let base_options: RHashMap = options + .entries() + .into_iter() + .filter_map(|entry| entry.value.map(|value| (entry.key, value))) + .map(|(key, value)| (key.into(), value.into())) + .collect(); + + let mut extensions = FFI_ExtensionOptions::default(); + for (extension_name, extension) in options.extensions.iter() { + for entry in extension.entries().iter() { + if let Some(value) = entry.value.as_ref() { + extensions + .set(format!("{extension_name}.{}", entry.key).as_str(), value) + .expect("FFI_ExtensionOptions set should always return Ok"); + } + } + } + + Self { + base_options, + extensions, + } + } +} + +impl TryFrom for ConfigOptions { + type Error = DataFusionError; + fn try_from(ffi_options: FFI_ConfigOptions) -> Result { + let mut options = ConfigOptions::default(); + options.extensions.insert(ffi_options.extensions); + + for kv_tuple in ffi_options.base_options.iter() { + options.set(kv_tuple.0.as_str(), kv_tuple.1.as_str())?; + } + + Ok(options) + } +} diff --git a/datafusion/ffi/src/lib.rs b/datafusion/ffi/src/lib.rs index 2ca9b8f6f495a..a5759e397df11 100644 --- a/datafusion/ffi/src/lib.rs +++ b/datafusion/ffi/src/lib.rs @@ -28,6 +28,7 @@ pub mod arrow_wrappers; pub mod catalog_provider; pub mod catalog_provider_list; +pub mod config; pub mod execution; pub mod execution_plan; pub mod expr; diff --git a/datafusion/ffi/src/session/config.rs b/datafusion/ffi/src/session/config.rs index eb9c4e2c6986a..c9fb9f7fa0c77 100644 --- a/datafusion/ffi/src/session/config.rs +++ b/datafusion/ffi/src/session/config.rs @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::ffi::c_void; +use crate::config::FFI_ConfigOptions; use abi_stable::StableAbi; -use abi_stable::std_types::{RHashMap, RString}; +use datafusion_common::config::ConfigOptions; use datafusion_common::error::{DataFusionError, Result}; use datafusion_execution::config::SessionConfig; @@ -39,7 +39,7 @@ use datafusion_execution::config::SessionConfig; pub struct FFI_SessionConfig { /// Return a hash map from key to value of the config options represented /// by string values. - pub config_options: unsafe extern "C" fn(config: &Self) -> RHashMap, + pub config_options: FFI_ConfigOptions, /// Used to create a clone on the provider of the execution plan. This should /// only need to be called by the receiver of the plan. @@ -67,21 +67,6 @@ impl FFI_SessionConfig { } } -unsafe extern "C" fn config_options_fn_wrapper( - config: &FFI_SessionConfig, -) -> RHashMap { - let config_options = config.inner().options(); - - let mut options = RHashMap::default(); - for config_entry in config_options.entries() { - if let Some(value) = config_entry.value { - options.insert(config_entry.key.into(), value.into()); - } - } - - options -} - unsafe extern "C" fn release_fn_wrapper(config: &mut FFI_SessionConfig) { unsafe { debug_assert!(!config.private_data.is_null()); @@ -100,7 +85,7 @@ unsafe extern "C" fn clone_fn_wrapper(config: &FFI_SessionConfig) -> FFI_Session let private_data = Box::new(SessionConfigPrivateData { config: old_config }); FFI_SessionConfig { - config_options: config_options_fn_wrapper, + config_options: config.config_options.clone(), private_data: Box::into_raw(private_data) as *mut c_void, clone: clone_fn_wrapper, release: release_fn_wrapper, @@ -119,8 +104,10 @@ impl From<&SessionConfig> for FFI_SessionConfig { config: session.clone(), }); + let config_options = FFI_ConfigOptions::from(session.options().as_ref()); + Self { - config_options: config_options_fn_wrapper, + config_options, private_data: Box::into_raw(private_data) as *mut c_void, clone: clone_fn_wrapper, release: release_fn_wrapper, @@ -149,14 +136,9 @@ impl TryFrom<&FFI_SessionConfig> for SessionConfig { return Ok(config.inner().clone()); } - let config_options = unsafe { (config.config_options)(config) }; - - let mut options_map = HashMap::new(); - config_options.iter().for_each(|kv_pair| { - options_map.insert(kv_pair.0.to_string(), kv_pair.1.to_string()); - }); + let config_options = ConfigOptions::try_from(config.config_options.clone())?; - SessionConfig::from_string_hash_map(&options_map) + Ok(SessionConfig::from(config_options)) } } From d8195fcfb3b5fcf2e2fbce3e4941f0dc2ce8ce41 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 16 Feb 2026 21:47:34 -0500 Subject: [PATCH 2/7] Add in FFI integration test for config extensions --- datafusion/ffi/src/config/mod.rs | 52 ++++++++++++++++++++- datafusion/ffi/src/tests/config.rs | 51 +++++++++++++++++++++ datafusion/ffi/src/tests/mod.rs | 6 +++ datafusion/ffi/tests/ffi_config.rs | 73 ++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 datafusion/ffi/src/tests/config.rs create mode 100644 datafusion/ffi/tests/ffi_config.rs diff --git a/datafusion/ffi/src/config/mod.rs b/datafusion/ffi/src/config/mod.rs index ead1eed8b654d..d0c5cbdd05ae9 100644 --- a/datafusion/ffi/src/config/mod.rs +++ b/datafusion/ffi/src/config/mod.rs @@ -19,7 +19,7 @@ pub mod extension_options; use abi_stable::StableAbi; use abi_stable::std_types::{RHashMap, RString}; -use datafusion_common::config::{ConfigOptions, ExtensionOptions}; +use datafusion_common::config::{ConfigOptions, ExtensionOptions, TableOptions}; use datafusion_common::{DataFusionError, Result}; use crate::config::extension_options::FFI_ExtensionOptions; @@ -73,3 +73,53 @@ impl TryFrom for ConfigOptions { Ok(options) } } + +// TODO(tsaucer) add text about how extension options will require user to convert to concrete type. +#[repr(C)] +#[derive(Debug, Clone, StableAbi)] +pub struct FFI_TableOptions { + base_options: RHashMap, + + extensions: FFI_ExtensionOptions, +} + +impl From<&TableOptions> for FFI_TableOptions { + fn from(options: &TableOptions) -> Self { + let base_options: RHashMap = options + .entries() + .into_iter() + .filter_map(|entry| entry.value.map(|value| (entry.key, value))) + .map(|(key, value)| (key.into(), value.into())) + .collect(); + + let mut extensions = FFI_ExtensionOptions::default(); + for (extension_name, extension) in options.extensions.iter() { + for entry in extension.entries().iter() { + if let Some(value) = entry.value.as_ref() { + extensions + .set(format!("{extension_name}.{}", entry.key).as_str(), value) + .expect("FFI_ExtensionOptions set should always return Ok"); + } + } + } + + Self { + base_options, + extensions, + } + } +} + +impl TryFrom for TableOptions { + type Error = DataFusionError; + fn try_from(ffi_options: FFI_TableOptions) -> Result { + let mut options = TableOptions::default(); + options.extensions.insert(ffi_options.extensions); + + for kv_tuple in ffi_options.base_options.iter() { + options.set(kv_tuple.0.as_str(), kv_tuple.1.as_str())?; + } + + Ok(options) + } +} diff --git a/datafusion/ffi/src/tests/config.rs b/datafusion/ffi/src/tests/config.rs new file mode 100644 index 0000000000000..46fc9756203e3 --- /dev/null +++ b/datafusion/ffi/src/tests/config.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_common::config::ConfigExtension; +use datafusion_common::extensions_options; + +use crate::config::extension_options::FFI_ExtensionOptions; + +extensions_options! { + pub struct ExternalConfig { + /// Should "foo" be replaced by "bar"? + pub is_enabled: bool, default = true + + /// Some value to be extracted + pub base_number: usize, default = 1000 + } +} + +impl PartialEq for ExternalConfig { + fn eq(&self, other: &Self) -> bool { + self.base_number == other.base_number && self.is_enabled == other.is_enabled + } +} +impl Eq for ExternalConfig {} + +impl ConfigExtension for ExternalConfig { + const PREFIX: &'static str = "external_config"; +} + +pub(crate) extern "C" fn create_extension_options() -> FFI_ExtensionOptions { + let mut extensions = FFI_ExtensionOptions::default(); + extensions + .add_config(&ExternalConfig::default()) + .expect("add_config should be infallible for ExternalConfig"); + + extensions +} diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs index 9bcd7e0031083..971a3be717010 100644 --- a/datafusion/ffi/src/tests/mod.rs +++ b/datafusion/ffi/src/tests/mod.rs @@ -38,6 +38,7 @@ use super::table_provider::FFI_TableProvider; use super::udf::FFI_ScalarUDF; use crate::catalog_provider::FFI_CatalogProvider; use crate::catalog_provider_list::FFI_CatalogProviderList; +use crate::config::extension_options::FFI_ExtensionOptions; use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use crate::tests::catalog::create_catalog_provider_list; use crate::udaf::FFI_AggregateUDF; @@ -46,6 +47,7 @@ use crate::udwf::FFI_WindowUDF; mod async_provider; pub mod catalog; +pub mod config; mod sync_provider; mod udf_udaf_udwf; pub mod utils; @@ -87,6 +89,9 @@ pub struct ForeignLibraryModule { pub create_rank_udwf: extern "C" fn() -> FFI_WindowUDF, + /// Create extension options, for either ConfigOptions or TableOptions + pub create_extension_options: extern "C" fn() -> FFI_ExtensionOptions, + pub version: extern "C" fn() -> u64, } @@ -141,6 +146,7 @@ pub fn get_foreign_library_module() -> ForeignLibraryModuleRef { create_sum_udaf: create_ffi_sum_func, create_stddev_udaf: create_ffi_stddev_func, create_rank_udwf: create_ffi_rank_func, + create_extension_options: config::create_extension_options, version: super::version, } .leak_into_prefix() diff --git a/datafusion/ffi/tests/ffi_config.rs b/datafusion/ffi/tests/ffi_config.rs new file mode 100644 index 0000000000000..f74ea21343561 --- /dev/null +++ b/datafusion/ffi/tests/ffi_config.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Add an additional module here for convenience to scope this to only +/// when the feature integration-tests is built +#[cfg(feature = "integration-tests")] +mod tests { + use datafusion::error::{DataFusionError, Result}; + use datafusion_common::config::{ConfigExtension, ConfigOptions}; + use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; + use datafusion_ffi::tests::config::ExternalConfig; + use datafusion_ffi::tests::utils::get_module; + + #[test] + fn test_ffi_extension_options() -> Result<()> { + let module = get_module()?; + + let extension_options = + module + .create_extension_options() + .ok_or(DataFusionError::NotImplemented( + "External test library failed to implement create_extension_options" + .to_string(), + ))?(); + + println!("{:?} {}", extension_options, FFI_ExtensionOptions::PREFIX); + + let mut config = ConfigOptions::new(); + config.extensions.insert(extension_options); + + fn extract_config(config: &ConfigOptions) -> ExternalConfig { + // For our use case of this test, we do not need to check + // using .get::() but it is left here as an + // example to users of this crate. + config + .extensions + .get::() + .map(|v| v.to_owned()) + .or_else(|| { + config + .extensions + .get::() + .and_then(|ext| ext.to_extension().ok()) + }) + .expect("Should be able to get ExternalConfig") + } + + // Verify default values are as expected + let returned_config = extract_config(&config); + + assert_eq!(returned_config, ExternalConfig::default()); + + config.set("external_config.is_enabled", "false")?; + let returned_config = extract_config(&config); + assert!(!returned_config.is_enabled); + + Ok(()) + } +} From 238ba001d1b2426bdd493b4a6930efe9b007929f Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 17 Feb 2026 08:54:29 -0500 Subject: [PATCH 3/7] More unit tests --- datafusion/ffi/tests/ffi_config.rs | 100 +++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/datafusion/ffi/tests/ffi_config.rs b/datafusion/ffi/tests/ffi_config.rs index f74ea21343561..e145344483920 100644 --- a/datafusion/ffi/tests/ffi_config.rs +++ b/datafusion/ffi/tests/ffi_config.rs @@ -20,13 +20,15 @@ #[cfg(feature = "integration-tests")] mod tests { use datafusion::error::{DataFusionError, Result}; - use datafusion_common::config::{ConfigExtension, ConfigOptions}; + use datafusion_common::ScalarValue; + use datafusion_common::config::{ConfigOptions, TableOptions}; + use datafusion_execution::config::SessionConfig; use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; use datafusion_ffi::tests::config::ExternalConfig; use datafusion_ffi::tests::utils::get_module; #[test] - fn test_ffi_extension_options() -> Result<()> { + fn test_ffi_config_options_extension() -> Result<()> { let module = get_module()?; let extension_options = @@ -37,8 +39,6 @@ mod tests { .to_string(), ))?(); - println!("{:?} {}", extension_options, FFI_ExtensionOptions::PREFIX); - let mut config = ConfigOptions::new(); config.extensions.insert(extension_options); @@ -70,4 +70,96 @@ mod tests { Ok(()) } + + #[test] + fn test_ffi_table_options_extension() -> Result<()> { + let module = get_module()?; + + let extension_options = + module + .create_extension_options() + .ok_or(DataFusionError::NotImplemented( + "External test library failed to implement create_extension_options" + .to_string(), + ))?(); + + let mut table_options = TableOptions::new(); + table_options.extensions.insert(extension_options); + + fn extract_options(options: &TableOptions) -> ExternalConfig { + // For our use case of this test, we do not need to check + // using .get::() but it is left here as an + // example to users of this crate. + options + .extensions + .get::() + .map(|v| v.to_owned()) + .or_else(|| { + options + .extensions + .get::() + .and_then(|ext| ext.to_extension().ok()) + }) + .expect("Should be able to get ExternalConfig") + } + + // Verify default values are as expected + let returned_options = extract_options(&table_options); + + assert_eq!(returned_options, ExternalConfig::default()); + + table_options.set("external_config.is_enabled", "false")?; + let returned_config = extract_options(&table_options); + assert!(!returned_config.is_enabled); + + Ok(()) + } + + #[test] + fn test_ffi_session_config_options_extension() -> Result<()> { + let module = get_module()?; + + let extension_options = + module + .create_extension_options() + .ok_or(DataFusionError::NotImplemented( + "External test library failed to implement create_extension_options" + .to_string(), + ))?(); + + let mut config = SessionConfig::new().with_option_extension(extension_options); + + fn extract_config(config: &SessionConfig) -> ExternalConfig { + // For our use case of this test, we do not need to check + // using .get::() but it is left here as an + // example to users of this crate. + + config + .options() + .extensions + .get::() + .map(|v| v.to_owned()) + .or_else(|| { + config + .options() + .extensions + .get::() + .and_then(|ext| ext.to_extension().ok()) + }) + .expect("Should be able to get ExternalConfig") + } + + // Verify default values are as expected + let returned_config = extract_config(&config); + assert_eq!(returned_config, ExternalConfig::default()); + + config = config.set( + "external_config.is_enabled", + &ScalarValue::Boolean(Some(false)), + ); + let returned_config = extract_config(&config); + assert!(!returned_config.is_enabled); + + Ok(()) + } } From cc6ea44c4c89ddee82211f39a73916b59c23581d Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 18 Feb 2026 07:45:09 -0500 Subject: [PATCH 4/7] Add trait to simplify pattern for accessing FFI extension options --- datafusion/ffi/src/config/mod.rs | 46 ++++++++++++++-- datafusion/ffi/tests/ffi_config.rs | 85 ++++++++---------------------- 2 files changed, 65 insertions(+), 66 deletions(-) diff --git a/datafusion/ffi/src/config/mod.rs b/datafusion/ffi/src/config/mod.rs index d0c5cbdd05ae9..4eb1a9b4ea350 100644 --- a/datafusion/ffi/src/config/mod.rs +++ b/datafusion/ffi/src/config/mod.rs @@ -19,12 +19,18 @@ pub mod extension_options; use abi_stable::StableAbi; use abi_stable::std_types::{RHashMap, RString}; -use datafusion_common::config::{ConfigOptions, ExtensionOptions, TableOptions}; +use datafusion_common::config::{ + ConfigExtension, ConfigOptions, ExtensionOptions, TableOptions, +}; use datafusion_common::{DataFusionError, Result}; use crate::config::extension_options::FFI_ExtensionOptions; -// TODO(tsaucer) add text about how extension options will require user to convert to concrete type. +/// A stable struct for sharing [`ConfigOptions`] across FFI boundaries. +/// +/// Accessing FFI extension options require a slightly different pattern +/// than local extensions. The trait [`ExtensionOptionsFFIProvider`] can +/// be used to simplify accessing FFI extensions. #[repr(C)] #[derive(Debug, Clone, StableAbi)] pub struct FFI_ConfigOptions { @@ -74,7 +80,41 @@ impl TryFrom for ConfigOptions { } } -// TODO(tsaucer) add text about how extension options will require user to convert to concrete type. +pub trait ExtensionOptionsFFIProvider { + fn ffi_extension(&self) -> Option; +} + +impl ExtensionOptionsFFIProvider for ConfigOptions { + fn ffi_extension(&self) -> Option { + self.extensions + .get::() + .map(|v| v.to_owned()) + .or_else(|| { + self.extensions + .get::() + .and_then(|ffi_ext| ffi_ext.to_extension().ok()) + }) + } +} + +impl ExtensionOptionsFFIProvider for TableOptions { + fn ffi_extension(&self) -> Option { + self.extensions + .get::() + .map(|v| v.to_owned()) + .or_else(|| { + self.extensions + .get::() + .and_then(|ffi_ext| ffi_ext.to_extension().ok()) + }) + } +} + +/// A stable struct for sharing [`TableOptions`] across FFI boundaries. +/// +/// Accessing FFI extension options require a slightly different pattern +/// than local extensions. The trait [`ExtensionOptionsFFIProvider`] can +/// be used to simplify accessing FFI extensions. #[repr(C)] #[derive(Debug, Clone, StableAbi)] pub struct FFI_TableOptions { diff --git a/datafusion/ffi/tests/ffi_config.rs b/datafusion/ffi/tests/ffi_config.rs index e145344483920..fb56e3f525e75 100644 --- a/datafusion/ffi/tests/ffi_config.rs +++ b/datafusion/ffi/tests/ffi_config.rs @@ -23,7 +23,7 @@ mod tests { use datafusion_common::ScalarValue; use datafusion_common::config::{ConfigOptions, TableOptions}; use datafusion_execution::config::SessionConfig; - use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; + use datafusion_ffi::config::ExtensionOptionsFFIProvider; use datafusion_ffi::tests::config::ExternalConfig; use datafusion_ffi::tests::utils::get_module; @@ -42,30 +42,16 @@ mod tests { let mut config = ConfigOptions::new(); config.extensions.insert(extension_options); - fn extract_config(config: &ConfigOptions) -> ExternalConfig { - // For our use case of this test, we do not need to check - // using .get::() but it is left here as an - // example to users of this crate. - config - .extensions - .get::() - .map(|v| v.to_owned()) - .or_else(|| { - config - .extensions - .get::() - .and_then(|ext| ext.to_extension().ok()) - }) - .expect("Should be able to get ExternalConfig") - } - // Verify default values are as expected - let returned_config = extract_config(&config); - + let returned_config: ExternalConfig = config + .ffi_extension() + .expect("should have external config extension"); assert_eq!(returned_config, ExternalConfig::default()); config.set("external_config.is_enabled", "false")?; - let returned_config = extract_config(&config); + let returned_config: ExternalConfig = config + .ffi_extension() + .expect("should have external config extension"); assert!(!returned_config.is_enabled); Ok(()) @@ -86,31 +72,18 @@ mod tests { let mut table_options = TableOptions::new(); table_options.extensions.insert(extension_options); - fn extract_options(options: &TableOptions) -> ExternalConfig { - // For our use case of this test, we do not need to check - // using .get::() but it is left here as an - // example to users of this crate. - options - .extensions - .get::() - .map(|v| v.to_owned()) - .or_else(|| { - options - .extensions - .get::() - .and_then(|ext| ext.to_extension().ok()) - }) - .expect("Should be able to get ExternalConfig") - } - // Verify default values are as expected - let returned_options = extract_options(&table_options); + let returned_options: ExternalConfig = table_options + .ffi_extension() + .expect("should have external config extension"); assert_eq!(returned_options, ExternalConfig::default()); table_options.set("external_config.is_enabled", "false")?; - let returned_config = extract_options(&table_options); - assert!(!returned_config.is_enabled); + let returned_options: ExternalConfig = table_options + .ffi_extension() + .expect("should have external config extension"); + assert!(!returned_options.is_enabled); Ok(()) } @@ -129,35 +102,21 @@ mod tests { let mut config = SessionConfig::new().with_option_extension(extension_options); - fn extract_config(config: &SessionConfig) -> ExternalConfig { - // For our use case of this test, we do not need to check - // using .get::() but it is left here as an - // example to users of this crate. - - config - .options() - .extensions - .get::() - .map(|v| v.to_owned()) - .or_else(|| { - config - .options() - .extensions - .get::() - .and_then(|ext| ext.to_extension().ok()) - }) - .expect("Should be able to get ExternalConfig") - } - // Verify default values are as expected - let returned_config = extract_config(&config); + let returned_config: ExternalConfig = config + .options() + .ffi_extension() + .expect("should have external config extension"); assert_eq!(returned_config, ExternalConfig::default()); config = config.set( "external_config.is_enabled", &ScalarValue::Boolean(Some(false)), ); - let returned_config = extract_config(&config); + let returned_config: ExternalConfig = config + .options() + .ffi_extension() + .expect("should have external config extension"); assert!(!returned_config.is_enabled); Ok(()) From ff6358bcfb9df08daaf157feb48772a04752e26d Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 18 Feb 2026 15:23:45 -0500 Subject: [PATCH 5/7] Clarify usage with documentation and naming --- datafusion/ffi/src/config/mod.rs | 10 +++++++--- datafusion/ffi/tests/ffi_config.rs | 12 ++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/datafusion/ffi/src/config/mod.rs b/datafusion/ffi/src/config/mod.rs index 4eb1a9b4ea350..850a4dc337336 100644 --- a/datafusion/ffi/src/config/mod.rs +++ b/datafusion/ffi/src/config/mod.rs @@ -81,11 +81,15 @@ impl TryFrom for ConfigOptions { } pub trait ExtensionOptionsFFIProvider { - fn ffi_extension(&self) -> Option; + /// Extract a [`ConfigExtension`]. This method should attempt to first extract + /// the extension from the local options when possible. Should that fail, it + /// should attempt to extract the FFI options and then convert them to the + /// desired [`ConfigExtension`]. + fn local_or_ffi_extension(&self) -> Option; } impl ExtensionOptionsFFIProvider for ConfigOptions { - fn ffi_extension(&self) -> Option { + fn local_or_ffi_extension(&self) -> Option { self.extensions .get::() .map(|v| v.to_owned()) @@ -98,7 +102,7 @@ impl ExtensionOptionsFFIProvider for ConfigOptions { } impl ExtensionOptionsFFIProvider for TableOptions { - fn ffi_extension(&self) -> Option { + fn local_or_ffi_extension(&self) -> Option { self.extensions .get::() .map(|v| v.to_owned()) diff --git a/datafusion/ffi/tests/ffi_config.rs b/datafusion/ffi/tests/ffi_config.rs index fb56e3f525e75..ca0a3e31e8de6 100644 --- a/datafusion/ffi/tests/ffi_config.rs +++ b/datafusion/ffi/tests/ffi_config.rs @@ -44,13 +44,13 @@ mod tests { // Verify default values are as expected let returned_config: ExternalConfig = config - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert_eq!(returned_config, ExternalConfig::default()); config.set("external_config.is_enabled", "false")?; let returned_config: ExternalConfig = config - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert!(!returned_config.is_enabled); @@ -74,14 +74,14 @@ mod tests { // Verify default values are as expected let returned_options: ExternalConfig = table_options - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert_eq!(returned_options, ExternalConfig::default()); table_options.set("external_config.is_enabled", "false")?; let returned_options: ExternalConfig = table_options - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert!(!returned_options.is_enabled); @@ -105,7 +105,7 @@ mod tests { // Verify default values are as expected let returned_config: ExternalConfig = config .options() - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert_eq!(returned_config, ExternalConfig::default()); @@ -115,7 +115,7 @@ mod tests { ); let returned_config: ExternalConfig = config .options() - .ffi_extension() + .local_or_ffi_extension() .expect("should have external config extension"); assert!(!returned_config.is_enabled); From cc7bf6fa1ddca060e31ee01b128c8c45d93d020a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 19 Feb 2026 10:42:00 -0500 Subject: [PATCH 6/7] Apply review suggestions --- datafusion/ffi/src/config/extension_options.rs | 11 +++++++---- datafusion/ffi/src/session/config.rs | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/datafusion/ffi/src/config/extension_options.rs b/datafusion/ffi/src/config/extension_options.rs index 09137dac8b27e..48fd4e710921a 100644 --- a/datafusion/ffi/src/config/extension_options.rs +++ b/datafusion/ffi/src/config/extension_options.rs @@ -109,10 +109,13 @@ unsafe extern "C" fn entries_fn_wrapper( } unsafe extern "C" fn release_fn_wrapper(options: &mut FFI_ExtensionOptions) { - let private_data = unsafe { - Box::from_raw(options.private_data as *mut ExtensionOptionsPrivateData) - }; - drop(private_data); + unsafe { + debug_assert!(!options.private_data.is_null()); + let private_data = + Box::from_raw(options.private_data as *mut ExtensionOptionsPrivateData); + drop(private_data); + options.private_data = std::ptr::null_mut(); + } } impl Default for FFI_ExtensionOptions { diff --git a/datafusion/ffi/src/session/config.rs b/datafusion/ffi/src/session/config.rs index c9fb9f7fa0c77..63f0f20ecc7d5 100644 --- a/datafusion/ffi/src/session/config.rs +++ b/datafusion/ffi/src/session/config.rs @@ -37,8 +37,7 @@ use datafusion_execution::config::SessionConfig; #[repr(C)] #[derive(Debug, StableAbi)] pub struct FFI_SessionConfig { - /// Return a hash map from key to value of the config options represented - /// by string values. + /// FFI stable configuration options. pub config_options: FFI_ConfigOptions, /// Used to create a clone on the provider of the execution plan. This should From d1aff2189d950022694a2f8795a14f5aca790ed9 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 24 Feb 2026 07:50:13 -0500 Subject: [PATCH 7/7] Add docstring --- datafusion/common/src/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 3b9507b1acf1a..d71af206c78d5 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1352,6 +1352,8 @@ impl ConfigField for ConfigOptions { } } +/// This namespace is reserved for interacting with Foreign Function Interface +/// (FFI) based configuration extensions. pub const DATAFUSION_FFI_CONFIG_NAMESPACE: &str = "datafusion_ffi"; impl ConfigOptions {