feat(storage): support Jindo for OSS scan planning

bindings/python/src/context.rs

@@ -83,7 +83,47 @@ fn pyarrow_compatible_batch(batch: &RecordBatch) -> arrow::error::Result<RecordB
     RecordBatch::try_new_with_options(schema, columns, &options)
 }
 
-fn build_paimon_catalog(catalog_options: HashMap<String, String>) -> PyResult<Arc<dyn Catalog>> {
+const OSS_IMPL: &str = "fs.oss.impl";
+const JINDO_LIBRARY_PATH: &str = "fs.jindo.library.path";
+
+fn discover_pyjindo_library() -> Option<PathBuf> {
+    Python::attach(|py| {
+        let spec = py
+            .import("importlib.util")
+            .ok()?
+            .call_method1("find_spec", ("pyjindo",))
+            .ok()?;
+        if spec.is_none() {
+            return None;
+        }
+        let origin = PathBuf::from(spec.getattr("origin").ok()?.extract::<String>().ok()?);
+        pyjindo_library_in(origin.parent()?)
+    })
+}
+
+fn pyjindo_library_in(directory: &std::path::Path) -> Option<PathBuf> {
+    let names = if cfg!(target_os = "macos") {
+        ["libjindosdk_c.dylib", "libjindosdk_python.dylib"]
+    } else {
+        ["libjindosdk_c.so", "libjindosdk_python.so"]
+    };
+    names
+        .iter()
+        .map(|name| directory.join(name))
+        .find(|path| path.is_file())
+}
+
+fn build_paimon_catalog(
+    mut catalog_options: HashMap<String, String>,
+) -> PyResult<Arc<dyn Catalog>> {
+    let use_jindo = catalog_options
+        .get(OSS_IMPL)
+        .is_some_and(|value| value.eq_ignore_ascii_case("jindo"));
+    if use_jindo && !catalog_options.contains_key(JINDO_LIBRARY_PATH) {
+        if let Some(path) = discover_pyjindo_library() {
+            catalog_options.insert(JINDO_LIBRARY_PATH.to_string(), path.display().to_string());
+        }
+    }
     let rt = runtime();
     rt.block_on(async {
         let options = Options::from_map(catalog_options);
@@ -92,6 +132,33 @@ fn build_paimon_catalog(catalog_options: HashMap<String, String>) -> PyResult<Ar
     })
 }
 
+#[cfg(test)]
+mod tests {
+    use std::fs;
+
+    use super::*;
+
+    #[test]
+    fn test_find_pyjindo_library() {
+        let directory = std::env::temp_dir().join(format!(
+            "paimon-rust-pyjindo-{}-{}",
+            std::process::id(),
+            std::thread::current().name().unwrap_or("test")
+        ));
+        fs::create_dir_all(&directory).unwrap();
+        let library = directory.join(if cfg!(target_os = "macos") {
+            "libjindosdk_python.dylib"
+        } else {
+            "libjindosdk_python.so"
+        });
+        fs::write(&library, []).unwrap();
+
+        assert_eq!(pyjindo_library_in(&directory), Some(library));
+
+        fs::remove_dir_all(directory).unwrap();
+    }
+}
+
 fn ffi_logical_codec_from_pycapsule(obj: Bound<'_, PyAny>) -> PyResult<FFI_LogicalExtensionCodec> {
     let attr_name = "__datafusion_logical_extension_codec__";
     let capsule = if obj.hasattr(attr_name)? {

crates/paimon/Cargo.toml

@@ -41,6 +41,7 @@ storage-all = [
     "storage-obs",
     "storage-gcs",
     "storage-hdfs",
+    "storage-jindo",
 ]
 fulltext = ["dep:paimon-ftindex-core", "dep:tempfile"]
 vortex = ["dep:vortex"]
@@ -54,6 +55,7 @@ storage-azdls = ["dep:opendal-http-transport-reqwest", "dep:opendal-service-azdl
 storage-obs = ["dep:opendal-http-transport-reqwest", "dep:opendal-service-obs"]
 storage-gcs = ["dep:opendal-http-transport-reqwest", "dep:opendal-service-gcs"]
 storage-hdfs = ["dep:opendal-service-hdfs-native"]
+storage-jindo = ["storage-oss"]
 
 [dependencies]
 url = "2.5.2"

crates/paimon/src/io/mod.rs

@@ -62,6 +62,11 @@ pub(crate) mod storage_oss;
 #[cfg(feature = "storage-oss")]
 use storage_oss::*;
 
+#[cfg(feature = "storage-jindo")]
+mod storage_jindo;
+#[cfg(feature = "storage-jindo")]
+use storage_jindo::*;
+
 #[cfg(feature = "storage-s3")]
 mod storage_s3;
 #[cfg(feature = "storage-s3")]

crates/paimon/src/io/storage.rs

@@ -39,6 +39,8 @@ use std::sync::MutexGuard;
 
 #[cfg(feature = "storage-azdls")]
 use super::AzdlsStorageConfig;
+#[cfg(feature = "storage-jindo")]
+use super::JindoStorageConfig;
 use opendal::Operator;
 #[cfg(feature = "storage-cos")]
 use opendal_service_cos::CosConfig;
@@ -80,6 +82,11 @@ pub enum Storage {
         config: Box<OssConfig>,
         operators: Mutex<HashMap<String, Operator>>,
     },
+    #[cfg(feature = "storage-jindo")]
+    Jindo {
+        config: Box<JindoStorageConfig>,
+        operators: Mutex<HashMap<String, Operator>>,
+    },
     #[cfg(feature = "storage-s3")]
     S3 {
         config: Box<S3Config>,
@@ -130,6 +137,23 @@ impl Storage {
             }),
             #[cfg(feature = "storage-oss")]
             "oss" => {
+                #[cfg(feature = "storage-jindo")]
+                if super::use_jindo(&props)? {
+                    let config = super::jindo_config_parse(props)?;
+                    return Ok(Self::Jindo {
+                        config: Box::new(config),
+                        operators: Mutex::new(HashMap::new()),
+                    });
+                }
+                #[cfg(not(feature = "storage-jindo"))]
+                if props
+                    .get("fs.oss.impl")
+                    .is_some_and(|value| value.eq_ignore_ascii_case("jindo"))
+                {
+                    return Err(error::Error::IoUnsupported {
+                        message: "Jindo requires the storage-jindo feature".to_string(),
+                    });
+                }
                 let config = super::oss_config_parse(props)?;
                 Ok(Self::Oss {
                     config: Box::new(config),
@@ -221,6 +245,15 @@ impl Storage {
                 let op = Self::cached_oss_operator(config, operators, path, &bucket)?;
                 Ok((op, Cow::Borrowed(relative_path)))
             }
+            #[cfg(feature = "storage-jindo")]
+            Storage::Jindo { config, operators } => {
+                let (bucket, relative_path) =
+                    Self::bucket_and_relative_path(path, "Jindo OSS", &["oss"])?;
+                let op = Self::cached_operator(operators, "Jindo OSS", &bucket, || {
+                    super::jindo_config_build(config, &bucket)
+                })?;
+                Ok((op, Cow::Borrowed(relative_path)))
+            }
             #[cfg(feature = "storage-s3")]
             Storage::S3 { config, operators } => {
                 let (bucket, relative_path) =
@@ -333,6 +366,7 @@ impl Storage {
     #[cfg(any(
         feature = "storage-cos",
         feature = "storage-gcs",
+        feature = "storage-jindo",
         feature = "storage-obs",
         feature = "storage-oss",
         feature = "storage-s3"
@@ -372,6 +406,7 @@ impl Storage {
         feature = "storage-azdls",
         feature = "storage-cos",
         feature = "storage-gcs",
+        feature = "storage-jindo",
         feature = "storage-oss",
         feature = "storage-obs",
         feature = "storage-s3"
@@ -474,6 +509,14 @@ mod scheme_tests {
         }
     }
 
+    #[cfg(feature = "storage-jindo")]
+    #[test]
+    fn jindo_oss_implementation_is_selected() {
+        let storage =
+            Storage::build(FileIOBuilder::new("oss").with_prop("fs.oss.impl", "jindo")).unwrap();
+        assert!(matches!(storage, Storage::Jindo { .. }));
+    }
+
     #[cfg(feature = "storage-s3")]
     #[test]
     fn s3_scheme_aliases_are_compatible() {