diff --git a/.typos.toml b/.typos.toml index e9fa0028f5..cde21f7091 100644 --- a/.typos.toml +++ b/.typos.toml @@ -21,6 +21,7 @@ extend-ignore-identifiers-re = ["^bimap$"] [default.extend-words] AGS = "AGS" ags = "ags" +ser = "ser" [files] extend-exclude = ["**/testdata", "CHANGELOG.md", "**/public-api.txt"] diff --git a/Cargo.lock b/Cargo.lock index 0f264cc1f5..d48f9beb79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4007,6 +4007,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iceberg-property-macro" +version = "0.10.0" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + [[package]] name = "iceberg-sqllogictest" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index a789ef1967..bcdd080347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/catalog/*", "crates/examples", "crates/iceberg", + "crates/property-macro", "crates/integration_tests", "crates/integrations/*", "crates/sqllogictest", diff --git a/crates/property-macro/Cargo.toml b/crates/property-macro/Cargo.toml new file mode 100644 index 0000000000..075cf73f63 --- /dev/null +++ b/crates/property-macro/Cargo.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-property-macro" +publish = true +readme = "README.md" +rust-version = { workspace = true } +version = { workspace = true } + +license = { workspace = true } +repository = { workspace = true } + +categories = ["database"] +description = "Property derive macro for Apache Iceberg Rust" +keywords = ["iceberg"] + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full"] } + +[dev-dependencies] +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/crates/property-macro/README.md b/crates/property-macro/README.md new file mode 100644 index 0000000000..7187e5ce26 --- /dev/null +++ b/crates/property-macro/README.md @@ -0,0 +1,240 @@ + + +# Iceberg property derive macro + +`Properties` parses a typed struct from a flat `HashMap` and +can generate opt-in read-only getters. It deliberately does not generate +property-map serialization or implement `Default`, `Serialize`, `Deserialize`, +or any other trait. + +## Generated API + +For every annotated struct, `#[derive(Properties)]` generates this inherent +constructor: + +```text +impl MyProperties { + pub fn from_properties( + properties: &HashMap, + ) -> Result; +} +``` + +`from_properties` borrows the source map, parses every modeled property, and +uses its annotated default when a property is absent. Unknown keys are ignored. +An invalid value returns an error containing its primary property key. + +Adding `pub(getter)` to a field generates an immutable accessor with the field +name. Structurally known `Copy` types return `T`; other types return `&T`. +Documentation attributes on the field are copied to the generated getter. The +macro generates no setters, backing fields, or conversion back to a property +map. + +## Complete example + +This example covers exact keys and defaults, optional values, case-insensitive +booleans, prefixed maps, nested groups, custom single-value parsing, custom +multi-key parsing, lists of additional keys, read-only getters, ignored unknown +keys, and contextual errors. + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FANOUT: &str = "write.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const LOCATION: &str = "write.data.path"; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_location(value: &str) -> Result { + let location = value.trim().trim_end_matches('/'); + if location.is_empty() { + Err("location must not be empty") + } else { + Ok(location.to_string()) + } +} + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> Result<(u64, u64, u64), String> { + if additional_keys.len() != 2 { + return Err("dimensions require height and depth keys".to_string()); + } + let parse = |key: &str, default| { + properties + .get(key) + .map(|value| value.parse::().map_err(|error| error.to_string())) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, pub(getter))] + retries: usize, +} + +#[derive(Debug, Properties)] +struct TableLikeProperties { + /// Nested groups parse from the same flat property map. + #[property(nested, pub(getter))] + commit: CommitProperties, + + /// Option distinguishes an absent property from a present value. + #[property(key = OWNER, default = None, pub(getter))] + owner: Option, + + /// Boolean values are parsed case-insensitively. + #[property(key = FANOUT, default = true, pub(getter))] + fanout_enabled: bool, + + /// A prefix captures suffix/value pairs into a typed map. + #[property(prefix = COLUMN_FPP_PREFIX, default = HashMap::new(), pub(getter))] + column_fpp: HashMap, + + /// A single-key parser can validate and normalize a property value. + #[property( + key = LOCATION, + default = "warehouse", + parse_with = parse_location, + pub(getter) + )] + location: String, + + /// A full-map parser can model one field with multiple property keys. + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + pub(getter) + )] + dimensions: (u64, u64, u64), +} + +fn main() -> Result<(), String> { + let defaults = TableLikeProperties::from_properties(&HashMap::new())?; + assert_eq!(defaults.commit().retries(), 4); + assert_eq!(defaults.owner(), &None); + assert!(defaults.fanout_enabled()); + assert!(defaults.column_fpp().is_empty()); + assert_eq!(defaults.location(), "warehouse"); + assert_eq!(defaults.dimensions(), (640, 480, 320)); + + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FANOUT.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (LOCATION.to_string(), " s3://bucket/table/ ".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unmodeled".to_string(), "ignored".to_string()), + ]); + + let properties = TableLikeProperties::from_properties(&raw)?; + assert_eq!(properties.commit().retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.location(), "s3://bucket/table"); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); + + let error = TableLikeProperties::from_properties(&HashMap::from([( + LOCATION.to_string(), + "/".to_string(), + )])) + .unwrap_err(); + assert!(error.contains(LOCATION)); + + Ok(()) +} +``` + +## Using ordinary derives together + +`Properties` does not implicitly derive other traits, so `Default`, +`Serialize`, and `Deserialize` can be selected independently and behave like +ordinary Rust derives: + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct ReadProperties { + #[property( + key = "commit.retry.num-retries", + default = 4, + pub(getter) + )] + retries: u64, +} + +fn main() -> Result<(), Box> { + // The property annotation supplies the default used by from_properties. + let properties = ReadProperties::from_properties(&HashMap::new())?; + assert_eq!(properties.retries(), 4); + + // The ordinary Default derive uses the field's Rust default instead. + let defaults = ReadProperties::default(); + assert_eq!(defaults.retries(), 0); + + // Ordinary Serde derives use Rust field names, not property keys. + let json = serde_json::to_string(&properties)?; + assert_eq!(json, r#"{"retries":4}"#); + let decoded: ReadProperties = serde_json::from_str(r#"{"retries":7}"#)?; + assert_eq!(decoded.retries(), 7); + Ok(()) +} +``` + +All field settings must be grouped under `#[property(...)]`. This keeps `key`, +`default`, `prefix`, `nested`, parser hooks, additional keys, and getter +generation in one attribute and avoids collisions with ordinary Rust derives. + +The `prefix` setting requires `HashMap`. `nested` embeds another +`Properties` struct while reading the same flat map. `parse_with` customizes +parsing for one exact-key field. `parse_properties_with` receives the complete +property map, and `additional_keys` supplies its list of secondary keys. + +Boolean values are parsed case-insensitively. Other values require `FromStr` +unless a custom parser is supplied. String-literal and path defaults are +converted into their field type with `Into`. diff --git a/crates/property-macro/public-api.txt b/crates/property-macro/public-api.txt new file mode 100644 index 0000000000..8edb5d0952 --- /dev/null +++ b/crates/property-macro/public-api.txt @@ -0,0 +1,2 @@ +pub mod iceberg_property_macro +pub proc macro iceberg_property_macro::#[derive(Properties)] diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs new file mode 100644 index 0000000000..e73b48c0f7 --- /dev/null +++ b/crates/property-macro/src/lib.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![doc = include_str!("../README.md")] + +use proc_macro::TokenStream; +use syn::{DeriveInput, parse_macro_input}; + +mod properties; + +/// Derives property-map parsing and opt-in read-only accessors for a struct. +#[proc_macro_derive(Properties, attributes(property))] +pub fn derive_properties(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + match properties::expand_properties(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} diff --git a/crates/property-macro/src/properties.rs b/crates/property-macro/src/properties.rs new file mode 100644 index 0000000000..5416aba19f --- /dev/null +++ b/crates/property-macro/src/properties.rs @@ -0,0 +1,632 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{ + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, GenericArgument, + Ident, Lit, Path, PathArguments, Token, Type, parenthesized, +}; + +struct PropertyField { + ident: Ident, + ty: Type, + key: Option, + additional_keys: Option>, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + option_inner_type: Option, + map_value_type: Option, + public_getter: bool, + doc_attributes: Vec, +} + +struct PublicGetter; + +enum PropertyOption { + Key(Expr), + AdditionalKeys(Vec), + Prefix(Expr), + Nested, + Default(Expr), + ParseWith(Path), + ParsePropertiesWith(Path), + Getter(PublicGetter), +} + +#[derive(Default)] +struct PropertyOptions { + key: Option, + additional_keys: Option>, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + public_getter: bool, +} + +impl Parse for PublicGetter { + fn parse(input: ParseStream<'_>) -> syn::Result { + input.parse::()?; + let content; + parenthesized!(content in input); + let accessor = content.parse::()?; + if !content.is_empty() { + return Err(content.error("expected getter")); + } + + if accessor == "getter" { + Ok(Self) + } else { + Err(Error::new_spanned(accessor, "expected getter")) + } + } +} + +impl Parse for PropertyOption { + fn parse(input: ParseStream<'_>) -> syn::Result { + if input.peek(Token![pub]) { + return input.parse().map(Self::Getter); + } + + let name = input.parse::()?; + let option_name = name.to_string(); + if option_name == "nested" { + return Ok(Self::Nested); + } + + input.parse::()?; + let expression = input.parse::()?; + match option_name.as_str() { + "key" => Ok(Self::Key(expression)), + "additional_keys" => { + expression_list(expression, "additional_keys").map(Self::AdditionalKeys) + } + "prefix" => Ok(Self::Prefix(expression)), + "default" => Ok(Self::Default(expression)), + "parse_with" => expression_path(expression, "parse_with").map(Self::ParseWith), + "parse_properties_with" => { + expression_path(expression, "parse_properties_with").map(Self::ParsePropertiesWith) + } + _ => Err(Error::new_spanned(name, "unknown property option")), + } + } +} + +pub(crate) fn expand_properties(input: DeriveInput) -> syn::Result { + let struct_name = input.ident; + let generics = input.generics; + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs with named fields", + )); + } + }, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs", + )); + } + }; + + let fields = fields + .iter() + .map(|field| parse_property_field(field, property_options(field)?)) + .collect::>>()?; + let parses = fields.iter().map(parse_field); + let accessors = fields.iter().map(field_getter); + let (impl_generics, type_generics, where_clause) = generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics #struct_name #type_generics #where_clause { + #(#accessors)* + + pub fn from_properties( + properties: &::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) -> ::std::result::Result { + Ok(Self { + #(#parses,)* + }) + } + } + }) +} + +fn parse_property_field( + field: &Field, + property_options: PropertyOptions, +) -> syn::Result { + let ident = field + .ident + .clone() + .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; + let PropertyOptions { + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + public_getter, + } = property_options; + + if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of key, prefix, or nested in #[property(...)]", + )); + } + + if nested && default.is_some() { + return Err(Error::new_spanned( + field, + "nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)]", + )); + } + if !nested && default.is_none() { + return Err(Error::new_spanned( + field, + "Properties leaf fields must declare default in #[property(...)]", + )); + } + + let map_value_type = hash_map_value_type(&field.ty); + if prefix.is_some() && map_value_type.is_none() { + return Err(Error::new_spanned( + &field.ty, + "property prefix fields must have type HashMap", + )); + } + + if additional_keys.is_some() && parse_properties_with.is_none() { + return Err(Error::new_spanned( + field, + "additional_keys requires parse_properties_with in #[property(...)]", + )); + } + if (prefix.is_some() || nested) + && (additional_keys.is_some() || parse_with.is_some() || parse_properties_with.is_some()) + { + return Err(Error::new_spanned( + field, + "prefix and nested fields do not support custom parse functions", + )); + } + if parse_with.is_some() && parse_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both parse_with and parse_properties_with", + )); + } + Ok(PropertyField { + ident, + ty: field.ty.clone(), + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + option_inner_type: option_inner_type(&field.ty), + map_value_type, + public_getter, + doc_attributes: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), + }) +} + +fn property_options(field: &Field) -> syn::Result { + let Some(attribute) = find_attribute(&field.attrs, "property")? else { + return Err(Error::new_spanned( + field, + "Properties fields must declare #[property(...)]", + )); + }; + + let parsed = + attribute.parse_args_with(Punctuated::::parse_terminated)?; + if parsed.is_empty() { + return Err(Error::new_spanned( + attribute, + "property must declare at least one option", + )); + } + + let mut options = PropertyOptions::default(); + for option in parsed { + match option { + PropertyOption::Key(value) => { + set_property_option(&mut options.key, value, attribute, "key")? + } + PropertyOption::AdditionalKeys(value) => set_property_option( + &mut options.additional_keys, + value, + attribute, + "additional_keys", + )?, + PropertyOption::Prefix(value) => { + set_property_option(&mut options.prefix, value, attribute, "prefix")? + } + PropertyOption::Nested => { + if options.nested { + return Err(Error::new_spanned( + attribute, + "duplicate nested property option", + )); + } + options.nested = true; + } + PropertyOption::Default(value) => { + set_property_option(&mut options.default, value, attribute, "default")? + } + PropertyOption::ParseWith(value) => { + set_property_option(&mut options.parse_with, value, attribute, "parse_with")? + } + PropertyOption::ParsePropertiesWith(value) => set_property_option( + &mut options.parse_properties_with, + value, + attribute, + "parse_properties_with", + )?, + PropertyOption::Getter(_) => { + if options.public_getter { + return Err(Error::new_spanned(attribute, "duplicate property accessor")); + } + options.public_getter = true; + } + } + } + + Ok(options) +} + +fn set_property_option( + target: &mut Option, + value: T, + attribute: &Attribute, + name: &str, +) -> syn::Result<()> { + if target.is_some() { + return Err(Error::new_spanned( + attribute, + format!("duplicate {name} property option"), + )); + } + *target = Some(value); + Ok(()) +} + +fn field_getter(field: &PropertyField) -> TokenStream2 { + if !field.public_getter { + return TokenStream2::new(); + } + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.doc_attributes; + if is_copy_type(ty) { + quote! { + #(#docs)* + pub fn #ident(&self) -> #ty { + self.#ident + } + } + } else { + quote! { + #(#docs)* + pub fn #ident(&self) -> &#ty { + &self.#ident + } + } + } +} + +fn expression_path(expression: Expr, name: &str) -> syn::Result { + match expression { + Expr::Path(ExprPath { path, .. }) => Ok(path), + _ => Err(Error::new_spanned( + expression, + format!("{name} must be a path"), + )), + } +} + +fn expression_list(expression: Expr, name: &str) -> syn::Result> { + let Expr::Array(array) = expression else { + return Err(Error::new_spanned( + expression, + format!("{name} must be an array of keys"), + )); + }; + if array.elems.is_empty() { + return Err(Error::new_spanned( + array, + format!("{name} must contain at least one key"), + )); + } + Ok(array.elems.into_iter().collect()) +} + +fn find_attribute<'a>( + attributes: &'a [Attribute], + name: &str, +) -> syn::Result> { + let mut matching = attributes + .iter() + .filter(|attribute| attribute.path().is_ident(name)); + let first = matching.next(); + if let Some(duplicate) = matching.next() { + return Err(Error::new_spanned( + duplicate, + format!("duplicate #[{name}] attribute"), + )); + } + Ok(first) +} + +fn parse_field(field: &PropertyField) -> TokenStream2 { + let ident = &field.ident; + if field.nested { + let ty = &field.ty; + return quote!(#ident: <#ty>::from_properties(properties)?); + } + + let ty = &field.ty; + let default = typed_default(field); + + if let Some(parse_properties_with) = &field.parse_properties_with { + let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match &field.additional_keys { + Some(additional_keys) => { + quote!(#parse_properties_with(properties, #key, &[#(#additional_keys),*], #default)) + } + None => quote!(#parse_properties_with(properties, #key, #default)), + }; + return quote! { + #ident: #parse.map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }; + } + + if let Some(prefix) = &field.prefix { + let value_type = field + .map_value_type + .as_ref() + .expect("prefix fields are validated as maps"); + let parse = if is_bool(value_type) { + quote!(value.to_ascii_lowercase().parse::<#value_type>()) + } else { + quote!(value.parse::<#value_type>()) + }; + return quote! { + #ident: { + let parsed = properties + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(#prefix).map(|suffix| { + #parse + .map(|parsed| (suffix.to_string(), parsed)) + .map_err(|error| format!("Invalid value for {key}: {error}")) + }) + }) + .collect::<::std::result::Result< + ::std::collections::HashMap<_, _>, + ::std::string::String, + >>()?; + if parsed.is_empty() { + #default + } else { + parsed + } + } + }; + } + + let key = field.key.as_ref().expect("exact-key fields have a key"); + let parse = match (&field.parse_with, &field.option_inner_type) { + (Some(parse_with), _) => quote! { + #parse_with(value).map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + (None, Some(inner_type)) if is_bool(inner_type) => quote! { + Some(value.to_ascii_lowercase().parse::<#inner_type>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })?) + }, + (None, Some(inner_type)) => quote! { + Some(value.parse::<#inner_type>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })?) + }, + (None, None) if is_bool(ty) => quote! { + value.to_ascii_lowercase().parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + (None, None) => quote! { + value.parse::<#ty>().map_err(|error| { + format!("Invalid value for {}: {error}", #key) + })? + }, + }; + + quote! { + #ident: match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + } +} + +fn typed_default(field: &PropertyField) -> TokenStream2 { + let ty = &field.ty; + let default = default_value( + field.default.as_ref().expect("leaf fields have defaults"), + ty, + ); + quote!({ + let value: #ty = #default; + value + }) +} + +fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { + if matches!( + default, + Expr::Lit(ExprLit { + lit: Lit::Str(_), + .. + }) | Expr::Path(_) + ) { + quote!(::std::convert::Into::<#ty>::into(#default)) + } else { + quote!(#default) + } +} + +fn option_inner_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "Option" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(GenericArgument::Type(inner_type)) = arguments.args.first() else { + return None; + }; + + Some(inner_type.clone()) +} + +fn hash_map_value_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "HashMap" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let mut arguments = arguments.args.iter(); + let Some(GenericArgument::Type(key_type)) = arguments.next() else { + return None; + }; + let Some(GenericArgument::Type(value_type)) = arguments.next() else { + return None; + }; + if !is_named_type(key_type, "String") { + return None; + } + + Some(value_type.clone()) +} + +fn is_bool(ty: &Type) -> bool { + is_named_type(ty, "bool") +} + +fn is_copy_type(ty: &Type) -> bool { + match ty { + Type::Array(array) => is_copy_type(&array.elem), + Type::BareFn(_) | Type::Never(_) | Type::Ptr(_) => true, + Type::Group(group) => is_copy_type(&group.elem), + Type::Paren(paren) => is_copy_type(&paren.elem), + Type::Reference(reference) => reference.mutability.is_none(), + Type::Tuple(tuple) => tuple.elems.iter().all(is_copy_type), + Type::Path(type_path) if type_path.qself.is_none() => { + let Some(segment) = type_path.path.segments.last() else { + return false; + }; + if matches!( + segment.ident.to_string().as_str(), + "bool" + | "char" + | "f32" + | "f64" + | "i8" + | "i16" + | "i32" + | "i64" + | "i128" + | "isize" + | "u8" + | "u16" + | "u32" + | "u64" + | "u128" + | "usize" + ) { + return true; + } + if segment.ident != "Option" && segment.ident != "Result" { + return false; + } + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return false; + }; + arguments.args.iter().all(|argument| match argument { + GenericArgument::Lifetime(_) => true, + GenericArgument::Type(ty) => is_copy_type(ty), + _ => false, + }) + } + _ => false, + } +} + +fn is_named_type(ty: &Type, name: &str) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + + type_path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == name) +} diff --git a/crates/property-macro/tests/properties.rs b/crates/property-macro/tests/properties.rs new file mode 100644 index 0000000000..12bb4f3322 --- /dev/null +++ b/crates/property-macro/tests/properties.rs @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> Result<(u64, u64, u64), String> { + if additional_keys.len() != 2 { + return Err("dimensions require height and depth keys".to_string()); + } + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| value.parse::().map_err(|error| error.to_string())) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct TestProperties { + #[property(key = RETRIES, default = 4, pub(getter))] + retries: u64, + + #[property(key = OWNER, default = None, pub(getter))] + owner: Option, + + #[property(key = FORMAT, default = "parquet", pub(getter))] + format: String, + + #[property(key = FANOUT_ENABLED, default = true, pub(getter))] + fanout_enabled: bool, + + #[property( + prefix = COLUMN_FPP_PREFIX, + default = HashMap::new(), + pub(getter) + )] + column_fpp: HashMap, + + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + pub(getter) + )] + dimensions: (u64, u64, u64), +} + +#[test] +fn reads_defaults_through_generated_getters() { + let properties = TestProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.retries(), 4); + assert_eq!(properties.owner(), &None); + assert_eq!(properties.format(), "parquet"); + assert!(properties.fanout_enabled()); + assert!(properties.column_fpp().is_empty()); + assert_eq!(properties.dimensions(), (640, 480, 320)); +} + +#[test] +fn reads_overrides_and_ignores_unknown_properties() { + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FORMAT.to_string(), "orc".to_string()), + (FANOUT_ENABLED.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unknown".to_string(), "ignored".to_string()), + ]); + let properties = TestProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert_eq!(properties.format(), "orc"); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); +} + +#[test] +fn reports_the_property_with_an_invalid_value() { + let numeric_error = TestProperties::from_properties(&HashMap::from([( + RETRIES.to_string(), + "many".to_string(), + )])) + .unwrap_err(); + assert!(numeric_error.contains(RETRIES)); + + let boolean_error = TestProperties::from_properties(&HashMap::from([( + FANOUT_ENABLED.to_string(), + "sometimes".to_string(), + )])) + .unwrap_err(); + assert!(boolean_error.contains(FANOUT_ENABLED)); + + let prefixed_key = format!("{COLUMN_FPP_PREFIX}id"); + let prefix_error = TestProperties::from_properties(&HashMap::from([( + prefixed_key.clone(), + "low".to_string(), + )])) + .unwrap_err(); + assert!(prefix_error.contains(&prefixed_key)); +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, pub(getter))] + retries: u64, +} + +#[derive(Debug, Properties)] +struct NestedProperties { + #[property(nested, pub(getter))] + commit: CommitProperties, +} + +#[test] +fn nested_properties_read_the_same_flat_map() { + let raw = HashMap::from([(RETRIES.to_string(), "9".to_string())]); + let properties = NestedProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.commit().retries(), 9); +} + +fn parse_non_empty(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + Err("value must not be empty") + } else { + Ok(value.to_string()) + } +} + +#[derive(Debug, Properties)] +struct ValidatedProperties { + #[property( + key = "location", + default = "default", + parse_with = parse_non_empty, + pub(getter) + )] + location: String, +} + +#[test] +fn custom_single_value_parser_can_validate_and_normalize() { + let parsed = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " path ".to_string(), + )])) + .unwrap(); + assert_eq!(parsed.location(), "path"); + + let error = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " ".to_string(), + )])) + .unwrap_err(); + assert_eq!(error, "Invalid value for location: value must not be empty"); +} + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct DerivedTraitProperties { + #[property(key = RETRIES, default = 4, pub(getter))] + retries: u64, +} + +#[test] +fn coexists_with_default_serialize_and_deserialize_derives() { + let defaults = DerivedTraitProperties::default(); + assert_eq!(defaults.retries(), 0); + + let properties = DerivedTraitProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(properties.retries(), 4); + assert_eq!( + serde_json::to_string(&properties).unwrap(), + r#"{"retries":4}"# + ); + + let decoded: DerivedTraitProperties = serde_json::from_str(r#"{"retries":7}"#).unwrap(); + assert_eq!(decoded.retries(), 7); +}