Skip to content

Add UUID support for the Avro schema #1706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 24, 2025
+27 −38
Merged

Add UUID support for the Avro schema #1706

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 27 additions & 38 deletions crates/iceberg/src/avro/schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,7 @@ const ELEMENT_ID: &str = "element-id";
const FIELD_ID_PROP: &str = "field-id";
const KEY_ID: &str = "key-id";
const VALUE_ID: &str = "value-id";
const UUID_BYTES: usize = 16;
const UUID_LOGICAL_TYPE: &str = "uuid";
const MAP_LOGICAL_TYPE: &str = "map";
// # TODO: https://github.com/apache/iceberg-rust/issues/86
// This const may better to maintain in avro-rs.
const LOGICAL_TYPE: &str = "logicalType";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i tried but couldnt find this constant exposed by avro-rs


Expand Down Expand Up @@ -237,8 +234,8 @@ impl SchemaVisitor for SchemaToAvroSchema {
PrimitiveType::TimestampNs => AvroSchema::TimestampNanos,
PrimitiveType::TimestamptzNs => AvroSchema::TimestampNanos,
PrimitiveType::String => AvroSchema::String,
PrimitiveType::Uuid => avro_fixed_schema(UUID_BYTES, Some(UUID_LOGICAL_TYPE))?,
PrimitiveType::Fixed(len) => avro_fixed_schema((*len) as usize, None)?,
PrimitiveType::Uuid => AvroSchema::Uuid,
PrimitiveType::Fixed(len) => avro_fixed_schema((*len) as usize)?,
PrimitiveType::Binary => AvroSchema::Bytes,
PrimitiveType::Decimal { precision, scale } => {
avro_decimal_schema(*precision as usize, *scale as usize)?
Expand Down Expand Up @@ -274,21 +271,13 @@ fn avro_record_schema(name: &str, fields: Vec<AvroRecordField>) -> Result<AvroSc
}))
}

pub(crate) fn avro_fixed_schema(len: usize, logical_type: Option<&str>) -> Result<AvroSchema> {
let attributes = if let Some(logical_type) = logical_type {
BTreeMap::from([(
LOGICAL_TYPE.to_string(),
Value::String(logical_type.to_string()),
)])
} else {
Default::default()
};
pub(crate) fn avro_fixed_schema(len: usize) -> Result<AvroSchema> {
Ok(AvroSchema::Fixed(FixedSchema {
name: Name::new(format!("fixed_{len}").as_str())?,
aliases: None,
doc: None,
size: len,
attributes,
attributes: Default::default(),
default: None,
}))
}
Expand Down Expand Up @@ -533,30 +522,9 @@ impl AvroSchemaVisitor for AvroSchemaToSchema {
AvroSchema::Long => Type::Primitive(PrimitiveType::Long),
AvroSchema::Float => Type::Primitive(PrimitiveType::Float),
AvroSchema::Double => Type::Primitive(PrimitiveType::Double),
AvroSchema::Uuid => Type::Primitive(PrimitiveType::Uuid),
AvroSchema::String | AvroSchema::Enum(_) => Type::Primitive(PrimitiveType::String),
AvroSchema::Fixed(fixed) => {
if let Some(logical_type) = fixed.attributes.get(LOGICAL_TYPE) {
let logical_type = logical_type.as_str().ok_or_else(|| {
Error::new(
ErrorKind::DataInvalid,
"logicalType in attributes of avro schema is not a string type",
)
})?;
match logical_type {
UUID_LOGICAL_TYPE => Type::Primitive(PrimitiveType::Uuid),
ty => {
return Err(Error::new(
ErrorKind::FeatureUnsupported,
format!(
"Logical type {ty} is not support in iceberg primitive type.",
),
));
}
}
} else {
Type::Primitive(PrimitiveType::Fixed(fixed.size as u64))
}
}
AvroSchema::Fixed(fixed) => Type::Primitive(PrimitiveType::Fixed(fixed.size as u64)),
AvroSchema::Bytes => Type::Primitive(PrimitiveType::Binary),
AvroSchema::Null => return Ok(None),
_ => {
Expand Down Expand Up @@ -1223,4 +1191,25 @@ mod tests {
converter.primitive(&AvroSchema::Date).unwrap().unwrap()
);
}

#[test]
fn test_uuid_type() {
let avro_schema = {
AvroSchema::parse_str(
r#"
{"name": "test", "type": "fixed", "size": 16, "logicalType": "uuid"}
"#,
)
.unwrap()
};

let mut converter = AvroSchemaToSchema;

let iceberg_type = Type::from(PrimitiveType::Uuid);

assert_eq!(
iceberg_type,
converter.primitive(&avro_schema).unwrap().unwrap()
);
}
}
Loading