From 4e7c3f65ca99710481aafc01957ee281ec62ba93 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Mon, 1 Sep 2025 14:32:52 +0200 Subject: [PATCH 1/5] feat(PII): Add dynamic PII derivation This adds the ability to "dynamically" define PII to `metastructure`. It works like this: if `compute_pii` is a function `fn(&ProcessingState) -> Pii`, you can annotate a container or field with `#[metastructure(pii = "compute_pii")]` (in addition to the existing values of `"true"`, `"false"`, and `"maybe"`) which will cause the PII value of the field/container to depend on the current `ProcessingState`. See the new `test_dynamic_pii` test for a simple example of this feature in action. The intended use of this is to allow the PII status of `AttributeValue::value` to be fetched from `relay-conventions` based on the attribute's name. --- relay-event-derive/src/lib.rs | 30 +++++-- relay-event-schema/src/processor/attrs.rs | 104 ++++++++++++++++++++-- relay-pii/src/attachments.rs | 2 +- relay-pii/src/generate_selectors.rs | 2 +- relay-pii/src/processor.rs | 2 +- relay-pii/src/selector.rs | 2 +- 6 files changed, 123 insertions(+), 19 deletions(-) diff --git a/relay-event-derive/src/lib.rs b/relay-event-derive/src/lib.rs index 6a1854661fc..e3090a4621a 100644 --- a/relay-event-derive/src/lib.rs +++ b/relay-event-derive/src/lib.rs @@ -13,7 +13,7 @@ use proc_macro2::{Span, TokenStream}; use quote::{ToTokens, quote}; use syn::meta::ParseNestedMeta; -use syn::{Ident, Lit, LitBool, LitInt, LitStr}; +use syn::{ExprPath, Ident, Lit, LitBool, LitInt, LitStr}; use synstructure::decl_derive; mod utils; @@ -300,19 +300,27 @@ fn parse_type_attributes(s: &synstructure::Structure<'_>) -> syn::Result TokenStream { + fn as_tokens(&self) -> TokenStream { match self { - Pii::True => quote!(crate::processor::Pii::True), - Pii::False => quote!(crate::processor::Pii::False), - Pii::Maybe => quote!(crate::processor::Pii::Maybe), + Pii::True => quote!(crate::processor::PiiExtended::Static( + crate::processor::Pii::True + )), + Pii::False => quote!(crate::processor::PiiExtended::Static( + crate::processor::Pii::False + )), + Pii::Maybe => quote!(crate::processor::PiiExtended::Static( + crate::processor::Pii::Maybe + )), + Pii::Dynamic(fun) => quote!(crate::processor::PiiExtended::Dynamic(#fun)), } } } @@ -373,12 +381,14 @@ impl FieldAttrs { quote!(false) }; - let pii = if let Some(pii) = self.pii.or(type_attrs.pii) { + let pii = if let Some(pii) = self.pii.as_ref().or(type_attrs.pii.as_ref()) { pii.as_tokens() } else if let Some(ref parent_attrs) = inherit_from_field_attrs { quote!(#parent_attrs.pii) } else { - quote!(crate::processor::Pii::False) + quote!(crate::processor::PiiExtended::Static( + crate::processor::Pii::False + )) }; let trim = if let Some(trim) = self.trim.or(type_attrs.trim) { @@ -596,6 +606,8 @@ fn parse_pii_value(value: LitStr, meta: &ParseNestedMeta) -> syn::Result Pii::True, "false" => Pii::False, "maybe" => Pii::Maybe, - _ => return Err(meta.error("Expected one of `true`, `false`, `maybe`")), + _ => Pii::Dynamic(value.parse().map_err(|_| { + meta.error("Expected one of `true`, `false`, `maybe`, or a function name") + })?), })) } diff --git a/relay-event-schema/src/processor/attrs.rs b/relay-event-schema/src/processor/attrs.rs index 5abfc306ab1..e2923d032d3 100644 --- a/relay-event-schema/src/processor/attrs.rs +++ b/relay-event-schema/src/processor/attrs.rs @@ -106,6 +106,15 @@ pub enum Pii { Maybe, } +/// A static or dynamic `Pii` value. +#[derive(Debug, Clone, Copy)] +pub enum PiiExtended { + /// A static value. + Static(Pii), + /// A dynamic value, computed based on a `ProcessingState`. + Dynamic(fn(&ProcessingState) -> Pii), +} + /// Meta information about a field. #[derive(Debug, Clone, Copy)] pub struct FieldAttrs { @@ -128,7 +137,7 @@ pub struct FieldAttrs { /// The maximum number of bytes of this field. pub max_bytes: Option, /// The type of PII on the field. - pub pii: Pii, + pub pii: PiiExtended, /// Whether additional properties should be retained during normalization. pub retain: bool, /// Whether the trimming processor is allowed to shorten or drop this field. @@ -170,7 +179,7 @@ impl FieldAttrs { max_chars_allowance: 0, max_depth: None, max_bytes: None, - pii: Pii::False, + pii: PiiExtended::Static(Pii::False), retain: false, trim: true, } @@ -199,7 +208,13 @@ impl FieldAttrs { /// Sets whether this field contains PII. pub const fn pii(mut self, pii: Pii) -> Self { - self.pii = pii; + self.pii = PiiExtended::Static(pii); + self + } + + /// Sets whether this field contains PII dynamically based on the current state. + pub const fn pii_dynamic(mut self, pii: fn(&ProcessingState) -> Pii) -> Self { + self.pii = PiiExtended::Dynamic(pii); self } @@ -443,9 +458,18 @@ impl<'a> ProcessingState<'a> { /// Derives the attrs for recursion. pub fn inner_attrs(&self) -> Option> { match self.attrs().pii { - Pii::True => Some(Cow::Borrowed(&PII_TRUE_FIELD_ATTRS)), - Pii::False => None, - Pii::Maybe => Some(Cow::Borrowed(&PII_MAYBE_FIELD_ATTRS)), + PiiExtended::Static(Pii::True) => Some(Cow::Borrowed(&PII_TRUE_FIELD_ATTRS)), + PiiExtended::Static(Pii::False) => None, + PiiExtended::Static(Pii::Maybe) => Some(Cow::Borrowed(&PII_MAYBE_FIELD_ATTRS)), + PiiExtended::Dynamic(_) => None, + } + } + + /// Returns the PII status for this state. + pub fn pii(&self) -> Pii { + match self.attrs().pii { + PiiExtended::Static(pii) => pii, + PiiExtended::Dynamic(pii_fn) => pii_fn(self), } } @@ -561,6 +585,11 @@ impl Path<'_> { self.0.attrs() } + /// Returns the PII status for this path. + pub fn pii(&self) -> Pii { + self.0.pii() + } + /// Iterates through the states in this path. pub fn iter(&self) -> ProcessingStateIter<'_> { self.0.iter() @@ -585,3 +614,66 @@ impl fmt::Display for Path<'_> { Ok(()) } } + +#[cfg(test)] +mod tests { + + use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, SerializableAnnotated}; + + use crate::processor::attrs::ROOT_STATE; + use crate::processor::{Pii, ProcessValue, ProcessingState, Processor, process_value}; + + fn pii_from_item_name(state: &ProcessingState) -> Pii { + match state.path_item().and_then(|p| p.key()) { + Some("true_item") => Pii::True, + Some("false_item") => Pii::False, + _ => Pii::Maybe, + } + } + + #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)] + #[metastructure(pii = "pii_from_item_name")] + struct TestValue(String); + + struct TestProcessor; + + impl Processor for TestProcessor { + fn process_string( + &mut self, + value: &mut String, + _meta: &mut relay_protocol::Meta, + state: &ProcessingState<'_>, + ) -> crate::processor::ProcessingResult where { + match state.pii() { + Pii::True => *value = "true".to_owned(), + Pii::False => *value = "false".to_owned(), + Pii::Maybe => *value = "maybe".to_owned(), + } + Ok(()) + } + } + + #[test] + fn test_dynamic_pii() { + let mut object: Annotated> = Annotated::from_json( + r#" + { + "false_item": "replace me", + "other_item": "replace me", + "true_item": "replace me" + } + "#, + ) + .unwrap(); + + process_value(&mut object, &mut TestProcessor, &ROOT_STATE).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&object), @r###" + { + "false_item": "false", + "other_item": "maybe", + "true_item": "true" + } + "###); + } +} diff --git a/relay-pii/src/attachments.rs b/relay-pii/src/attachments.rs index e452767f073..5a7645ff90a 100644 --- a/relay-pii/src/attachments.rs +++ b/relay-pii/src/attachments.rs @@ -395,7 +395,7 @@ impl<'a> PiiAttachmentsProcessor<'a> { state: &ProcessingState<'_>, encodings: ScrubEncodings, ) -> bool { - let pii = state.attrs().pii; + let pii = state.pii(); if pii == Pii::False { return false; } diff --git a/relay-pii/src/generate_selectors.rs b/relay-pii/src/generate_selectors.rs index 7d12cac6c95..581466fc4b5 100644 --- a/relay-pii/src/generate_selectors.rs +++ b/relay-pii/src/generate_selectors.rs @@ -35,7 +35,7 @@ impl Processor for GenerateSelectorsProcessor { // The following skip-conditions are in sync with what the PiiProcessor does. if state.value_type().contains(ValueType::Boolean) || value.is_none() - || state.attrs().pii == Pii::False + || state.pii() == Pii::False { return Ok(()); } diff --git a/relay-pii/src/processor.rs b/relay-pii/src/processor.rs index a5f84488cb2..453fdbad89a 100644 --- a/relay-pii/src/processor.rs +++ b/relay-pii/src/processor.rs @@ -38,7 +38,7 @@ impl<'a> PiiProcessor<'a> { state: &ProcessingState<'_>, mut value: Option<&mut String>, ) -> ProcessingResult { - let pii = state.attrs().pii; + let pii = state.pii(); if pii == Pii::False { return Ok(()); } diff --git a/relay-pii/src/selector.rs b/relay-pii/src/selector.rs index 3d63799ce4b..064f4a3d79f 100644 --- a/relay-pii/src/selector.rs +++ b/relay-pii/src/selector.rs @@ -197,7 +197,7 @@ impl SelectorSpec { /// This walks both the selector and the path starting at the end and towards the root /// to determine if the selector matches the current path. pub fn matches_path(&self, path: &Path) -> bool { - let pii = path.attrs().pii; + let pii = path.pii(); if pii == Pii::False { return false; } From e395740e6af911e4402c301ffb72865a117276f5 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Mon, 1 Sep 2025 16:57:19 +0200 Subject: [PATCH 2/5] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d0206b004a..af13a4f37da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ **Features**: - Add InstallableBuild and SizeAnalysis data categories. ([#5084](https://github.com/getsentry/relay/pull/5084)) +- Add dynamic PII derivation to `metastructure`. ([#5107](https://github.com/getsentry/relay/pull/5107)) **Internal**: From d9837e83f420d0ab7f4bf770fa1473af61e88171 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 2 Sep 2025 10:14:31 +0200 Subject: [PATCH 3/5] Expand docs of ProcessingState::pii --- relay-event-schema/src/processor/attrs.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/relay-event-schema/src/processor/attrs.rs b/relay-event-schema/src/processor/attrs.rs index e2923d032d3..0c6089191d9 100644 --- a/relay-event-schema/src/processor/attrs.rs +++ b/relay-event-schema/src/processor/attrs.rs @@ -466,6 +466,10 @@ impl<'a> ProcessingState<'a> { } /// Returns the PII status for this state. + /// + /// If the state's `FieldAttrs` contain a fixed PII status, + /// it is returned. If they contain a dynamic PII status (a function), + /// it is applied to this state and the output returned. pub fn pii(&self) -> Pii { match self.attrs().pii { PiiExtended::Static(pii) => pii, From 43fd421000692333966e226229418d17cbf6aa4b Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 2 Sep 2025 10:15:00 +0200 Subject: [PATCH 4/5] Simplify ProcessingState::inner_attrs --- relay-event-schema/src/processor/attrs.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/relay-event-schema/src/processor/attrs.rs b/relay-event-schema/src/processor/attrs.rs index 0c6089191d9..854eee19ef1 100644 --- a/relay-event-schema/src/processor/attrs.rs +++ b/relay-event-schema/src/processor/attrs.rs @@ -457,11 +457,10 @@ impl<'a> ProcessingState<'a> { /// Derives the attrs for recursion. pub fn inner_attrs(&self) -> Option> { - match self.attrs().pii { - PiiExtended::Static(Pii::True) => Some(Cow::Borrowed(&PII_TRUE_FIELD_ATTRS)), - PiiExtended::Static(Pii::False) => None, - PiiExtended::Static(Pii::Maybe) => Some(Cow::Borrowed(&PII_MAYBE_FIELD_ATTRS)), - PiiExtended::Dynamic(_) => None, + match self.pii() { + Pii::True => Some(Cow::Borrowed(&PII_TRUE_FIELD_ATTRS)), + Pii::False => None, + Pii::Maybe => Some(Cow::Borrowed(&PII_MAYBE_FIELD_ATTRS)), } } From d3450c22e891b90debfc5d99d1ba94c3770ab5bb Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 2 Sep 2025 10:18:47 +0200 Subject: [PATCH 5/5] Rename PiiExtended to PiiMode --- relay-event-derive/src/lib.rs | 10 +++++----- relay-event-schema/src/processor/attrs.rs | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/relay-event-derive/src/lib.rs b/relay-event-derive/src/lib.rs index e3090a4621a..0ef6b3d1c81 100644 --- a/relay-event-derive/src/lib.rs +++ b/relay-event-derive/src/lib.rs @@ -311,16 +311,16 @@ enum Pii { impl Pii { fn as_tokens(&self) -> TokenStream { match self { - Pii::True => quote!(crate::processor::PiiExtended::Static( + Pii::True => quote!(crate::processor::PiiMode::Static( crate::processor::Pii::True )), - Pii::False => quote!(crate::processor::PiiExtended::Static( + Pii::False => quote!(crate::processor::PiiMode::Static( crate::processor::Pii::False )), - Pii::Maybe => quote!(crate::processor::PiiExtended::Static( + Pii::Maybe => quote!(crate::processor::PiiMode::Static( crate::processor::Pii::Maybe )), - Pii::Dynamic(fun) => quote!(crate::processor::PiiExtended::Dynamic(#fun)), + Pii::Dynamic(fun) => quote!(crate::processor::PiiMode::Dynamic(#fun)), } } } @@ -386,7 +386,7 @@ impl FieldAttrs { } else if let Some(ref parent_attrs) = inherit_from_field_attrs { quote!(#parent_attrs.pii) } else { - quote!(crate::processor::PiiExtended::Static( + quote!(crate::processor::PiiMode::Static( crate::processor::Pii::False )) }; diff --git a/relay-event-schema/src/processor/attrs.rs b/relay-event-schema/src/processor/attrs.rs index 854eee19ef1..891cb3a8dd8 100644 --- a/relay-event-schema/src/processor/attrs.rs +++ b/relay-event-schema/src/processor/attrs.rs @@ -108,7 +108,7 @@ pub enum Pii { /// A static or dynamic `Pii` value. #[derive(Debug, Clone, Copy)] -pub enum PiiExtended { +pub enum PiiMode { /// A static value. Static(Pii), /// A dynamic value, computed based on a `ProcessingState`. @@ -137,7 +137,7 @@ pub struct FieldAttrs { /// The maximum number of bytes of this field. pub max_bytes: Option, /// The type of PII on the field. - pub pii: PiiExtended, + pub pii: PiiMode, /// Whether additional properties should be retained during normalization. pub retain: bool, /// Whether the trimming processor is allowed to shorten or drop this field. @@ -179,7 +179,7 @@ impl FieldAttrs { max_chars_allowance: 0, max_depth: None, max_bytes: None, - pii: PiiExtended::Static(Pii::False), + pii: PiiMode::Static(Pii::False), retain: false, trim: true, } @@ -208,13 +208,13 @@ impl FieldAttrs { /// Sets whether this field contains PII. pub const fn pii(mut self, pii: Pii) -> Self { - self.pii = PiiExtended::Static(pii); + self.pii = PiiMode::Static(pii); self } /// Sets whether this field contains PII dynamically based on the current state. pub const fn pii_dynamic(mut self, pii: fn(&ProcessingState) -> Pii) -> Self { - self.pii = PiiExtended::Dynamic(pii); + self.pii = PiiMode::Dynamic(pii); self } @@ -471,8 +471,8 @@ impl<'a> ProcessingState<'a> { /// it is applied to this state and the output returned. pub fn pii(&self) -> Pii { match self.attrs().pii { - PiiExtended::Static(pii) => pii, - PiiExtended::Dynamic(pii_fn) => pii_fn(self), + PiiMode::Static(pii) => pii, + PiiMode::Dynamic(pii_fn) => pii_fn(self), } }