diff --git a/CHANGELOG.md b/CHANGELOG.md index 68c92f5b6f2..fb195aaa550 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Improved PII Scrubbing for attributes / logs ([#5061](https://github.com/getsentry/relay/pull/5061))) + ## 25.8.0 - Add data categories for Prevent. ([#5052](https://github.com/getsentry/relay/pull/5052)) diff --git a/py/CHANGELOG.md b/py/CHANGELOG.md index f81e44a9325..cd92b0e3d24 100644 --- a/py/CHANGELOG.md +++ b/py/CHANGELOG.md @@ -5,6 +5,7 @@ - Add `trusted_relay_settings` to the project configuration. ([#4772](https://github.com/getsentry/relay/pull/4772)) - Add `downsampled_event_retention` to the project configuration. ([#5013](https://github.com/getsentry/relay/pull/5013)) - Add data categories for Prevent. ([#5052](https://github.com/getsentry/relay/pull/5052)) +- Improved PII Scrubbing for attributes / logs ([#5061](https://github.com/getsentry/relay/pull/5061))) ## 0.9.12 diff --git a/relay-event-schema/src/processor/attrs.rs b/relay-event-schema/src/processor/attrs.rs index 1121d76c306..5abfc306ab1 100644 --- a/relay-event-schema/src/processor/attrs.rs +++ b/relay-event-schema/src/processor/attrs.rs @@ -85,7 +85,8 @@ relay_common::derive_fromstr_and_display!(ValueType, UnknownValueTypeError, { ValueType::Message => "message", ValueType::Thread => "thread", ValueType::Breadcrumb => "breadcrumb", - ValueType::OurLog => "ourlog", + ValueType::OurLog => "log", + ValueType::Span => "span", ValueType::ClientSdkInfo => "sdk", ValueType::Minidump => "minidump", diff --git a/relay-event-schema/src/processor/traits.rs b/relay-event-schema/src/processor/traits.rs index 6a7058e5643..b6bf80a3124 100644 --- a/relay-event-schema/src/processor/traits.rs +++ b/relay-event-schema/src/processor/traits.rs @@ -114,6 +114,7 @@ pub trait Processor: Sized { process_method!(process_trace_context, crate::protocol::TraceContext); process_method!(process_native_image_path, crate::protocol::NativeImagePath); process_method!(process_contexts, crate::protocol::Contexts); + process_method!(process_attributes, crate::protocol::Attributes); fn process_other( &mut self, diff --git a/relay-event-schema/src/protocol/attributes.rs b/relay-event-schema/src/protocol/attributes.rs index 254ae0625a8..8000966adde 100644 --- a/relay-event-schema/src/protocol/attributes.rs +++ b/relay-event-schema/src/protocol/attributes.rs @@ -1,7 +1,10 @@ -use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, SkipSerialization, Value}; +use enumset::EnumSet; +use relay_protocol::{ + Annotated, Empty, FromValue, IntoValue, Meta, Object, SkipSerialization, Value, +}; use std::{borrow::Borrow, fmt}; -use crate::processor::ProcessValue; +use crate::processor::{ProcessValue, ProcessingResult, ProcessingState, Processor, ValueType}; #[derive(Clone, PartialEq, Empty, FromValue, IntoValue, ProcessValue)] pub struct Attribute { @@ -161,7 +164,7 @@ impl IntoValue for AttributeType { } /// Wrapper struct around a collection of attributes with some convenience methods. -#[derive(Debug, Clone, Default, PartialEq, Empty, FromValue, IntoValue, ProcessValue)] +#[derive(Debug, Clone, Default, PartialEq, Empty, FromValue, IntoValue)] pub struct Attributes(pub Object); impl Attributes { @@ -249,3 +252,34 @@ impl FromIterator<(String, Annotated)> for Attributes { Self(Object::from_iter(iter)) } } + +impl ProcessValue for Attributes { + #[inline] + fn value_type(&self) -> EnumSet { + EnumSet::only(ValueType::Object) + } + + #[inline] + fn process_value

( + &mut self, + meta: &mut Meta, + processor: &mut P, + state: &ProcessingState<'_>, + ) -> ProcessingResult + where + P: Processor, + { + processor.process_attributes(self, meta, state) + } + + fn process_child_values

( + &mut self, + _processor: &mut P, + _state: &ProcessingState<'_>, + ) -> ProcessingResult + where + P: Processor, + { + Ok(()) + } +} diff --git a/relay-event-schema/src/protocol/ourlog.rs b/relay-event-schema/src/protocol/ourlog.rs index d4247cb0721..07daba8690b 100644 --- a/relay-event-schema/src/protocol/ourlog.rs +++ b/relay-event-schema/src/protocol/ourlog.rs @@ -33,7 +33,7 @@ pub struct OurLog { pub body: Annotated, /// Arbitrary attributes on a log. - #[metastructure(pii = "true", trim = false)] + #[metastructure(pii = "maybe", trim = false)] pub attributes: Annotated, /// Additional arbitrary fields for forwards compatibility. diff --git a/relay-pii/src/generate_selectors.rs b/relay-pii/src/generate_selectors.rs index f7672e9e0d8..7d12cac6c95 100644 --- a/relay-pii/src/generate_selectors.rs +++ b/relay-pii/src/generate_selectors.rs @@ -139,6 +139,14 @@ pub fn selector_suggestions_from_value( selectors: BTreeSet::new(), }; + // OurLog (logs) should not appear in selector suggestions as that is used by the old modal. + // This should not be possible unless we are explicitly sending ourlog json to 'relay_pii_selector_suggestions_from_event', but guarding against it just in case. + if let Some(value) = value.value_mut() + && value.value_type().contains(ValueType::OurLog) + { + return BTreeSet::new(); + } + processor::process_value(value, &mut processor, ProcessingState::root()) .expect("This processor is supposed to be infallible"); @@ -147,7 +155,7 @@ pub fn selector_suggestions_from_value( #[cfg(test)] mod tests { - use relay_event_schema::protocol::Event; + use relay_event_schema::protocol::{Event, OurLog}; use super::*; @@ -290,4 +298,28 @@ mod tests { ] "###); } + + #[test] + fn test_attributes() { + let mut annotated_log = Annotated::::from_json( + r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test", + "attributes": { + "example_attribute": { + "type": "string", + "value": "abc123" + } + } + }"#, + ) + .unwrap(); + + let selectors = selector_suggestions_from_value(&mut annotated_log); + insta::assert_json_snapshot!(selectors, @"[]"); + } } diff --git a/relay-pii/src/processor.rs b/relay-pii/src/processor.rs index bad4f865344..a5f84488cb2 100644 --- a/relay-pii/src/processor.rs +++ b/relay-pii/src/processor.rs @@ -199,6 +199,15 @@ impl Processor for PiiProcessor<'_> { utils::process_pairlist(self, value, state) } + fn process_attributes( + &mut self, + value: &mut relay_event_schema::protocol::Attributes, + _meta: &mut Meta, + state: &ProcessingState, + ) -> ProcessingResult { + utils::process_attributes(value, self, state) + } + fn process_user( &mut self, user: &mut User, diff --git a/relay-pii/src/selector.rs b/relay-pii/src/selector.rs index 51de94d3ada..3d63799ce4b 100644 --- a/relay-pii/src/selector.rs +++ b/relay-pii/src/selector.rs @@ -659,4 +659,16 @@ mod tests { // WAT. We have the full path to a field here. assert_matches_pii_true!(minidump_state_inner, "$attachments.$minidump.$binary",); } + + #[test] + fn test_logs_matching() { + let event_state = ProcessingState::new_root(None, None); + let log_state = event_state.enter_static("", None, Some(ValueType::OurLog)); // . + let body_state = log_state.enter_static("body", None, Some(ValueType::String)); + let attributes_state = log_state.enter_static("attributes", None, Some(ValueType::Object)); + + assert_matches_pii_maybe!(log_state, "$log",); + assert_matches_pii_true!(body_state, "$log.body",); + assert_matches_pii_true!(attributes_state, "$log.attributes",); + } } diff --git a/relay-pii/src/utils.rs b/relay-pii/src/utils.rs index 5b83fc9159f..e6757ed888b 100644 --- a/relay-pii/src/utils.rs +++ b/relay-pii/src/utils.rs @@ -1,8 +1,12 @@ +use std::borrow::Cow; + use hmac::{Hmac, Mac}; + use relay_event_schema::processor::{ - self, ProcessValue, ProcessingResult, ProcessingState, Processor, ValueType, + self, FieldAttrs, Pii, ProcessValue, ProcessingResult, ProcessingState, Processor, ValueType, + enum_set, }; -use relay_event_schema::protocol::{AsPair, PairList}; +use relay_event_schema::protocol::{AsPair, Attributes, PairList}; use sha1::Sha1; pub fn process_pairlist( @@ -37,6 +41,46 @@ pub fn process_pairlist( Ok(()) } +pub fn process_attributes( + value: &mut Attributes, + slf: &mut P, + state: &ProcessingState, +) -> ProcessingResult { + // Check if we're in default rules (no root ValueType) + // This is a workaround to support explicit selectors eg. $log.attributes.KEY.value to work but keep implicit selectors for default rules. + let is_advanced_rules = state + .iter() + .nth(1) + .is_some_and(|s| s.value_type().contains(ValueType::OurLog)); + + for (key, annotated_attribute) in value.iter_mut() { + if let Some(attribute) = annotated_attribute.value_mut() { + if is_advanced_rules { + // Process normally to allow explicit selectors like $log.attributes.KEY.value to work + let field_value_type = ValueType::for_field(annotated_attribute); + let key_state = state.enter_borrowed(key, state.inner_attrs(), field_value_type); + processor::process_value(annotated_attribute, slf, &key_state)?; + } else { + // For the default rules, we want Pii::True since we want them to always be scrubbed. + let attrs = FieldAttrs::new().pii(Pii::True); + let inner_value = &mut attribute.value.value; + let inner_value_type = ValueType::for_field(inner_value); + let entered = + state.enter_borrowed(key, Some(Cow::Borrowed(&attrs)), inner_value_type); + processor::process_value(inner_value, slf, &entered)?; + + let object_value_type = enum_set!(ValueType::Object); + for (key, value) in attribute.other.iter_mut() { + let other_state = + state.enter_borrowed(key, Some(Cow::Borrowed(&attrs)), object_value_type); + processor::process_value(value, slf, &other_state)?; + } + } + } + } + Ok(()) +} + pub fn hash_value(data: &[u8]) -> String { let mut mac = Hmac::::new_from_slice(&[]).unwrap(); mac.update(data); diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 9434a7527eb..08974014f64 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -1,7 +1,7 @@ use relay_event_normalization::{ ClientHints, FromUserAgentInfo as _, RawUserAgentInfo, SchemaProcessor, }; -use relay_event_schema::processor::{ProcessingState, process_value}; +use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::{AttributeType, BrowserContext, OurLog, OurLogHeader}; use relay_metrics::UnixTimestamp; use relay_ourlogs::OtelLog; @@ -132,7 +132,7 @@ fn expand_log_container(item: &Item, trust: RequestTrust) -> Result, ctx: Context<'_>) -> Result<()> { - let pii_config = ctx + let pii_config_from_scrubbing = ctx .project_info .config .datascrubbing_settings @@ -141,12 +141,16 @@ fn scrub_log(log: &mut Annotated, ctx: Context<'_>) -> Result<()> { if let Some(ref config) = ctx.project_info.config.pii_config { let mut processor = PiiProcessor::new(config.compiled()); - process_value(log, &mut processor, ProcessingState::root())?; + let root_state = ProcessingState::root().enter_static("", None, Some(ValueType::OurLog)); + process_value(log, &mut processor, &root_state)?; } - if let Some(config) = pii_config { + if let Some(config) = pii_config_from_scrubbing { let mut processor = PiiProcessor::new(config.compiled()); - process_value(log, &mut processor, ProcessingState::root())?; + // Use empty root (assumed to be Event) for legacy/default scrubbing rules. + // process_attributes will collapse Attribute into it's value for the default rules. + let root_state = ProcessingState::root(); + process_value(log, &mut processor, root_state)?; } Ok(()) @@ -247,10 +251,38 @@ fn process_attribute_types(ourlog: &mut OurLog) { #[cfg(test)] mod tests { + use relay_pii::{DataScrubbingConfig, PiiConfig}; use relay_protocol::SerializableAnnotated; + use crate::services::projects::project::ProjectInfo; + use super::*; + fn make_context( + scrubbing_config: DataScrubbingConfig, + pii_config: Option, + ) -> Context<'static> { + let config = Box::leak(Box::new(relay_config::Config::default())); + let global_config = Box::leak(Box::new(relay_dynamic_config::GlobalConfig::default())); + let project_info = Box::leak(Box::new(ProjectInfo { + config: relay_dynamic_config::ProjectConfig { + pii_config, + datascrubbing_settings: scrubbing_config, + ..Default::default() + }, + ..Default::default() + })); + let rate_limits = Box::leak(Box::new(relay_quotas::RateLimits::default())); + + Context { + config, + global_config, + project_info, + rate_limits, + sampling_project_info: None, + } + } + #[test] fn test_process_attribute_types() { let json = r#"{ @@ -442,4 +474,1360 @@ mod tests { &Value::String("131.0.0".to_owned()), ); } + + #[test] + fn test_scrub_log_pii() { + let json = r#"{ + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log message with sensitive data: user@example.com and 4571234567890111", + "attributes": { + "password": { + "type": "string", + "value": "secret123" + }, + "secret": { + "type": "string", + "value": "topsecret" + }, + "api_key": { + "type": "string", + "value": "sk-1234567890abcdef" + }, + "auth_token": { + "type": "string", + "value": "bearer_token_123" + }, + "credit_card": { + "type": "string", + "value": "4571234567890111" + }, + "visa_card": { + "type": "string", + "value": "4571 2345 6789 0111" + }, + "bank_account": { + "type": "string", + "value": "DE89370400440532013000" + }, + "iban_code": { + "type": "string", + "value": "NO9386011117945" + }, + "authorization": { + "type": "string", + "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + }, + "private_key": { + "type": "string", + "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" + }, + "database_url": { + "type": "string", + "value": "https://username:password@example.com/db" + }, + "file_path": { + "type": "string", + "value": "C:\\Users\\john\\Documents\\secret.txt" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "social_security": { + "type": "string", + "value": "078-05-1120" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "client_ip": { + "type": "string", + "value": "192.168.1.100" + }, + "ipv6_addr": { + "type": "string", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "very_sensitive_data": { + "type": "string", + "value": "very_sensitive_data" + } + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = true; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ]; + scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; + + let ctx = make_context(scrubbing_config, None); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "api_key": { + "type": "string", + "value": "[Filtered]" + }, + "auth_token": { + "type": "string", + "value": "[Filtered]" + }, + "authorization": { + "type": "string", + "value": "[Filtered]" + }, + "bank_account": { + "type": "string", + "value": "[Filtered]" + }, + "client_ip": { + "type": "string", + "value": "[ip]" + }, + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "database_url": { + "type": "string", + "value": "[Filtered]" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "file_path": { + "type": "string", + "value": "[Filtered]" + }, + "iban_code": { + "type": "string", + "value": "[Filtered]" + }, + "ipv6_addr": { + "type": "string", + "value": "[ip]" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "private_key": { + "type": "string", + "value": "[Filtered]" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "secret": { + "type": "string", + "value": "[Filtered]" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "social_security": { + "type": "string", + "value": "[Filtered]" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "very_sensitive_data": { + "type": "string", + "value": "[Filtered]" + }, + "visa_card": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "api_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "auth_token": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "authorization": { + "value": { + "": { + "rem": [ + [ + "@bearer:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + }, + "bank_account": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 22 + } + } + }, + "client_ip": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 13 + } + } + }, + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "database_url": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 40 + } + } + }, + "file_path": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 34 + } + } + }, + "iban_code": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + }, + "ipv6_addr": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 39 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "private_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 118 + } + } + }, + "secret": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "social_security": { + "value": { + "": { + "rem": [ + [ + "@usssn:filter", + "s", + 0, + 10 + ] + ], + "len": 11 + } + } + }, + "very_sensitive_data": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "visa_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_log_pii_custom_object_rules() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log message with sensitive data: user@example.com and 4571234567890111", + "attributes": { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ] + } + } + )) + .unwrap(); + + let ctx = make_context(scrubbing_config, Some(config)); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "_meta": { + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_log_pii_string_rules() { + let test_cases = vec![ + // IP rules + ("@ip", "127.0.0.1"), + ("@ip:replace", "127.0.0.1"), + ("@ip:remove", "127.0.0.1"), + ("@ip:mask", "127.0.0.1"), + // Email rules + ("@email", "test@example.com"), + ("@email:replace", "test@example.com"), + ("@email:remove", "test@example.com"), + ("@email:mask", "test@example.com"), + // Credit card rules + ("@creditcard", "4242424242424242"), + ("@creditcard:replace", "4242424242424242"), + ("@creditcard:remove", "4242424242424242"), + ("@creditcard:mask", "4242424242424242"), + // IBAN rules + ("@iban", "DE89370400440532013000"), + ("@iban:replace", "DE89370400440532013000"), + ("@iban:remove", "DE89370400440532013000"), + ("@iban:mask", "DE89370400440532013000"), + // MAC address rules + ("@mac", "4a:00:04:10:9b:50"), + ("@mac:replace", "4a:00:04:10:9b:50"), + ("@mac:remove", "4a:00:04:10:9b:50"), + ("@mac:mask", "4a:00:04:10:9b:50"), + // UUID rules + ("@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + // IMEI rules + ("@imei", "356938035643809"), + ("@imei:replace", "356938035643809"), + ("@imei:remove", "356938035643809"), + // PEM key rules + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + ( + "@pemkey:replace", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + ( + "@pemkey:remove", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + // URL auth rules + ("@urlauth", "https://username:password@example.com/"), + ("@urlauth:replace", "https://username:password@example.com/"), + ("@urlauth:remove", "https://username:password@example.com/"), + ("@urlauth:mask", "https://username:password@example.com/"), + // US SSN rules + ("@usssn", "078-05-1120"), + ("@usssn:replace", "078-05-1120"), + ("@usssn:remove", "078-05-1120"), + ("@usssn:mask", "078-05-1120"), + // User path rules + ("@userpath", "/Users/john/Documents"), + ("@userpath:replace", "/Users/john/Documents"), + ("@userpath:remove", "/Users/john/Documents"), + ("@userpath:mask", "/Users/john/Documents"), + // Password rules + ("@password", "my_password_123"), // @password defaults to remove + ("@password:remove", "my_password_123"), + ("@password:replace", "my_password_123"), + ("@password:mask", "my_password_123"), + // Bearer token rules + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"), + ( + "@bearer:replace", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ( + "@bearer:remove", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ( + "@bearer:mask", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ]; + + for (rule_type, test_value) in test_cases { + let json = format!( + r#"{{ + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = rule_type, + value = test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut data = Annotated::::from_json(&json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": [rule_type] + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes)); + } + } + + #[test] + fn test_scrub_log_base_fields() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + "attributes": {} + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": ["@password:filter"], + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + + let ctx = make_context(scrubbing_config, Some(config)); + scrub_log(&mut data, ctx).unwrap(); + insta::assert_json_snapshot!(SerializableAnnotated(&data)); + } + + #[test] + fn test_scrub_log_implicit_attribute_value_does_not_match() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "$log.attributes.remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "###); + } + + #[test] + fn test_scrub_log_explicit_attribute_value_does_match() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "$log.attributes.remove_this_string_abc123.value": ["remove_abc123"], + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc---" + } + }, + "_meta": { + "attributes": { + "remove_this_string_abc123": { + "value": { + "": { + "rem": [ + [ + "remove_abc123", + "s", + 0, + 6 + ] + ], + "len": 6 + } + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_log_sensitive_fields() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": { + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "should_be_removed" + }, + "another_sensitive": { + "type": "integer", + "value": 42 + }, + "my_value": { + "type": "string", + "value": "this_should_be_removed_as_sensitive" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "sensitive_custom".to_owned(), + "another_sensitive".to_owned(), + ]; + + let ctx = make_context(scrubbing_config, None); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "another_sensitive": { + "type": "integer", + "value": null + }, + "my_value": { + "type": "string", + "value": "[Filtered]" + }, + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "another_sensitive": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "x" + ] + ] + } + } + }, + "my_value": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 35 + } + } + }, + "sensitive_custom": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 17 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_log_safe_fields() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log with email@example.com", + "attributes": { + "password": { + "type": "string", + "value": "secret123" + }, + "credit_card": { + "type": "string", + "value": "4242424242424242" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = false; + scrubbing_config.exclude_fields = vec!["secret".to_owned()]; // Only 'secret' is safe + let ctx = make_context(scrubbing_config, None); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + }, + "_meta": { + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_log_deep_wild_cards() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "normal_value", + "attributes": { + "normal_field": { + "type": "string", + "value": "normal_value" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let deep_wildcard_config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_normal_field": { + "type": "pattern", + "pattern": "normal_value", + "redaction": { + "method": "replace", + "text": "[REDACTED]" + } + } + }, + "applications": { + "**": ["remove_normal_field"] + } + })) + .unwrap(); + + let ctx = make_context(DataScrubbingConfig::default(), Some(deep_wildcard_config)); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "[REDACTED]", + "attributes": { + "normal_field": { + "type": "string", + "value": "[REDACTED]" + } + }, + "_meta": { + "attributes": { + "normal_field": { + "value": { + "": { + "rem": [ + [ + "remove_normal_field", + "s", + 0, + 10 + ] + ], + "len": 12 + } + } + } + }, + "body": { + "": { + "rem": [ + [ + "remove_normal_field", + "s", + 0, + 10 + ] + ], + "len": 12 + } + } + } + } + "###); + + // If a log specific negation is used, then log attributes appear again. + data = Annotated::::from_json(json).unwrap(); + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "should_not_remove_normal_field": { + "type": "pattern", + "pattern": "normal_value", + "redaction": { + "method": "replace", + "text": "[REDACTED]" + } + } + }, + "applications": { + "** && !$log.**": ["should_not_remove_normal_field"] + } + })) + .unwrap(); + + let ctx = make_context(DataScrubbingConfig::default(), Some(config)); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "normal_value", + "attributes": { + "normal_field": { + "type": "string", + "value": "normal_value" + } + } + } + "###); + } + + #[test] + fn test_scrub_log_deep_wild_cards_anything() { + let json = r#" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "normal_value", + "attributes": { + "normal_field": { + "type": "string", + "value": "normal_value" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_normal_field": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[REDACTED]" + } + } + }, + "applications": { + "**": ["remove_normal_field"] + } + })) + .unwrap(); + + let ctx = make_context(DataScrubbingConfig::default(), Some(config)); + scrub_log(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "[REDACTED]", + "attributes": { + "normal_field": { + "type": "string", + "value": "[REDACTED]" + } + }, + "_meta": { + "attributes": { + "normal_field": { + "value": { + "": { + "rem": [ + [ + "remove_normal_field", + "s", + 0, + 10 + ] + ], + "len": 12 + } + } + } + }, + "body": { + "": { + "rem": [ + [ + "remove_normal_field", + "s", + 0, + 10 + ] + ], + "len": 12 + } + } + } + } + "###); + } } diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_base_fields.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_base_fields.snap new file mode 100644 index 00000000000..1f6f329aa64 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_base_fields.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data) +--- +{ + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "[Filtered]", + "attributes": {}, + "_meta": { + "body": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap new file mode 100644 index 00000000000..41c9d6f79ca --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:replace|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard:replace|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:replace", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap new file mode 100644 index 00000000000..913c5f144b6 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:remove|4242424242424242": { + "type": "string", + "value": "" + }, + "_meta": { + "@creditcard:remove|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap new file mode 100644 index 00000000000..657f4914f6a --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:mask|4242424242424242": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@creditcard:mask|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap new file mode 100644 index 00000000000..19a90e43cf7 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap new file mode 100644 index 00000000000..9fd82d71461 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:replace|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban:replace|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:replace", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap new file mode 100644 index 00000000000..ee757166106 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:remove|DE89370400440532013000": { + "type": "string", + "value": "" + }, + "_meta": { + "@iban:remove|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:remove", + "x", + 0, + 0 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap new file mode 100644 index 00000000000..28fc20365c3 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:mask|DE89370400440532013000": { + "type": "string", + "value": "**********************" + }, + "_meta": { + "@iban:mask|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:mask", + "m", + 0, + 22 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap new file mode 100644 index 00000000000..a31b5045b9c --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap new file mode 100644 index 00000000000..374def85bb7 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:replace|4a:00:04:10:9b:50": { + "type": "string", + "value": "[mac]" + }, + "_meta": { + "@mac:replace|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:replace", + "s", + 0, + 5 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap new file mode 100644 index 00000000000..53d0822aacb --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:remove|4a:00:04:10:9b:50": { + "type": "string", + "value": "" + }, + "_meta": { + "@mac:remove|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:remove", + "x", + 0, + 0 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap new file mode 100644 index 00000000000..bfab00f6000 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:replace|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip:replace|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap new file mode 100644 index 00000000000..0e54edf838d --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:mask|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac:mask|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:mask", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap new file mode 100644 index 00000000000..9d2fdf8e1b3 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap new file mode 100644 index 00000000000..6add3fbe4c2 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "[uuid]" + }, + "_meta": { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:replace", + "s", + 0, + 6 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap new file mode 100644 index 00000000000..ecd17f3f6a2 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "" + }, + "_meta": { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:remove", + "x", + 0, + 0 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap new file mode 100644 index 00000000000..9dd20e89202 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:mask", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap new file mode 100644 index 00000000000..0091511248e --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap new file mode 100644 index 00000000000..f05004683e7 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei:replace|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei:replace|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:replace", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap new file mode 100644 index 00000000000..f207cc734da --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei:remove|356938035643809": { + "type": "string", + "value": "" + }, + "_meta": { + "@imei:remove|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap new file mode 100644 index 00000000000..a547b9b429e --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap new file mode 100644 index 00000000000..23a25e68a32 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:replace", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap new file mode 100644 index 00000000000..8a66721fcb4 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:remove|127.0.0.1": { + "type": "string", + "value": "" + }, + "_meta": { + "@ip:remove|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:remove", + "x", + 0, + 0 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap new file mode 100644 index 00000000000..dae1df1dca2 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:remove", + "x", + 31, + 31 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap new file mode 100644 index 00000000000..47d54445c74 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap new file mode 100644 index 00000000000..f5167af0386 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:replace|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth:replace|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:replace", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap new file mode 100644 index 00000000000..4d87f406b35 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:remove|https://username:password@example.com/": { + "type": "string", + "value": "https://@example.com/" + }, + "_meta": { + "@urlauth:remove|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:remove", + "x", + 8, + 8 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap new file mode 100644 index 00000000000..4223b7551db --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:mask|https://username:password@example.com/": { + "type": "string", + "value": "https://*****************@example.com/" + }, + "_meta": { + "@urlauth:mask|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:mask", + "m", + 8, + 25 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap new file mode 100644 index 00000000000..2f67315fcd1 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap new file mode 100644 index 00000000000..5e97251cb03 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:replace|078-05-1120": { + "type": "string", + "value": "[us-ssn]" + }, + "_meta": { + "@usssn:replace|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:replace", + "s", + 0, + 8 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap new file mode 100644 index 00000000000..e3a7906f097 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:remove|078-05-1120": { + "type": "string", + "value": "" + }, + "_meta": { + "@usssn:remove|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:remove", + "x", + 0, + 0 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap new file mode 100644 index 00000000000..9623a53cea0 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:mask|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn:mask|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:mask", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap new file mode 100644 index 00000000000..a66df61b449 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap new file mode 100644 index 00000000000..ce8078e830c --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:mask|127.0.0.1": { + "type": "string", + "value": "*********" + }, + "_meta": { + "@ip:mask|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:mask", + "m", + 0, + 9 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap new file mode 100644 index 00000000000..547fd830f26 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:replace|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath:replace|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:replace", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap new file mode 100644 index 00000000000..5b05bee508e --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:remove|/Users/john/Documents": { + "type": "string", + "value": "/Users//Documents" + }, + "_meta": { + "@userpath:remove|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:remove", + "x", + 7, + 7 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap new file mode 100644 index 00000000000..08036842544 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:mask|/Users/john/Documents": { + "type": "string", + "value": "/Users/****/Documents" + }, + "_meta": { + "@userpath:mask|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:mask", + "m", + 7, + 11 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap new file mode 100644 index 00000000000..5747256ab36 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap new file mode 100644 index 00000000000..bfe78f0ce11 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:remove|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password:remove|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap new file mode 100644 index 00000000000..1146e3722c8 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:replace|my_password_123": { + "type": "string", + "value": "[password]" + }, + "_meta": { + "@password:replace|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:replace", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap new file mode 100644 index 00000000000..c2534332bb1 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:mask|my_password_123": { + "type": "string", + "value": "***************" + }, + "_meta": { + "@password:mask|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:mask", + "m", + 0, + 15 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap new file mode 100644 index 00000000000..f5cac4a7f8f --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap new file mode 100644 index 00000000000..5d31471635c --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:replace", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap new file mode 100644 index 00000000000..d4c861f0031 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "" + }, + "_meta": { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:remove", + "x", + 0, + 0 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap new file mode 100644 index 00000000000..fbb15d10c5a --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap new file mode 100644 index 00000000000..498748ea4e4 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "*******************************************" + }, + "_meta": { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:mask", + "m", + 0, + 43 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap new file mode 100644 index 00000000000..4f8eafb6159 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:replace|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email:replace|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:replace", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap new file mode 100644 index 00000000000..a52fd1dce91 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:remove|test@example.com": { + "type": "string", + "value": "" + }, + "_meta": { + "@email:remove|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap new file mode 100644 index 00000000000..6e00d96e019 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:mask|test@example.com": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@email:mask|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap new file mode 100644 index 00000000000..d16ef12fee6 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap new file mode 100644 index 00000000000..370b5fc7527 --- /dev/null +++ b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/logs/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index e486d220d99..ce06c85e21a 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -384,6 +384,259 @@ def test_ourlog_extraction_with_sentry_logs( ] +@pytest.mark.parametrize( + "rule_type,test_value,expected_scrubbed", + [ + ("@ip", "127.0.0.1", "[ip]"), + ("@email", "test@example.com", "[email]"), + ("@creditcard", "4242424242424242", "[creditcard]"), + ("@iban", "DE89370400440532013000", "[iban]"), + ("@mac", "4a:00:04:10:9b:50", "*****************"), + ( + "@uuid", + "ceee0822-ed8f-4622-b2a3-789e73e75cd1", + "************************************", + ), + ("@imei", "356938035643809", "[imei]"), + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", + ), + ( + "@urlauth", + "https://username:password@example.com/", + "https://[auth]@example.com/", + ), + ("@usssn", "078-05-1120", "***********"), + ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), + ("@password", "my_password_123", ""), + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), + ], +) +def test_ourlog_extraction_with_string_pii_scrubbing( + mini_sentry, + relay, + items_consumer, + rule_type, + test_value, + expected_scrubbed, +): + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:ourlogs-ingestion", + ] + + project_config["config"]["piiConfig"]["applications"] = {"$string": [rule_type]} + + relay_instance = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_sentry_logs( + { + "timestamp": ts.timestamp(), + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": { + "test_pii": {"value": test_value, "type": "string"}, + }, + } + ) + + relay_instance.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + attributes = item["attributes"] + + assert "test_pii" in attributes + assert attributes["test_pii"]["value"] == expected_scrubbed + assert "_meta" in item + meta = item["_meta"]["attributes"]["test_pii"]["value"][""] + assert "rem" in meta + + # Check that the rule type is mentioned in the metadata + rem_info = meta["rem"][0] + assert rule_type in rem_info[0] + + +@pytest.mark.parametrize( + "attribute_key,attribute_value,expected_value,rule_type", + [ + ("password", "my_password_123", "[Filtered]", "@password:filter"), + ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), + ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), + ], +) +def test_ourlog_extraction_default_pii_scrubbing_attributes( + mini_sentry, + relay, + items_consumer, + attribute_key, + attribute_value, + expected_value, + rule_type, +): + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:ourlogs-ingestion", + ] + project_config["config"].setdefault( + "datascrubbingSettings", + { + "scrubData": True, + "scrubDefaults": True, + "scrubIpAddresses": True, + }, + ) + + relay_instance = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_sentry_logs( + { + "timestamp": ts.timestamp(), + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": { + attribute_key: {"value": attribute_value, "type": "string"}, + }, + } + ) + + relay_instance.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + attributes = item["attributes"] + + assert attribute_key in attributes + assert attributes[attribute_key]["value"] == expected_value + assert "_meta" in item + meta = item["_meta"]["attributes"][attribute_key]["value"][""] + assert "rem" in meta + rem_info = meta["rem"] + assert len(rem_info) == 1 + assert rem_info[0][0] == rule_type + + +def test_ourlog_extraction_default_pii_scrubbing_does_not_scrub_default_attributes( + mini_sentry, + relay_with_processing, + items_consumer, +): + items_consumer = items_consumer() + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:ourlogs-ingestion", + ] + project_config["config"].setdefault( + "datascrubbingSettings", + { + "scrubData": True, + "scrubDefaults": True, + "scrubIpAddresses": True, + }, + ) + + # Testing the 'anything' filter as it's the most egregious with deep wildcards + project_config["config"]["piiConfig"] = { + "rules": { + "remove_custom_field": { + "type": "anything", + "redaction": {"method": "replace", "text": "[REDACTED]"}, + } + }, + "applications": {"**": ["remove_custom_field"]}, + } + + relay = relay_with_processing(options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_sentry_logs( + { + "timestamp": ts.timestamp(), + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": { + "custom_field": {"value": "custom_value", "type": "string"}, + }, + } + ) + + relay.send_envelope(project_id, envelope) + + item = items_consumer.get_item() + # Some of the following should not be redacted, but currently they are inside the `Attributes` object which + # treats every PII field as 'true' (default field), thus it's targetable by rules like '**'. + assert item == { + "attributes": { + "sentry._meta.fields.attributes.sentry.browser.version": { + "stringValue": '{"meta":{"value":{"":{"rem":[["remove_custom_field","s",0,10]],"len":4}}}}' + }, + "sentry._meta.fields.attributes.custom_field": { + "stringValue": '{"meta":{"value":{"":{"rem":[["remove_custom_field","s",0,10]],"len":12}}}}' + }, + "sentry.browser.version": {"stringValue": "[REDACTED]"}, + "sentry._meta.fields.attributes.sentry.observed_timestamp_nanos": { + "stringValue": '{"meta":{"value":{"":{"rem":[["remove_custom_field","s",0,10]],"len":19}}}}' + }, + "custom_field": {"stringValue": "[REDACTED]"}, + "sentry.body": {"stringValue": "[REDACTED]"}, + "sentry._meta.fields.attributes.sentry.browser.name": { + "stringValue": '{"meta":{"value":{"":{"rem":[["remove_custom_field","s",0,10]],"len":15}}}}' + }, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.observed_timestamp_nanos": {"stringValue": "[REDACTED]"}, + "sentry.span_id": {"stringValue": "eee19b7ec3c1b174"}, + "sentry.payload_size_bytes": mock.ANY, + "sentry.browser.name": {"stringValue": "[REDACTED]"}, + "sentry._meta.fields.body": { + "stringValue": '{"meta":{"":{"rem":[["remove_custom_field","s",0,10]],"len":8}}}' + }, + "sentry.timestamp_nanos": { + "stringValue": time_within_delta( + ts, + delta=timedelta(seconds=0), + expect_resolution="ns", + precision="us", + ) + }, + "sentry.timestamp_precise": { + "intValue": time_within_delta( + ts, + delta=timedelta(seconds=0), + expect_resolution="ns", + precision="us", + ) + }, + }, + "clientSampleRate": 1.0, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "organizationId": "1", + "projectId": "42", + "received": time_within_delta(), + "retentionDays": 90, + "serverSampleRate": 1.0, + "timestamp": time_within_delta( + ts, delta=timedelta(seconds=1), expect_resolution="ns" + ), + "traceId": "5b8efff798038103d269b633813fc60c", + } + + def test_ourlog_extraction_with_sentry_logs_with_missing_fields( mini_sentry, relay_with_processing, @@ -395,6 +648,14 @@ def test_ourlog_extraction_with_sentry_logs_with_missing_fields( project_config["config"]["features"] = [ "organizations:ourlogs-ingestion", ] + project_config["config"].setdefault( + "datascrubbingSettings", + { + "scrubData": True, + "scrubDefaults": True, + "scrubIpAddresses": True, + }, + ) relay = relay_with_processing(options=TEST_CONFIG) ts = datetime.now(timezone.utc)