Skip to content

feat(pii): Implement smart PII scrubbing for logentry.formatted #4985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Aug 4, 2025
Merged

feat(pii): Implement smart PII scrubbing for logentry.formatted #4985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
339c49b
Implement smart PII scrubbing for logentry.formatted
olksdr Jul 28, 2025
2dd79fc
Add changelog entry
olksdr Jul 28, 2025
bb86615
Add Bearer token scrubbing.
olksdr Jul 28, 2025
1523d12
simplify after review
olksdr Jul 28, 2025
8c06ae9
Added integration test
olksdr Jul 28, 2025
fd886be
tests
olksdr Jul 28, 2025
1f0656e
Use rulesets and specs to remove PII when scrubbing enabled
olksdr Jul 29, 2025
e8bbe64
Rename the rule
olksdr Jul 29, 2025
ae6f30b
Merge branch 'master' into fix/pii-logentry-formatted
olksdr Jul 29, 2025
6c129a3
Add proper builtin selector for bearer token
olksdr Jul 29, 2025
106cf34
Add test to check if the common filters apply to the logentry.formatted
olksdr Jul 29, 2025
85b1026
Scrub bearer in values only
olksdr Jul 29, 2025
219e225
Merge remote-tracking branch 'origin/master' into fix/pii-logentry-fo…
olksdr Jul 29, 2025
20cfec5
Move py tests to parametrized
olksdr Jul 30, 2025
2971d98
review comments: inlined snapshots and changelog
olksdr Jul 30, 2025
4822fce
Merge remote-tracking branch 'origin/master' into fix/pii-logentry-fo…
olksdr Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

- Add `unhandled` status type for Release Health `session` and `sessions` envelopes. ([#4939](https://github.com/getsentry/relay/pull/4939))
- Always emit a span usage metric, independent of span feature flags. ([#4976](https://github.com/getsentry/relay/pull/4976))
- Improve PII scrubbing for `logentry.formatted` by ensuring only sensitive data is redacted, rather than replacing the entire field value. ([#4985](https://github.com/getsentry/relay/pull/4985))

**Bug Fixes**:

Expand Down
29 changes: 29 additions & 0 deletions relay-pii/src/builtin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ declare_builtin_rules! {
"@pemkey".into(),
"@urlauth".into(),
"@userpath".into(),
"@bearer".into(),
"@password".into(),
"@usssn".into(),
],
Expand All @@ -59,6 +60,7 @@ declare_builtin_rules! {
"@pemkey:filter".into(),
"@urlauth:legacy".into(),
"@userpath:filter".into(),
"@bearer:filter".into(),
"@password:filter".into(),
"@usssn:filter".into(),
],
Expand Down Expand Up @@ -90,6 +92,33 @@ declare_builtin_rules! {
redaction: Redaction::Replace(ReplaceRedaction::default()),
};

// bearer
"@bearer" => rule_alias!("@bearer:replace");
"@bearer:replace" => RuleSpec {
ty: RuleType::Bearer,
redaction: Redaction::Replace(ReplaceRedaction {
text: "Bearer [token]".into(),
}),
};
"@bearer:filter" => RuleSpec {
ty: RuleType::Bearer,
redaction: Redaction::Replace(ReplaceRedaction {
text: "[Filtered]".into(),
}),
};
"@bearer:hash" => RuleSpec {
ty: RuleType::Bearer,
redaction: Redaction::Hash,
};
"@bearer:mask" => RuleSpec {
ty: RuleType::Bearer,
redaction: Redaction::Mask,
};
"@bearer:remove" => RuleSpec {
ty: RuleType::Bearer,
redaction: Redaction::Remove,
};

// ip rules
"@ip" => rule_alias!("@ip:replace");
"@ip:replace" => RuleSpec {
Expand Down
1 change: 1 addition & 0 deletions relay-pii/src/compiledconfig.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ impl CompiledPiiConfig {
| RuleType::Pemkey
| RuleType::UrlAuth
| RuleType::UsSsn
| RuleType::Bearer
| RuleType::Password
| RuleType::Multiple(_)
| RuleType::Alias(_)
Expand Down
2 changes: 2 additions & 0 deletions relay-pii/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ pub enum RuleType {
UrlAuth,
/// US SSN.
UsSsn,
/// A Bearer token
Bearer,
/// Keys that look like passwords
Password,
/// When a regex matches a key, a value is removed
Expand Down
76 changes: 64 additions & 12 deletions relay-pii/src/convert.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,23 @@ pub fn to_pii_config(
SENSITIVE_COOKIES.clone(),
vec!["@anything:filter".to_owned()],
);

let logentry_selector: SelectorSpec = SelectorSpec::Path(vec![
SelectorPathItem::Type(ValueType::LogEntry),
SelectorPathItem::Key("formatted".to_owned()),
]);

// Apply smart scrubbing rules only to logentry.formatted
applications.insert(
logentry_selector,
vec![
"@email:replace".to_owned(),
"@creditcard:replace".to_owned(),
"@iban:replace".to_owned(),
"@usssn:replace".to_owned(),
"@bearer:replace".to_owned(),
],
);
}

if datascrubbing_config.scrub_ip_addresses {
Expand Down Expand Up @@ -282,7 +299,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv

#[test]
fn test_convert_default_pii_config() {
insta::assert_json_snapshot!(simple_enabled_pii_config(), @r###"
insta::assert_json_snapshot!(simple_enabled_pii_config(), @r#"
{
"applications": {
"($string || $number || $array || $object) && !(debug_meta.** || $frame.filename || $frame.abs_path || $logentry.formatted || $error.value || $http.headers.user-agent)": [
Expand All @@ -294,10 +311,17 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
],
"*.cookies.sentrysid || *.cookies.sudo || *.cookies.su || *.cookies.session || *.cookies.__session || *.cookies.sessionid || *.cookies.user_session || *.cookies.symfony || *.cookies.phpsessid || *.cookies.fasthttpsessionid || *.cookies.mysession || *.cookies.irissessionid || *.cookies.csrf || *.cookies.xsrf || *.cookies._xsrf || *.cookies._csrf || *.cookies.csrf-token || *.cookies.csrf_token || *.cookies.xsrf-token || *.cookies.xsrf_token || *.cookies.fastcsrf || *.cookies._iris_csrf": [
"@anything:filter"
],
"$logentry.formatted": [
"@email:replace",
"@creditcard:replace",
"@iban:replace",
"@usssn:replace",
"@bearer:replace"
]
}
}
"###);
"#);
}

#[test]
Expand All @@ -307,7 +331,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
..simple_enabled_config()
});

insta::assert_json_snapshot!(pii_config, @r###"
insta::assert_json_snapshot!(pii_config, @r#"
Copy link
Member

@Dav1dde Dav1dde Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think ### is the 'new' syntax insta wants and cargo insta --force-update-snapshots would change it back.

Copy link
Contributor Author

@olksdr olksdr Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I always thought it's another way around, but I will run it and see how changes: 👍

Copy link
Contributor Author

@olksdr olksdr Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm, it seems like I do not have this option --force-update-snapshots in my local installation

Copy link
Contributor Author

@olksdr olksdr Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, found 😄

Copy link
Contributor Author

@olksdr olksdr Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So it seems like i was right and running latest version of insta with cargo insta test --force-update-snapshots --all-features --workspace

cargo-insta 1.43.1

changes all the tests to single # or remove it all together when needed.

@Dav1dde should be do it in separate PR to match the latest insta and , make that change separate from this PR ?

Copy link
Member

@Dav1dde Dav1dde Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's strange, when I ran it, it did the opposite: https://github.com/getsentry/relay/pull/4908/files

Copy link
Member

@Dav1dde Dav1dde Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I am on an older version and they changed behavior in 1.41 -_-

yeah let's keep the insta stuff separate 👍

{
"applications": {
"($string || $number || $array || $object) && !(debug_meta.** || $frame.filename || $frame.abs_path || $logentry.formatted || $error.value || $http.headers.user-agent)": [
Expand All @@ -319,10 +343,17 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
],
"*.cookies.sentrysid || *.cookies.sudo || *.cookies.su || *.cookies.session || *.cookies.__session || *.cookies.sessionid || *.cookies.user_session || *.cookies.symfony || *.cookies.phpsessid || *.cookies.fasthttpsessionid || *.cookies.mysession || *.cookies.irissessionid || *.cookies.csrf || *.cookies.xsrf || *.cookies._xsrf || *.cookies._csrf || *.cookies.csrf-token || *.cookies.csrf_token || *.cookies.xsrf-token || *.cookies.xsrf_token || *.cookies.fastcsrf || *.cookies._iris_csrf": [
"@anything:filter"
],
"$logentry.formatted": [
"@email:replace",
"@creditcard:replace",
"@iban:replace",
"@usssn:replace",
"@bearer:replace"
]
}
}
"###);
"#);
}

#[test]
Expand All @@ -332,7 +363,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
..simple_enabled_config()
});

insta::assert_json_snapshot!(pii_config, @r###"
insta::assert_json_snapshot!(pii_config, @r#"
{
"rules": {
"strip-fields": {
Expand All @@ -355,10 +386,17 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
],
"*.cookies.sentrysid || *.cookies.sudo || *.cookies.su || *.cookies.session || *.cookies.__session || *.cookies.sessionid || *.cookies.user_session || *.cookies.symfony || *.cookies.phpsessid || *.cookies.fasthttpsessionid || *.cookies.mysession || *.cookies.irissessionid || *.cookies.csrf || *.cookies.xsrf || *.cookies._xsrf || *.cookies._csrf || *.cookies.csrf-token || *.cookies.csrf_token || *.cookies.xsrf-token || *.cookies.xsrf_token || *.cookies.fastcsrf || *.cookies._iris_csrf": [
"@anything:filter"
],
"$logentry.formatted": [
"@email:replace",
"@creditcard:replace",
"@iban:replace",
"@usssn:replace",
"@bearer:replace"
]
}
}
"###);
"#);
}

#[test]
Expand All @@ -368,7 +406,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
..simple_enabled_config()
});

insta::assert_json_snapshot!(pii_config, @r###"
insta::assert_json_snapshot!(pii_config, @r#"
{
"applications": {
"($string || $number || $array || $object) && !(debug_meta.** || $frame.filename || $frame.abs_path || $logentry.formatted || $error.value || $http.headers.user-agent) && !foobar": [
Expand All @@ -380,10 +418,17 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
],
"*.cookies.sentrysid || *.cookies.sudo || *.cookies.su || *.cookies.session || *.cookies.__session || *.cookies.sessionid || *.cookies.user_session || *.cookies.symfony || *.cookies.phpsessid || *.cookies.fasthttpsessionid || *.cookies.mysession || *.cookies.irissessionid || *.cookies.csrf || *.cookies.xsrf || *.cookies._xsrf || *.cookies._csrf || *.cookies.csrf-token || *.cookies.csrf_token || *.cookies.xsrf-token || *.cookies.xsrf_token || *.cookies.fastcsrf || *.cookies._iris_csrf": [
"@anything:filter"
],
"$logentry.formatted": [
"@email:replace",
"@creditcard:replace",
"@iban:replace",
"@usssn:replace",
"@bearer:replace"
]
}
}
"###);
"#);
}

#[test]
Expand All @@ -395,7 +440,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
..Default::default()
});

insta::assert_json_snapshot!(pii_config, @r###"
insta::assert_json_snapshot!(pii_config, @r#"
{
"applications": {
"($string || $number || $array || $object) && !(debug_meta.** || $frame.filename || $frame.abs_path || $logentry.formatted || $error.value || $http.headers.user-agent)": [
Expand All @@ -406,7 +451,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
]
}
}
"###);
"#);
}

#[test]
Expand Down Expand Up @@ -1271,7 +1316,7 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
..simple_enabled_config()
});

insta::assert_json_snapshot!(pii_config, @r###"
insta::assert_json_snapshot!(pii_config, @r#"
{
"rules": {
"strip-fields": {
Expand All @@ -1294,10 +1339,17 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv
],
"*.cookies.sentrysid || *.cookies.sudo || *.cookies.su || *.cookies.session || *.cookies.__session || *.cookies.sessionid || *.cookies.user_session || *.cookies.symfony || *.cookies.phpsessid || *.cookies.fasthttpsessionid || *.cookies.mysession || *.cookies.irissessionid || *.cookies.csrf || *.cookies.xsrf || *.cookies._xsrf || *.cookies._csrf || *.cookies.csrf-token || *.cookies.csrf_token || *.cookies.xsrf-token || *.cookies.xsrf_token || *.cookies.fastcsrf || *.cookies._iris_csrf": [
"@anything:filter"
],
"$logentry.formatted": [
"@email:replace",
"@creditcard:replace",
"@iban:replace",
"@usssn:replace",
"@bearer:replace"
]
}
}
"###);
"#);

let pii_config = pii_config.unwrap();
let mut pii_processor = PiiProcessor::new(pii_config.compiled());
Expand Down
Loading
Loading