From 1ade63cdd772ad6e969c0b8dc02dc7508f903d83 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Wed, 16 Jan 2019 20:54:20 +0100 Subject: [PATCH] feat(general): Implement advanced context normalization (#140) * feat(contexts): Add runtime build attribute * ref(general): Remove field trimming from contexts * feat(general): Implement os and runtime context normalization * feat(general): Normalize arbitrary context types to default --- general/src/protocol/contexts.rs | 25 +- general/src/store/normalize.rs | 13 +- general/src/store/normalize/contexts.rs | 318 ++++++++++++++++++++++++ 3 files changed, 338 insertions(+), 18 deletions(-) create mode 100644 general/src/store/normalize/contexts.rs diff --git a/general/src/protocol/contexts.rs b/general/src/protocol/contexts.rs index 1c3898d909..013fd82de0 100644 --- a/general/src/protocol/contexts.rs +++ b/general/src/protocol/contexts.rs @@ -66,26 +66,21 @@ pub struct DeviceContext { #[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, ToValue, ProcessValue)] pub struct OsContext { /// Name of the operating system. - #[metastructure(max_chars = "summary")] pub name: Annotated, /// Version of the operating system. - #[metastructure(max_chars = "summary")] pub version: Annotated, /// Internal build number of the operating system. - #[metastructure(max_chars = "summary")] - pub build: Annotated, + pub build: Annotated, /// Current kernel version. - #[metastructure(max_chars = "summary")] pub kernel_version: Annotated, /// Indicator if the OS is rooted (mobile mostly). pub rooted: Annotated, /// Unprocessed operating system info. - #[metastructure(max_chars = "summary")] pub raw_description: Annotated, /// Additional arbitrary fields for forwards compatibility. @@ -97,15 +92,15 @@ pub struct OsContext { #[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, ToValue, ProcessValue)] pub struct RuntimeContext { /// Runtime name. - #[metastructure(max_chars = "summary")] pub name: Annotated, - /// Runtime version. - #[metastructure(max_chars = "summary")] + /// Runtime version string. pub version: Annotated, + /// Application build string, if it is separate from the version. + pub build: Annotated, + /// Unprocessed runtime info. - #[metastructure(max_chars = "summary")] pub raw_description: Annotated, /// Additional arbitrary fields for forwards compatibility. @@ -120,12 +115,9 @@ pub struct AppContext { pub app_start_time: Annotated>, /// Device app hash (app specific device ID) - #[metastructure(pii = "true")] - #[metastructure(max_chars = "summary")] pub device_app_hash: Annotated, /// Build identicator. - #[metastructure(max_chars = "summary")] pub build_type: Annotated, /// App identifier (dotted bundle id). @@ -138,7 +130,6 @@ pub struct AppContext { pub app_version: Annotated, /// Internal build ID as it appears on the platform. - #[metastructure(max_chars = "summary")] pub app_build: Annotated, /// Additional arbitrary fields for forwards compatibility. @@ -150,11 +141,9 @@ pub struct AppContext { #[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, ToValue, ProcessValue)] pub struct BrowserContext { /// Runtime name. - #[metastructure(max_chars = "summary")] pub name: Annotated, /// Runtime version. - #[metastructure(max_chars = "summary")] pub version: Annotated, /// Additional arbitrary fields for forwards compatibility. @@ -288,7 +277,7 @@ fn test_os_context_roundtrip() { let context = Annotated::new(Context::Os(Box::new(OsContext { name: Annotated::new("iOS".to_string()), version: Annotated::new("11.4.2".to_string()), - build: Annotated::new("FEEDFACE".to_string()), + build: Annotated::new(LenientString("FEEDFACE".to_string())), kernel_version: Annotated::new("17.4.0".to_string()), rooted: Annotated::new(true), raw_description: Annotated::new("iOS 11.4.2 FEEDFACE (17.4.0)".to_string()), @@ -311,6 +300,7 @@ fn test_runtime_context_roundtrip() { let json = r#"{ "name": "rustc", "version": "1.27.0", + "build": "stable", "raw_description": "rustc 1.27.0 stable", "other": "value", "type": "runtime" @@ -318,6 +308,7 @@ fn test_runtime_context_roundtrip() { let context = Annotated::new(Context::Runtime(Box::new(RuntimeContext { name: Annotated::new("rustc".to_string()), version: Annotated::new("1.27.0".to_string()), + build: Annotated::new(LenientString("stable".to_string())), raw_description: Annotated::new("rustc 1.27.0 stable".to_string()), other: { let mut map = Object::new(); diff --git a/general/src/store/normalize.rs b/general/src/store/normalize.rs index 018d06feff..1dd7da0764 100644 --- a/general/src/store/normalize.rs +++ b/general/src/store/normalize.rs @@ -9,12 +9,13 @@ use uuid::Uuid; use crate::processor::{MaxChars, ProcessValue, ProcessingState, Processor}; use crate::protocol::{ - Breadcrumb, ClientSdkInfo, Event, EventId, EventType, Exception, Frame, IpAddr, Level, Request, + Breadcrumb, ClientSdkInfo, Context, Event, EventId, EventType, Exception, Frame, IpAddr, Level, Request, Stacktrace, Tags, User, }; use crate::store::{GeoIpLookup, StoreConfig}; use crate::types::{Annotated, Empty, Error, ErrorKind, Meta, Object, ValueAction}; +mod contexts; mod mechanism; mod request; mod stacktrace; @@ -455,6 +456,16 @@ impl<'a> Processor for NormalizeProcessor<'a> { ValueAction::Keep } + + fn process_context( + &mut self, + context: &mut Context, + _meta: &mut Meta, + _state: &ProcessingState<'_>, + ) -> ValueAction { + contexts::normalize_context(context); + ValueAction::Keep + } } #[cfg(test)] diff --git a/general/src/store/normalize/contexts.rs b/general/src/store/normalize/contexts.rs new file mode 100644 index 0000000000..65050582e7 --- /dev/null +++ b/general/src/store/normalize/contexts.rs @@ -0,0 +1,318 @@ +use regex::Regex; + +use crate::protocol::{Context, OsContext, RuntimeContext}; +use crate::types::{Object, Value}; + +lazy_static::lazy_static! { + /// Environment.OSVersion (GetVersionEx) or RuntimeInformation.OSDescription on Windows + static ref OS_WINDOWS_REGEX: Regex = Regex::new(r#"^(Microsoft )?Windows (NT )?(?P\d+\.\d+\.\d+).*$"#).unwrap(); + + /// Environment.OSVersion or RuntimeInformation.OSDescription (uname) on Mono and CoreCLR on + /// macOS, iOS, Linux, etc. + static ref OS_UNAME_REGEX: Regex = Regex::new(r#"^(?P[a-zA-Z]+) (?P\d+\.\d+\.\d+(\.[1-9]+)?).*$"#).unwrap(); + + /// Mono 5.4, .NET Core 2.0 + static ref RUNTIME_DOTNET_REGEX: Regex = Regex::new(r#"^(?P.*) (?P\d+\.\d+(\.\d+){0,2}).*$"#).unwrap(); +} + +fn normalize_runtime_context(runtime: &mut RuntimeContext) { + if runtime.name.value().is_none() && runtime.version.value().is_none() { + if let Some(raw_description) = runtime.raw_description.as_str() { + if let Some(captures) = RUNTIME_DOTNET_REGEX.captures(raw_description) { + runtime.name = captures.name("name").map(|m| m.as_str().to_string()).into(); + runtime.version = captures + .name("version") + .map(|m| m.as_str().to_string()) + .into(); + } + } + } + + // RuntimeInformation.FrameworkDescription doesn't return a very useful value. + // Example: ".NET Framework 4.7.3056.0" + // Use release keys from registry sent as #build + if let Some(name) = runtime.name.as_str() { + if let Some(build) = runtime.build.as_str() { + if name.starts_with(".NET Framework") { + let version = match build { + "378389" => Some("4.5".to_string()), + "378675" => Some("4.5.1".to_string()), + "378758" => Some("4.5.1".to_string()), + "379893" => Some("4.5.2".to_string()), + "393295" => Some("4.6".to_string()), + "393297" => Some("4.6".to_string()), + "394254" => Some("4.6.1".to_string()), + "394271" => Some("4.6.1".to_string()), + "394802" => Some("4.6.2".to_string()), + "394806" => Some("4.6.2".to_string()), + "460798" => Some("4.7".to_string()), + "460805" => Some("4.7".to_string()), + "461308" => Some("4.7.1".to_string()), + "461310" => Some("4.7.1".to_string()), + "461808" => Some("4.7.2".to_string()), + "461814" => Some("4.7.2".to_string()), + _ => None, + }; + + if let Some(version) = version { + runtime.version = version.into(); + } + } + } + } +} + +fn normalize_os_context(os: &mut OsContext) { + if os.name.value().is_some() || os.version.value().is_some() { + return; + } + + if let Some(raw_description) = os.raw_description.as_str() { + if let Some(captures) = OS_WINDOWS_REGEX.captures(raw_description) { + os.name = "Windows".to_string().into(); + os.version = captures + .name("version") + .map(|m| m.as_str().to_string()) + .into(); + } else if let Some(captures) = OS_UNAME_REGEX.captures(raw_description) { + os.name = captures.name("name").map(|m| m.as_str().to_string()).into(); + os.kernel_version = captures + .name("version") + .map(|m| m.as_str().to_string()) + .into(); + } + } +} + +fn normalize_other_context(other: &mut Object) { + if let Some(ty) = other.get_mut("type") { + if ty.as_str() != Some("default") { + ty.set_value(Some("default".to_string().into())); + } + } +} + +pub fn normalize_context(context: &mut Context) { + match context { + Context::Runtime(runtime) => normalize_runtime_context(runtime), + Context::Os(os) => normalize_os_context(os), + Context::Other(other) => normalize_other_context(other), + _ => (), + } +} + +#[cfg(test)] +use crate::{protocol::LenientString, types::Annotated}; + +#[test] +fn test_arbitrary_type() { + let mut other = Object::new(); + other.insert( + "type".to_string(), + Annotated::from(Value::String("foo".to_string())), + ); + + normalize_other_context(&mut other); + assert_eq_dbg!(other.get("type").and_then(|a| a.as_str()), Some("default")); +} + +#[test] +fn test_dotnet_framework_472() { + let mut runtime = RuntimeContext { + raw_description: ".NET Framework 4.7.3056.0".to_string().into(), + build: LenientString("461814".to_string()).into(), + ..RuntimeContext::default() + }; + + normalize_runtime_context(&mut runtime); + assert_eq_dbg!(Some(".NET Framework"), runtime.name.as_str()); + assert_eq_dbg!(Some("4.7.2"), runtime.version.as_str()); +} + +#[test] +fn test_dotnet_framework_future_version() { + let mut runtime = RuntimeContext { + raw_description: ".NET Framework 200.0".to_string().into(), + build: LenientString("999999".to_string()).into(), + ..RuntimeContext::default() + }; + + // Unmapped build number doesn't override version + normalize_runtime_context(&mut runtime); + assert_eq_dbg!(Some(".NET Framework"), runtime.name.as_str()); + assert_eq_dbg!(Some("200.0"), runtime.version.as_str()); +} + +#[test] +fn test_dotnet_native() { + let mut runtime = RuntimeContext { + raw_description: ".NET Native 2.0".to_string().into(), + ..RuntimeContext::default() + }; + + normalize_runtime_context(&mut runtime); + assert_eq_dbg!(Some(".NET Native"), runtime.name.as_str()); + assert_eq_dbg!(Some("2.0"), runtime.version.as_str()); +} + +#[test] +fn test_dotnet_core() { + let mut runtime = RuntimeContext { + raw_description: ".NET Core 2.0".to_string().into(), + ..RuntimeContext::default() + }; + + normalize_runtime_context(&mut runtime); + assert_eq_dbg!(Some(".NET Core"), runtime.name.as_str()); + assert_eq_dbg!(Some("2.0"), runtime.version.as_str()); +} + +#[test] +fn test_windows_7_or_server_2008() { + // Environment.OSVersion on Windows 7 (CoreCLR 1.0+, .NET Framework 1.1+, Mono 1+) + let mut os = OsContext { + raw_description: "Microsoft Windows NT 6.1.7601 Service Pack 1" + .to_string() + .into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Windows"), os.name.as_str()); + assert_eq_dbg!(Some("6.1.7601"), os.version.as_str()); +} + +#[test] +fn test_windows_8_or_server_2012_or_later() { + // Environment.OSVersion on Windows 10 (CoreCLR 1.0+, .NET Framework 1.1+, Mono 1+) + // *or later, due to GetVersionEx deprecated on Windows 8.1 + // It's a potentially really misleading API on newer platforms + // Only used if RuntimeInformation.OSDescription is not available (old runtimes) + let mut os = OsContext { + raw_description: "Microsoft Windows NT 6.2.9200.0".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Windows"), os.name.as_str()); + assert_eq_dbg!(Some("6.2.9200"), os.version.as_str()); +} + +#[test] +fn test_windows_10() { + // RuntimeInformation.OSDescription on Windows 10 (CoreCLR 2.0+, .NET + // Framework 4.7.1+, Mono 5.4+) + let mut os = OsContext { + raw_description: "Microsoft Windows 10.0.16299".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Windows"), os.name.as_str()); + assert_eq_dbg!(Some("10.0.16299"), os.version.as_str()); +} + +#[test] +fn test_macos_os_version() { + // Environment.OSVersion on macOS (CoreCLR 1.0+, Mono 1+) + let mut os = OsContext { + raw_description: "Unix 17.5.0.0".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Unix"), os.name.as_str()); + assert_eq_dbg!(Some("17.5.0"), os.kernel_version.as_str()); +} + +#[test] +fn test_macos_runtime() { + // RuntimeInformation.OSDescription on macOS (CoreCLR 2.0+, Mono 5.4+) + let mut os = OsContext { + raw_description: "Darwin 17.5.0 Darwin Kernel Version 17.5.0: Mon Mar 5 22:24:32 PST 2018; root:xnu-4570.51.1~1/RELEASE_X86_64".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Darwin"), os.name.as_str()); + assert_eq_dbg!(Some("17.5.0"), os.kernel_version.as_str()); +} + +#[test] +fn test_centos_os_version() { + // Environment.OSVersion on CentOS 7 (CoreCLR 1.0+, Mono 1+) + let mut os = OsContext { + raw_description: "Unix 3.10.0.693".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Unix"), os.name.as_str()); + assert_eq_dbg!(Some("3.10.0.693"), os.kernel_version.as_str()); +} + +#[test] +fn test_centos_runtime_info() { + // RuntimeInformation.OSDescription on CentOS 7 (CoreCLR 2.0+, Mono 5.4+) + let mut os = OsContext { + raw_description: "Linux 3.10.0-693.21.1.el7.x86_64 #1 SMP Wed Mar 7 19:03:37 UTC 2018" + .to_string() + .into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Linux"), os.name.as_str()); + assert_eq_dbg!(Some("3.10.0"), os.kernel_version.as_str()); +} + +#[test] +fn test_wsl_ubuntu() { + // RuntimeInformation.OSDescription on Windows Subsystem for Linux (Ubuntu) + // (CoreCLR 2.0+, Mono 5.4+) + let mut os = OsContext { + raw_description: "Linux 4.4.0-43-Microsoft #1-Microsoft Wed Dec 31 14:42:53 PST 2014" + .to_string() + .into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Linux"), os.name.as_str()); + assert_eq_dbg!(Some("4.4.0"), os.kernel_version.as_str()); +} + +#[test] +fn test_name_not_overwritten() { + let mut os = OsContext { + name: "Properly defined name".to_string().into(), + raw_description: "Linux 4.4.0".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Properly defined name"), os.name.as_str()); +} + +#[test] +fn test_version_not_overwritten() { + let mut os = OsContext { + version: "Properly defined version".to_string().into(), + raw_description: "Linux 4.4.0".to_string().into(), + ..OsContext::default() + }; + + normalize_os_context(&mut os); + assert_eq_dbg!(Some("Properly defined version"), os.version.as_str()); +} + +#[test] +fn test_no_name() { + let mut os = OsContext::default(); + + normalize_os_context(&mut os); + assert_eq_dbg!(None, os.name.value()); + assert_eq_dbg!(None, os.version.value()); + assert_eq_dbg!(None, os.kernel_version.value()); + assert_eq_dbg!(None, os.raw_description.value()); +}