From bbc69bf8fcf55273328e0e407892494cb3cb3758 Mon Sep 17 00:00:00 2001 From: Paul Marvin Date: Wed, 19 Dec 2018 15:03:08 -0600 Subject: [PATCH] * Implement a safe serializer --- Src/StackifyLib/Config.cs | 32 ++- Src/StackifyLib/Internal/Logs/LogClient.cs | 7 +- .../Serialization/JsonDotNetSerializer.cs | 58 ++++ .../Serialization/SafeJsonTextReader.cs | 49 ++++ .../Serialization/SafeJsonTextWriter.cs | 46 ++++ .../ShouldDeserializeContractResolver.cs | 28 ++ .../ShouldSerializerConstractResolver.cs | 28 ++ Src/StackifyLib/Utils/HelperFunctions.cs | 26 +- .../JsonDotNetSerializer_Tests.cs | 249 ++++++++++++++++++ .../JsonSerialization_Tests.cs | 12 +- 10 files changed, 509 insertions(+), 26 deletions(-) create mode 100644 Src/StackifyLib/Internal/Serialization/JsonDotNetSerializer.cs create mode 100644 Src/StackifyLib/Internal/Serialization/SafeJsonTextReader.cs create mode 100644 Src/StackifyLib/Internal/Serialization/SafeJsonTextWriter.cs create mode 100644 Src/StackifyLib/Internal/Serialization/ShouldDeserializeContractResolver.cs create mode 100644 Src/StackifyLib/Internal/Serialization/ShouldSerializerConstractResolver.cs create mode 100644 test/StackifyLib.UnitTests/JsonDotNetSerializer_Tests.cs diff --git a/Src/StackifyLib/Config.cs b/Src/StackifyLib/Config.cs index 43d7fac..69d0405 100644 --- a/Src/StackifyLib/Config.cs +++ b/Src/StackifyLib/Config.cs @@ -102,12 +102,30 @@ public static void LoadSettings() ApiLog = apiLog.Equals(bool.TrueString, StringComparison.CurrentCultureIgnoreCase); } - var loggingJsonMaxFields = Get("Stackify.Logging.JsonMaxFields", "50"); - if (string.IsNullOrWhiteSpace(loggingJsonMaxFields) == false) + var loggingMaxDepth = Get("Stackify.Logging.MaxDepth", "5"); + if (string.IsNullOrWhiteSpace(loggingMaxDepth) == false) { - if (int.TryParse(loggingJsonMaxFields, out int maxFields) && maxFields > 0 && maxFields < 100) + if (int.TryParse(loggingMaxDepth, out var maxDepth) && maxDepth > 0 && maxDepth < 10) { - LoggingJsonMaxFields = maxFields; + LoggingMaxDepth = maxDepth; + } + } + + var loggingMaxFields = Get("Stackify.Logging.MaxFields", "50"); + if (string.IsNullOrWhiteSpace(loggingMaxFields) == false) + { + if (int.TryParse(loggingMaxFields, out var maxFields) && maxFields > 0 && maxFields < 100) + { + LoggingMaxFields = maxFields; + } + } + + var loggingMaxStrLength = Get("Stackify.Logging.MaxStringLength", "32766"); + if (string.IsNullOrWhiteSpace(loggingMaxStrLength) == false) + { + if (int.TryParse(loggingMaxStrLength, out var maxStrLen) && maxStrLen > 0 && maxStrLen < 32766) + { + LoggingMaxStringLength = maxStrLen; } } } @@ -150,7 +168,11 @@ public static void LoadSettings() public static bool? ApiLog { get; set; } - public static int LoggingJsonMaxFields { get; set; } = 50; + public static int LoggingMaxDepth { get; set; } = 5; + + public static int LoggingMaxFields { get; set; } = 50; + + public static int LoggingMaxStringLength { get; set; } = 32766; /// diff --git a/Src/StackifyLib/Internal/Logs/LogClient.cs b/Src/StackifyLib/Internal/Logs/LogClient.cs index e8b69cf..60961d6 100644 --- a/Src/StackifyLib/Internal/Logs/LogClient.cs +++ b/Src/StackifyLib/Internal/Logs/LogClient.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net; using System.Threading; +using StackifyLib.Internal.Serialization; namespace StackifyLib.Internal.Logs { @@ -315,11 +316,11 @@ internal HttpClient.StackifyWebResponse SendLogsByGroups(LogMsg[] messages) var groups = SplitLogsToGroups(messages); - string jsonData = JsonConvert.SerializeObject(groups, new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore }); + var jdn = new JsonDotNetSerializer(new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, NullValueHandling = NullValueHandling.Ignore }); + var jsonData = jdn.SafeSerializeObject(groups); - string urlToUse = (_HttpClient.BaseAPIUrl) + "Log/SaveMultipleGroups"; - + var urlToUse = (_HttpClient.BaseAPIUrl) + "Log/SaveMultipleGroups"; if (!_ServicePointSet) { diff --git a/Src/StackifyLib/Internal/Serialization/JsonDotNetSerializer.cs b/Src/StackifyLib/Internal/Serialization/JsonDotNetSerializer.cs new file mode 100644 index 0000000..df800cf --- /dev/null +++ b/Src/StackifyLib/Internal/Serialization/JsonDotNetSerializer.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace StackifyLib.Internal.Serialization +{ + public class JsonDotNetSerializer + { + private readonly JsonSerializerSettings _settings; + private readonly int _maxDepth; + private readonly int _maxFields; + private readonly int _maxStringLength; + + + public JsonDotNetSerializer(JsonSerializerSettings settings = null, int? maxDepth = null, int? maxFields = null, int? maxStringLength = null) + { + _settings = settings; + + _maxDepth = maxDepth ?? Config.LoggingMaxDepth; + _maxFields = maxFields ?? Config.LoggingMaxFields; + _maxStringLength = maxStringLength ?? Config.LoggingMaxStringLength; + } + + + /// + /// Serialize an object to JSON with limitations on MaxDepth and MaxStringLength as provided in the ctor. + /// + /// Note, Serialization does not currently consider MaxFields. + /// + /// + /// + public string SafeSerializeObject(object obj) + { + string r; + + using (var writer = new StringWriter()) + { + using (var jsonWriter = new SafeJsonTextWriter(writer, _maxStringLength)) + { + // ReSharper disable AccessToDisposedClosure + bool IncludeDepth() => jsonWriter.CurrentDepth <= _maxDepth; + // ReSharper restore AccessToDisposedClosure + + var resolver = new ShouldSerializeContractResolver(IncludeDepth); + var serializer = JsonSerializer.CreateDefault(_settings); + serializer.ContractResolver = resolver; + + serializer.Serialize(jsonWriter, obj); + r = writer.ToString(); + } + } + + return r; + } + } +} diff --git a/Src/StackifyLib/Internal/Serialization/SafeJsonTextReader.cs b/Src/StackifyLib/Internal/Serialization/SafeJsonTextReader.cs new file mode 100644 index 0000000..b510177 --- /dev/null +++ b/Src/StackifyLib/Internal/Serialization/SafeJsonTextReader.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace StackifyLib.Internal.Serialization +{ + internal class SafeJsonTextReader : JsonTextReader + { + private readonly int _maxStringLength; + + public int CurrentDepth { get; private set; } + public decimal CurrentFields { get; private set; } = 1; + + + public SafeJsonTextReader(TextReader reader, int maxStringLength) : base(reader) + { + _maxStringLength = maxStringLength; + } + + + public override bool Read() + { + CurrentDepth = Path.Count(c => c == '.') + 1; + CurrentFields += .5M; + + return base.Read(); + } + + public override string ReadAsString() + { + var value = base.ReadAsString(); + + if (value != null) + { + if (value.Length > _maxStringLength) + { + value = value.Substring(0, _maxStringLength); + } + } + + return value; + } + } +} diff --git a/Src/StackifyLib/Internal/Serialization/SafeJsonTextWriter.cs b/Src/StackifyLib/Internal/Serialization/SafeJsonTextWriter.cs new file mode 100644 index 0000000..d7b5151 --- /dev/null +++ b/Src/StackifyLib/Internal/Serialization/SafeJsonTextWriter.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace StackifyLib.Internal.Serialization +{ + internal class SafeJsonTextWriter : JsonTextWriter + { + private readonly int _maxStringLength; + + public int CurrentDepth { get; private set; } + + + public SafeJsonTextWriter(TextWriter writer, int maxStringLength) : base(writer) + { + _maxStringLength = maxStringLength; + } + + + public override void WriteStartObject() + { + CurrentDepth++; + + base.WriteStartObject(); + } + + public override void WriteEndObject() + { + CurrentDepth--; + + base.WriteEndObject(); + } + + public override void WriteValue(string value) + { + if (value.Length > _maxStringLength) + { + value = value.Substring(0, _maxStringLength); + } + + base.WriteValue(value); + } + } +} diff --git a/Src/StackifyLib/Internal/Serialization/ShouldDeserializeContractResolver.cs b/Src/StackifyLib/Internal/Serialization/ShouldDeserializeContractResolver.cs new file mode 100644 index 0000000..36c88e2 --- /dev/null +++ b/Src/StackifyLib/Internal/Serialization/ShouldDeserializeContractResolver.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace StackifyLib.Internal.Serialization +{ + internal class ShouldDeserializeContractResolver : DefaultContractResolver + { + private readonly Func _include; + + public ShouldDeserializeContractResolver(Func include) + { + _include = include; + } + + protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) + { + var property = base.CreateProperty(member, memberSerialization); + + property.ShouldDeserialize = obj => _include(); + + return property; + } + } +} diff --git a/Src/StackifyLib/Internal/Serialization/ShouldSerializerConstractResolver.cs b/Src/StackifyLib/Internal/Serialization/ShouldSerializerConstractResolver.cs new file mode 100644 index 0000000..6256e52 --- /dev/null +++ b/Src/StackifyLib/Internal/Serialization/ShouldSerializerConstractResolver.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace StackifyLib.Internal.Serialization +{ + internal class ShouldSerializeContractResolver : DefaultContractResolver + { + private readonly Func _include; + + public ShouldSerializeContractResolver(Func include) + { + _include = include; + } + + protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) + { + var property = base.CreateProperty(member, memberSerialization); + + property.ShouldSerialize = obj => _include(); + + return property; + } + } +} diff --git a/Src/StackifyLib/Utils/HelperFunctions.cs b/Src/StackifyLib/Utils/HelperFunctions.cs index 0aac22b..7ad7f0f 100644 --- a/Src/StackifyLib/Utils/HelperFunctions.cs +++ b/Src/StackifyLib/Utils/HelperFunctions.cs @@ -1,19 +1,20 @@ -using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Text; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using StackifyLib.Internal.Serialization; namespace StackifyLib.Utils { public class HelperFunctions { - static List _BadTypes = new List() { "log4net.Util.SystemStringFormat", "System.Object[]" }; - static JsonSerializer serializer = new JsonSerializer { ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; - static JsonSerializerSettings serializerSettings = new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore, ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; + private static readonly List BadTypes = new List { "log4net.Util.SystemStringFormat", "System.Object[]" }; + private static readonly JsonSerializer Serializer = new JsonSerializer { ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; + private static readonly JsonSerializerSettings SerializerSettings = new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, NullValueHandling = NullValueHandling.Ignore, ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; /// /// Trying to serialize something that the user passed in. Sometimes this is used to serialize what we know is additional debug and sometimes it is the primary logged item. This is why the serializeSimpleTypes exists. For additional debug stuff we always serialize it. For the primary logged object we won't because it doesn't make any sense to put a string in the json as well as the main message. It's meant for objects. @@ -68,9 +69,9 @@ public static string SerializeDebugData(object logObject, bool serializeSimpleTy { } - else if (!_BadTypes.Contains(t.ToString())) + else if (!BadTypes.Contains(t.ToString())) { - var token = JToken.FromObject(logObject, serializer); + var token = JToken.FromObject(logObject, Serializer); if (token is JObject) { @@ -92,7 +93,7 @@ public static string SerializeDebugData(object logObject, bool serializeSimpleTy if (type.IsArray) { - var array = (Array) logObject; + var array = (Array)logObject; if (array.Length > 0) { @@ -160,9 +161,9 @@ public static string SerializeDebugData(object logObject, bool serializeSimpleTy } catch (Exception ex) { - lock (_BadTypes) + lock (BadTypes) { - _BadTypes.Add(t.ToString()); + BadTypes.Add(t.ToString()); } Utils.StackifyAPILogger.Log(ex.ToString()); } @@ -187,7 +188,7 @@ public static string SerializeDebugData(object logObject, bool serializeSimpleTy } else { - props.Add(prop.Key, JObject.FromObject(prop.Value, serializer)); + props.Add(prop.Key, JObject.FromObject(prop.Value, Serializer)); } } @@ -204,10 +205,11 @@ public static string SerializeDebugData(object logObject, bool serializeSimpleTy if (jObject != null) { + jObject = GetPrunedObject(jObject, Config.LoggingMaxFields); - jObject = GetPrunedObject(jObject, Config.LoggingJsonMaxFields); + var jdn = new JsonDotNetSerializer(SerializerSettings); - return JsonConvert.SerializeObject(jObject, serializerSettings); + return jdn.SafeSerializeObject(jObject); } return null; diff --git a/test/StackifyLib.UnitTests/JsonDotNetSerializer_Tests.cs b/test/StackifyLib.UnitTests/JsonDotNetSerializer_Tests.cs new file mode 100644 index 0000000..98d1ae3 --- /dev/null +++ b/test/StackifyLib.UnitTests/JsonDotNetSerializer_Tests.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using FluentAssertions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using StackifyLib.Internal.Serialization; +using Xunit; +using Xunit.Abstractions; + +namespace StackifyLib.UnitTests +{ + public class JsonDotNetSerializer_Tests + { + private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, NullValueHandling = NullValueHandling.Ignore, ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; + + private readonly ITestOutputHelper _output; + + public JsonDotNetSerializer_Tests(ITestOutputHelper output) + { + _output = output; + } + + + [Fact] + public void Serialize_Should_Prune_Fields() + { + var input = GetFieldTestObject(); + + var settings = new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, NullValueHandling = NullValueHandling.Ignore, ReferenceLoopHandling = ReferenceLoopHandling.Ignore }; + + var jdn = new JsonDotNetSerializer(Settings, 1, 5, 5); + + var json = jdn.SafeSerializeObject(input); + + _output.WriteLine(json); + + _output.WriteLine("Field limiting not yet available in Serialization"); + + //json.Should().NotContainAny("IntSix", "IntSeven", "IntEight", "IntNine", "IntTen"); + } + + [Fact] + public void Serialize_Should_Prune_Depth() + { + var input = GetNestedObj(10, 10); + + var jdn = new JsonDotNetSerializer(Settings, 5, 5, 5); + + var json = jdn.SafeSerializeObject(input); + + _output.WriteLine(json); + + json.Should().HaveLength(182); + } + + [Fact] + public void Serialize_Should_Prune_String() + { + var input = GetStringTestObject(1000); + + var jdn = new JsonDotNetSerializer(Settings, 5, 5, 5); + + var json = jdn.SafeSerializeObject(input); + + _output.WriteLine(json); + + json.Should().HaveLength(22); + } + + + + private static object GetDepthTestObject() + { + var obj = new + { + ObjOne = new + { + FieldOne = 1, + FieldTwo = 2, + FieldThree = 3, + FieldFour = 4, + FieldFive = 5, + FieldSix = 6, + FieldSeven = 7, + FieldEight = 8, + FieldNine = 9, + FieldTen = 10 + }, + FieldEleven = 11, + FieldTwelve = 12, + FieldThirteen = 13, + FieldFourteen = 14, + FieldFifteen = 15, + FieldSixteen = 16, + FieldSeventeen = 17, + FieldEighteen = 18, + FieldNineteen = 19, + FieldTwenty = 20, + FieldTwentyOne = 21, + FieldTwentyTwo = 22, + FieldTwentyFour = 24, + FieldTwentyFive = 25, + FieldTwentySix = 26, + FieldTwentySeven = 27, + FieldTwentyEight = 28, + FieldTwentyNine = 29, + FieldThirty = 30, + FieldThirtyOne = 31, + FieldThirtyTwo = 32, + FieldThirtyThree = 33, + FieldThirtyFour = 34, + FieldThirtyFive = 35, + FieldThirtySix = 36, + FieldThirtySeven = 37, + FieldThirtyEight = 38, + FieldThirtyNine = 39, + FieldFourty = 40, + FieldFourtyOne = 41, + FieldFourtyTwo = 42, + FieldFourtyThree = 43, + FieldFourtyFour = 44, + FieldFourtyFive = 45, + FieldFourtySix = 46, + FieldFourtySeven = 47, + FieldFourtyEight = 48, + FieldFourtyNine = 49, + FieldFifty = 50, + FieldFiftyOne = 51, + ObjTwo = new + { + FieldFiftyTwo = 52, + FieldFiftyThree = 53, + FieldFiftyFour = 54, + FieldFiftyFive = 55, + FieldFiftySix = 56, + FieldFiftySeven = 57, + FieldFiftyEight = 58, + FieldFiftyNine = 59, + FieldSixty = 60 + } + }; + + return obj; + } + + private static object GetFieldTestObject() + { + var obj = new TestField + { + IntOne = 1, + IntTwo = 2, + IntThree = 3, + IntFour = 4, + IntFive = 5, + IntSix = 6, + IntSeven = 7, + IntEight = 8, + IntNine = 9, + IntTen = 10 + }; + + return obj; + } + + private static object GetStringTestObject(int stringLength) + { + var obj = new TestString + { + StringProp = GetString(stringLength) + }; + + return obj; + } + + public class TestField + { + public int IntOne { get; set; } + public int IntTwo { get; set; } + public int IntThree { get; set; } + public int IntFour { get; set; } + public int IntFive { get; set; } + public int IntSix { get; set; } + public int IntSeven { get; set; } + public int IntEight { get; set; } + public int IntNine { get; set; } + public int IntTen { get; set; } + } + + public class TestString + { + public string StringProp { get; set; } + } + + public class Test + { + public Test NestedProp { get; set; } + public string StringProp { get; set; } + } + + private static Test GetNestedObj(int nestingDepth, int stringLength) + { + var obj = new Test { StringProp = GetString(stringLength) }; + + var currentDepth = 0; + + AddNestedObjRecursive(obj, nestingDepth, stringLength, ref currentDepth); + + return obj; + } + + private static void AddNestedObjRecursive(Test parent, int nestingDepth, int stringLength, ref int currentDepth) + { + if (currentDepth >= nestingDepth) + { + return; + } + + var obj = new Test { StringProp = GetString(stringLength) }; + + parent.NestedProp = obj; + + currentDepth++; + + AddNestedObjRecursive(obj, nestingDepth, stringLength, ref currentDepth); + } + + private static string GetString(int stringLength) + { + var ran = new Random(); + + var sb = new StringBuilder(); + + var current = 0; + + while (current < stringLength) + { + var next = ran.Next(0, 9); + sb.Append(next.ToString()); + current++; + } + + var r = sb.ToString(); + + return r; + } + } +} diff --git a/test/StackifyLib.UnitTests/JsonSerialization_Tests.cs b/test/StackifyLib.UnitTests/JsonSerialization_Tests.cs index 68ca968..abcc19e 100644 --- a/test/StackifyLib.UnitTests/JsonSerialization_Tests.cs +++ b/test/StackifyLib.UnitTests/JsonSerialization_Tests.cs @@ -12,22 +12,22 @@ namespace StackifyLib.UnitTests { public class JsonSerialization_Tests { - private readonly ITestOutputHelper output; + private readonly ITestOutputHelper _output; public JsonSerialization_Tests(ITestOutputHelper output) { - this.output = output; + _output = output; } [Fact] - public void Should_Prune_Object() + public void Should_Prune_MaxFields_Object() { var testMaxFields = GetTestObject(); var result = StackifyLib.Utils.HelperFunctions.SerializeDebugData(testMaxFields, false); - output.WriteLine(result); + _output.WriteLine(result); var obj = JObject.Parse(result); @@ -37,7 +37,7 @@ public void Should_Prune_Object() } [Fact] - public void Should_Prune_Object_Array() + public void Should_Prune_MaxFields_Object_Array() { var list = new List(); @@ -49,7 +49,7 @@ public void Should_Prune_Object_Array() var result = StackifyLib.Utils.HelperFunctions.SerializeDebugData(list, false); - output.WriteLine(result); + _output.WriteLine(result); var obj = JObject.Parse(result);