Merge NLog 4.7.7 to dev (#4258)

* fix: generate legal json for keys that contain quote (#4202)

* Improved 4.7.6 Changelog (#4205)

* Skip allocation of SingleCallContinuation when ThrowExceptions = false (#4207)

* StringBuilderExt ClearBuilder with better handling of low memory

* ReusableObjectCreator - Added fallback logic to repair internal pool state

* Travis dist changed to xenial

* Added ExcludeEmptyProperties to JsonLayout. along with unit tests.

* Avoid lookup CurrentProcessFilePath on Android platform

* JsonLayout - Unwind after invalid property value to avoid invalid Json

* Version 4.7.7 (#4257)

* Version 4.7.7


Co-authored-by: Rolf Kristensen <snakefoot@users.noreply.github.com>

Co-authored-by: Virgil Palanciuc <47669377+virgilp@users.noreply.github.com>
Co-authored-by: Pablo Ruiz <pablo.ruiz@gmail.com>
Co-authored-by: Julian Verdurmen <304NotModified@users.noreply.github.com>

CHANGELOG.md

@@ -4,11 +4,25 @@ Date format: (year/month/day)
 
 ## Change Log
 
+### V4.7.7 (2021/01/20)
+
+#### Bugfixes
+- [#4229](https://github.com/NLog/NLog/pull/4229) Skip lookup MainModule.FileName on Android platform to avoid crash  (#4229) (@snakefoot)
+- [#4202](https://github.com/NLog/NLog/pull/4202) JsonLayout - Generate correct json for keys that contain quote (#4202) (@virgilp)
+- [#4245](https://github.com/NLog/NLog/pull/4245) JsonLayout - Unwind after invalid property value to avoid invalid Json (#4245) (@snakefoot)
+
+#### Improvements
+- [#4222](https://github.com/NLog/NLog/pull/4222) Better handling of low memory (#4222) (@snakefoot)
+- [#4221](https://github.com/NLog/NLog/pull/4221) JsonLayout - Added new ExcludeEmptyProperties to skip GDC/MDC/MLDC properties with null or empty values (#4221) (@pruiz)
+
+#### Performance
+- [#4207](https://github.com/NLog/NLog/pull/4207) Skip allocation of SingleCallContinuation when ThrowExceptions = false (#4207) (@snakefoot)
+
 ### V4.7.6 (2020/12/06)
 
 #### Bugfixes
 - [#4142](https://github.com/NLog/NLog/pull/4142) JsonSerializer - Ensure invariant formatting of DateTimeOffset (#4142) (@snakefoot)
-- [#4176](https://github.com/NLog/NLog/pull/4176) AsyncTaskTarget - Flush when buffer is full should not block forever (#4176) (@snakefoot)
+- [#4172](https://github.com/NLog/NLog/pull/4172) AsyncTaskTarget - Flush when buffer is full should not block forever (#4172) (@snakefoot)
 - [#4182](https://github.com/NLog/NLog/pull/4182) Failing to lookup ProcessName because of Access Denied should fallback to Win32-API (#4182) (@snakefoot)
 
 #### Features
@@ -20,7 +34,7 @@ Date format: (year/month/day)
 - [#4190](https://github.com/NLog/NLog/pull/4190) Improving debugger-display for Logger.Properties and LogEventInfo.Properties (@snakefoot)
 
 #### Performance
-- [#4132](https://github.com/NLog/NLog/pull/4132) Improve thead concurrency when using wrapper cached=true (#4132) (@snakefoot)
+- [#4132](https://github.com/NLog/NLog/pull/4132) Improve thread concurrency when using wrapper cached=true (#4132) (@snakefoot)
 - [#4171](https://github.com/NLog/NLog/pull/4171) ConditionLayoutExpression - Skip allocating StringBuilder for every condition check (#4171) (@snakefoot)
 
 ### V4.7.5 (2020/09/27)

build.ps1

@@ -5,7 +5,7 @@ dotnet --version
 # dotnet restore .\src\NLog\
 # dotnet pack .\src\NLog\  --configuration release --include-symbols -o ..\..\artifacts
 
-$versionPrefix = "4.7.6"
+$versionPrefix = "4.7.7"
 $versionSuffix = ""
 $versionFile = $versionPrefix + "." + ${env:APPVEYOR_BUILD_NUMBER}
 $versionProduct = $versionPrefix;

src/NLog/Conditions/Expressions/ConditionLayoutExpression.cs

@@ -34,6 +34,7 @@
 namespace NLog.Conditions
 {
     using Layouts;
+    using NLog.Internal;
     using System.Text;
 
     /// <summary>
@@ -88,7 +89,7 @@ protected override object EvaluateNode(LogEventInfo context)
             }
             finally
             {
-                stringBuilder.Length = 0;
+                stringBuilder.ClearBuilder();
                 System.Threading.Interlocked.Exchange(ref _fastObjectPool, stringBuilder);
             }
         }

src/NLog/Config/LoggingConfigurationFileLoader.cs

@@ -221,16 +221,21 @@ private static bool ScanForBooleanParameter(string fileContent, string parameter
         /// </summary>
         public IEnumerable<string> GetDefaultCandidateConfigFilePaths(string filename = null)
         {
+            string baseDirectory = PathHelpers.TrimDirectorySeparators(_appEnvironment.AppDomainBaseDirectory);
+#if !NETSTANDARD1_3
+            string entryAssemblyLocation = PathHelpers.TrimDirectorySeparators(_appEnvironment.EntryAssemblyLocation);
+#else
+            string entryAssemblyLocation = string.Empty;
+#endif
             if (filename == null)
             {
                 // Scan for process specific nlog-files
-                foreach (var filePath in GetAppSpecificNLogLocations())
+                foreach (var filePath in GetAppSpecificNLogLocations(baseDirectory, entryAssemblyLocation))
                     yield return filePath;
             }
 
             // NLog.config from application directory
             string nlogConfigFile = filename ?? "NLog.config";
-            string baseDirectory = PathHelpers.TrimDirectorySeparators(_appEnvironment.AppDomainBaseDirectory);
             if (!string.IsNullOrEmpty(baseDirectory))
                 yield return Path.Combine(baseDirectory, nlogConfigFile);
 
@@ -239,11 +244,6 @@ public IEnumerable<string> GetDefaultCandidateConfigFilePaths(string filename =
             if (!platformFileSystemCaseInsensitive && !string.IsNullOrEmpty(baseDirectory))
                 yield return Path.Combine(baseDirectory, nLogConfigFileLowerCase);
 
-#if !NETSTANDARD1_3
-            string entryAssemblyLocation = PathHelpers.TrimDirectorySeparators(_appEnvironment.EntryAssemblyLocation);
-#else
-            string entryAssemblyLocation = string.Empty;
-#endif
             if (!string.IsNullOrEmpty(entryAssemblyLocation) && !string.Equals(entryAssemblyLocation, baseDirectory, StringComparison.OrdinalIgnoreCase))
             {
                 yield return Path.Combine(entryAssemblyLocation, nlogConfigFile);
@@ -283,7 +283,7 @@ private static string LookupNLogAssemblyLocation()
         /// <summary>
         /// Get default file paths (including filename) for possible NLog config files. 
         /// </summary>
-        public IEnumerable<string> GetAppSpecificNLogLocations()
+        public IEnumerable<string> GetAppSpecificNLogLocations(string baseDirectory, string entryAssemblyLocation)
         {
             // Current config file with .config renamed to .nlog
             string configurationFile = _appEnvironment.AppDomainConfigurationFile;
@@ -301,37 +301,30 @@ public IEnumerable<string> GetAppSpecificNLogLocations()
 #if NETSTANDARD && !NETSTANDARD1_3
             else
             {
-                string entryAssemblyLocation = PathHelpers.TrimDirectorySeparators(_appEnvironment.EntryAssemblyLocation);
-                string processFilePath = _appEnvironment.CurrentProcessFilePath;
-                string processDirectory = !string.IsNullOrEmpty(processFilePath) ? PathHelpers.TrimDirectorySeparators(Path.GetDirectoryName(processFilePath)) : string.Empty;
-
-                if (!IsValidProcessDirectory(processDirectory, entryAssemblyLocation, _appEnvironment))
-                {
-                    // Handle dotnet-process loading .NET Core-assembly, or IIS-process loading website
-                    string assemblyFileName = _appEnvironment.EntryAssemblyFileName;
-                    yield return Path.Combine(entryAssemblyLocation, assemblyFileName + ".nlog");
+                if (string.IsNullOrEmpty(entryAssemblyLocation))
+                    entryAssemblyLocation = baseDirectory;
 
-                    // Handle unpublished .NET Core Application
-                    assemblyFileName = Path.GetFileNameWithoutExtension(assemblyFileName);
-                    if (!string.IsNullOrEmpty(assemblyFileName))
-                        yield return Path.Combine(entryAssemblyLocation, assemblyFileName + ".exe.nlog");
-                }
-                else if (!string.IsNullOrEmpty(processFilePath))
+                if (PathHelpers.IsTempDir(entryAssemblyLocation, _appEnvironment.UserTempFilePath))
                 {
-                    yield return processFilePath + ".nlog";
-
-                    // Handle published .NET Core Application with assembly-nlog-file
-                    if (!string.IsNullOrEmpty(entryAssemblyLocation))
+                    // Handle Single File Published on NetCore 3.1 and loading side-by-side exe.nlog (Not relevant for Net5.0)
+                    string processFilePath = _appEnvironment.CurrentProcessFilePath;
+                    if (!string.IsNullOrEmpty(processFilePath))
                     {
-                        string assemblyFileName = _appEnvironment.EntryAssemblyFileName;
-                        if (!string.IsNullOrEmpty(assemblyFileName))
-                            yield return Path.Combine(entryAssemblyLocation, assemblyFileName + ".nlog");
+                        yield return processFilePath + ".nlog";
                     }
-                    else
+                }
+
+                if (!string.IsNullOrEmpty(entryAssemblyLocation))
+                {
+                    string assemblyFileName = _appEnvironment.EntryAssemblyFileName;
+                    if (!string.IsNullOrEmpty(assemblyFileName))
                     {
-                        string processFileName = Path.GetFileNameWithoutExtension(processFilePath);
-                        if (!string.IsNullOrEmpty(processFileName))
-                            yield return Path.Combine(processDirectory, processFileName + ".dll.nlog");
+                        // Handle unpublished .NET Core Application
+                        var assemblyBaseName = Path.GetFileNameWithoutExtension(assemblyFileName);
+                        if (!string.IsNullOrEmpty(assemblyBaseName))
+                            yield return Path.Combine(entryAssemblyLocation, assemblyBaseName + ".exe.nlog");
+
+                        yield return Path.Combine(entryAssemblyLocation, assemblyFileName + ".nlog");
                     }
                 }
             }
@@ -356,25 +349,6 @@ private IEnumerable<string> GetPrivateBinPathNLogLocations(string baseDirectory,
             }
         }
 
-#if NETSTANDARD && !NETSTANDARD1_3
-        private static bool IsValidProcessDirectory(string processDirectory, string entryAssemblyLocation, IAppEnvironment appEnvironment)
-        {
-            if (string.IsNullOrEmpty(entryAssemblyLocation))
-                return true;
-
-            if (string.Equals(entryAssemblyLocation, processDirectory, StringComparison.OrdinalIgnoreCase))
-                return true;
-
-            if (string.IsNullOrEmpty(processDirectory))
-                return false;
-
-            if (PathHelpers.IsTempDir(entryAssemblyLocation, appEnvironment.UserTempFilePath))
-                return true;    // Hack for .NET Core 3 - Single File Publish that unpacks Entry-Assembly into temp-folder, and process-directory is valid
-
-            return false;   // NetCore Application is not published and is possible being executed by dotnet-process
-        }
-#endif
-
         protected virtual void Dispose(bool disposing)
         {
             // Nothing to dispose

src/NLog/Internal/ObjectPools/ReusableAsyncLogEventList.cs

@@ -43,7 +43,7 @@ namespace NLog.Internal
     internal class ReusableAsyncLogEventList : ReusableObjectCreator<IList<AsyncLogEventInfo>>
     {
         public ReusableAsyncLogEventList(int capacity)
-            :base(new List<AsyncLogEventInfo>(capacity), (l) => l.Clear())
+            :base(capacity, (cap) => new List<AsyncLogEventInfo>(cap), (l) => l.Clear())
         {
         }
     }

src/NLog/Internal/ObjectPools/ReusableBufferCreator.cs

@@ -41,7 +41,7 @@ namespace NLog.Internal
     internal class ReusableBufferCreator : ReusableObjectCreator<char[]>
     {
         public ReusableBufferCreator(int capacity)
-            :base(new char[capacity], (b) => { })
+            :base(capacity, cap => new char[cap], (b) => { })
         {
         }
     }

src/NLog/Internal/ObjectPools/ReusableBuilderCreator.cs

@@ -41,7 +41,7 @@ namespace NLog.Internal
     internal class ReusableBuilderCreator : ReusableObjectCreator<StringBuilder>
     {
         public ReusableBuilderCreator()
-            : base(new StringBuilder(), (sb) => { sb.ClearBuilder(); })
+            : base(128, (cap) => new StringBuilder(cap), (sb) => { sb.ClearBuilder(); })
         {
         }
     }

src/NLog/Internal/ObjectPools/ReusableObjectCreator.cs

@@ -42,11 +42,18 @@ class ReusableObjectCreator<T> where T : class
     {
         protected T _reusableObject;
         private readonly Action<T> _clearObject;
+        private readonly Func<int, T> _createObject;
+        private readonly int _initialCapacity;
 
-        protected ReusableObjectCreator(T reusableObject, Action<T> clearObject)
+        /// <summary>Empty handle when <see cref="Targets.Target.OptimizeBufferReuse"/> is disabled</summary>
+        public readonly LockOject None = default(LockOject);
+
+        protected ReusableObjectCreator(int initialCapacity, Func<int, T> createObject, Action<T> clearObject)
         {
-            _reusableObject = reusableObject;
+            _reusableObject = createObject(initialCapacity);
             _clearObject = clearObject;
+            _createObject = createObject;
+            _initialCapacity = initialCapacity;
         }
 
         /// <summary>
@@ -55,7 +62,16 @@ protected ReusableObjectCreator(T reusableObject, Action<T> clearObject)
         /// <returns>Handle to the reusable item, that can release it again</returns>
         public LockOject Allocate()
         {
-            return new LockOject(this);
+            var reusableObject = _reusableObject ?? _createObject(_initialCapacity);
+            System.Diagnostics.Debug.Assert(_reusableObject != null);
+            _reusableObject = null;
+            return new LockOject(this, reusableObject);
+        }
+
+        private void Deallocate(T reusableObject)
+        {
+            _clearObject(reusableObject);
+            _reusableObject = reusableObject;
         }
 
         public struct LockOject : IDisposable
@@ -66,20 +82,15 @@ public struct LockOject : IDisposable
             public readonly T Result;
             private readonly ReusableObjectCreator<T> _owner;
 
-            public LockOject(ReusableObjectCreator<T> owner)
+            public LockOject(ReusableObjectCreator<T> owner, T reusableObject)
             {
-                Result = owner._reusableObject;
-                owner._reusableObject = null;
+                Result = reusableObject;
                 _owner = owner;
             }
 
             public void Dispose()
             {
-                if (Result != null)
-                {
-                    _owner._clearObject(Result);
-                    _owner._reusableObject = Result;
-                }
+                _owner?.Deallocate(Result);
             }
         }
     }

src/NLog/Internal/ObjectPools/ReusableStreamCreator.cs

@@ -38,10 +38,10 @@ namespace NLog.Internal
     /// <summary>
     /// Controls a single allocated MemoryStream for reuse (only one active user)
     /// </summary>
-    internal class ReusableStreamCreator : ReusableObjectCreator<System.IO.MemoryStream>, IDisposable
+    internal sealed class ReusableStreamCreator : ReusableObjectCreator<System.IO.MemoryStream>, IDisposable
     {
         public ReusableStreamCreator(int capacity)
-            :base(new System.IO.MemoryStream(capacity), (m) => { m.Position = 0; m.SetLength(0); })
+            :base(capacity, (cap) => new System.IO.MemoryStream(cap), (m) => { m.Position = 0; m.SetLength(0); })
         {
         }
 

src/NLog/Internal/ObjectPools/StringBuilderPool.cs

@@ -97,11 +97,15 @@ private void Release(StringBuilder stringBuilder, int poolIndex)
                 int maxBuilderCapacity = poolIndex == -1 ? _maxBuilderCapacity * 10 : _maxBuilderCapacity;
                 if (stringBuilder.Length > maxBuilderCapacity)
                 {
-                    stringBuilder = new StringBuilder(maxBuilderCapacity / 2);
+                    stringBuilder.Remove(0, stringBuilder.Length - 1);  // Attempt soft clear that skips re-allocation
+                    if (stringBuilder.Capacity > maxBuilderCapacity)
+                    {
+                        stringBuilder = new StringBuilder(maxBuilderCapacity / 2);
+                    }
                 }
             }
 
-            stringBuilder.Length = 0;
+            stringBuilder.ClearBuilder();
 
             if (poolIndex == -1)
             {

src/NLog/Internal/SingleCallContinuation.cs

@@ -42,6 +42,8 @@ namespace NLog.Internal
     /// </summary>
     internal class SingleCallContinuation
     {
+        internal static readonly AsyncContinuation Completed = new SingleCallContinuation(null).CompletedFunction;
+
         private AsyncContinuation _asyncContinuation;
 
         /// <summary>
@@ -77,5 +79,10 @@ public void Function(Exception exception)
                 }       
             }
         }
+
+        private void CompletedFunction(Exception exception)
+        {
+            // Completed, nothing to do
+        }
     }
 }

src/NLog/Internal/Strings/StringBuilderExt.cs

@@ -193,11 +193,22 @@ public static void AppendXmlDateTimeRoundTrip(this StringBuilder builder, DateTi
         /// <param name="builder"></param>
         public static void ClearBuilder(this StringBuilder builder)
         {
+            try
+            {
 #if !NET35
-            builder.Clear();
+                builder.Clear();
 #else
-            builder.Length = 0;
+                builder.Length = 0;
 #endif
+            }
+            catch
+            {
+                // Default StringBuilder Clear() can cause the StringBuilder to re-allocate new internal char-array
+                // This can fail in low memory conditions when StringBuilder is big, so instead try to clear the StringBuilder "gently"
+                if (builder.Length > 1)
+                    builder.Remove(0, builder.Length - 1);
+                builder.Remove(0, builder.Length);
+            }
         }
 
         /// <summary>