Skip to content

Add columnOptions.RemoveStandardColumns #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add columnOptions.RemoveStandardColumns #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ var log = new LoggerConfiguration()
"test_db.logs",
50,
TimeSpan.FromSeconds(30),
new ColumnOptions{
RemoveStandardColumns = new List<string>{"level","message"}
},
new List<AdditionalColumn>
{
new AdditionalColumn { Name = "source", Type = "String" },
Expand All @@ -41,6 +44,9 @@ var log = new LoggerConfiguration()
"tableName": "test_db.logs",
"batchPostingLimit": 50,
"period": "00:00:30",
"columnOptions": {
"RemoveStandardColumns": ["level","message"]
},
"additionalColumns": [{
"Name": "source",
"Type": "String"
Expand Down
7 changes: 5 additions & 2 deletions Serilog.Sinks.ClickHouse/Sinks/ClickHouse/ClickHouseSink.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,27 @@ public class ClickHouseSink : IBatchedLogEventSink
private readonly IFormatProvider _formatProvider;
private readonly ClickHouseProvider<ColumnFormatter> _provider;
private readonly IEnumerable<AdditionalColumn> _additionalColumns;
private readonly ColumnOptions _columnOptions;

public ClickHouseSink(
string connectionString,
string tableName,
ColumnOptions columnOptions = null,
IEnumerable<AdditionalColumn> additionalColumns = null,
IFormatProvider formatProvider = null,
bool autoCreateSqlTable = true)
{
_columnOptions = columnOptions;
_additionalColumns = additionalColumns;
_formatProvider = formatProvider;
_provider = new ClickHouseProvider<ColumnFormatter>(tableName, connectionString, additionalColumns, autoCreateSqlTable);
_provider = new ClickHouseProvider<ColumnFormatter>(tableName, connectionString, columnOptions, additionalColumns, autoCreateSqlTable);
}

public async Task EmitBatchAsync(IReadOnlyCollection<LogEvent> events)
{
try
{
await _provider.FlushAsync(events.Select(e => new ColumnFormatter(e, _formatProvider, _additionalColumns)));
await _provider.FlushAsync(events.Select(e => new ColumnFormatter(e, _formatProvider, _additionalColumns, _columnOptions?.RemoveStandardColumns)));
}
catch (Exception ex)
{
Expand Down
3 changes: 2 additions & 1 deletion Serilog.Sinks.ClickHouse/Sinks/ClickHouse/Provider/ClickHouseProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class ClickHouseProvider<TColumnFormatter>
public ClickHouseProvider(
string tableName,
string connectionString,
ColumnOptions columnOptions = null,
IEnumerable<AdditionalColumn> additionalColumns = null,
bool autoCreateSqlTable = true)
{
Expand All @@ -24,7 +25,7 @@ public ClickHouseProvider(
if (string.IsNullOrWhiteSpace(tableName))
throw new ArgumentNullException(nameof(tableName));

_table = new TableHelper<TColumnFormatter>(tableName, additionalColumns);
_table = new TableHelper<TColumnFormatter>(tableName, additionalColumns, columnOptions);
_connectionString = connectionString;

if (autoCreateSqlTable)
Expand Down
40 changes: 24 additions & 16 deletions Serilog.Sinks.ClickHouse/Sinks/ClickHouse/Provider/ColumnFormatter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using Serilog.Events;

namespace Serilog.Sinks.ClickHouse.Provider
{
class ColumnFormatter : IEnumerable
{
private static readonly List<PropertyInfo> _props = ColumnsHelper.Props<ColumnFormatter>();
private static readonly Dictionary<PropertyInfo,ColumnAttribute> _props = ColumnsHelper.Props<ColumnFormatter>();

private readonly LogEvent _message;
private readonly IFormatProvider _formatProvider;
private readonly IEnumerable<AdditionalColumn> _additionslColumns;
private readonly IEnumerable<string> _removeStandardColumns;

[Column(Name = "timestamp", Type = "DateTime")]
public DateTime Timestamp { get => _message.Timestamp.UtcDateTime; }
Expand All @@ -21,37 +23,43 @@ class ColumnFormatter : IEnumerable
[Column(Name = "message", Type = "String")]
public string Message { get => _message.RenderMessage(_formatProvider); }

public ColumnFormatter(LogEvent message, IFormatProvider formatProvider = null, IEnumerable<AdditionalColumn> additionalColumns = null)
public ColumnFormatter(LogEvent message, IFormatProvider formatProvider = null, IEnumerable<AdditionalColumn> additionalColumns = null, IEnumerable<string> removeStandardColumns = null)
{
_message = message;
_removeStandardColumns = removeStandardColumns;
_additionslColumns = additionalColumns;
_formatProvider = formatProvider;
}

public IEnumerator GetEnumerator()
{
foreach (var p in _props)
yield return p.GetValue(this);
{
if (_removeStandardColumns is null || !_removeStandardColumns.Contains(p.Value.Name))
{
yield return p.Key.GetValue(this);
}
}

if (_additionslColumns != null)
{
foreach (var col in _additionslColumns)
{
if (!_message.Properties.TryGetValue(col.Name, out var value))
foreach (var col in _additionslColumns)
{
yield return Default(col.Type);
continue;
}
if (!_message.Properties.TryGetValue(col.Name, out var value))
{
yield return Default(col.Type);
continue;
}

if(!(value is ScalarValue scalarValue))
{
yield return value.ToString();
continue;
}
if(!(value is ScalarValue scalarValue))
{
yield return value.ToString();
continue;
}

yield return scalarValue.Value;
yield return scalarValue.Value;
}
}
}
}

private object Default(string type)
Expand Down
9 changes: 9 additions & 0 deletions Serilog.Sinks.ClickHouse/Sinks/ClickHouse/Provider/ColumnOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
using System.Collections.Generic;

namespace Serilog.Sinks.ClickHouse.Provider
{
public class ColumnOptions
{
public IEnumerable<string> RemoveStandardColumns { get; set; }
}
}
11 changes: 7 additions & 4 deletions Serilog.Sinks.ClickHouse/Sinks/ClickHouse/Provider/ColumnsHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,17 @@ public static List<ColumnAttribute> Mapping<T>()
return dict;
}

public static List<PropertyInfo> Props<T>()
public static Dictionary<PropertyInfo,ColumnAttribute> Props<T>()
{
var dict = new List<PropertyInfo>();
var dict = new Dictionary<PropertyInfo,ColumnAttribute>();
var props = typeof(T).GetProperties();
foreach (var p in props)
{
if (p.GetCustomAttribute<ColumnAttribute>() != null)
dict.Add(p);
var columnAttribute = p.GetCustomAttribute<ColumnAttribute>();
if (columnAttribute != null)
{
dict.Add(p,columnAttribute);
}
}

return dict;
Expand Down
8 changes: 7 additions & 1 deletion Serilog.Sinks.ClickHouse/Sinks/ClickHouse/Provider/TableHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,15 @@ class TableHelper<T>
public string Create { get; private set; }
public string Insert { get; private set; }

public TableHelper(string name, IEnumerable<AdditionalColumn> additionalColumns = null)
public TableHelper(string name, IEnumerable<AdditionalColumn> additionalColumns = null, ColumnOptions columnOptions = null)
{
var mapping = ColumnsHelper.Mapping<T>();

if (columnOptions?.RemoveStandardColumns != null)
{
mapping.RemoveAll(x => columnOptions.RemoveStandardColumns.Contains(x.Name));
}

if (additionalColumns != null)
mapping = mapping.Union(additionalColumns.Select(c => new ColumnAttribute { Name = c.Name, Type = c.Type })).ToList();

Expand Down
24 changes: 24 additions & 0 deletions Serilog.Sinks.ClickHouse/Sinks/Extensions/ClickHouseSinkExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ public static LoggerConfiguration ClickHouse(
string tableName,
int batchPostingLimit,
TimeSpan period,
ColumnOptions columnOptions = null,
IEnumerable<AdditionalColumn> additionalColumns = null,
IFormatProvider formatProvider = null,
LogEventLevel restrictedToMinimumLevel = LogEventLevel.Verbose)
{
var sink = new ClickHouseSink(
connectionString,
tableName,
columnOptions,
additionalColumns,
formatProvider);

Expand All @@ -33,5 +35,27 @@ public static LoggerConfiguration ClickHouse(

return loggerConfiguration.Sink(sink, batchingOptions, restrictedToMinimumLevel);
}

public static LoggerConfiguration ClickHouse(
this LoggerSinkConfiguration loggerConfiguration,
string connectionString,
string tableName,
int batchPostingLimit,
TimeSpan period,
IEnumerable<AdditionalColumn> additionalColumns = null,
IFormatProvider formatProvider = null,
LogEventLevel restrictedToMinimumLevel = LogEventLevel.Verbose)
{
return loggerConfiguration.ClickHouse(
connectionString,
tableName,
batchPostingLimit,
period,
null,
additionalColumns,
formatProvider,
restrictedToMinimumLevel
);
}
}
}