Merge pull request #70 from VahidFarahmandian/67-partition-indexes-us…

…ing-partition-key

partitioning documents based on the partition key

01-Core/Jinget.Core/Security/SqlInjection/SqlInjection.cs

@@ -1,6 +1,4 @@
-using System.Text.Encodings.Web;
-
-namespace Jinget.Core.Security.SqlInjection
+namespace Jinget.Core.Security.SqlInjection
 {
     /// <summary>
     /// SqlInjection preventation class

03-Infrastructure/Jinget.Logger/Configuration/ElasticSearch/ElasticSearchConfiguration.cs

@@ -22,16 +22,40 @@ public static class ElasticSearchConfiguration
         {
             services.ConfigurePrerequisites();
 
-            services.TryAddScoped<IElasticSearchBaseDomainService<TErrorEntity, int>, ElasticSearchBaseDomainService<TErrorEntity, int>>();
-            services.TryAddScoped<IElasticSearchBaseDomainService<TOperationalEntity, int>, ElasticSearchBaseDomainService<TOperationalEntity, int>>();
-            services.TryAddScoped<IElasticSearchBaseDomainService<TCustomEntity, int>, ElasticSearchBaseDomainService<TCustomEntity, int>>();
+            services.TryAddScoped<IElasticSearchBaseDomainService<TErrorEntity>, ElasticSearchBaseDomainService<TErrorEntity>>();
+            services.TryAddScoped<IElasticSearchBaseDomainService<TOperationalEntity>, ElasticSearchBaseDomainService<TOperationalEntity>>();
+            services.TryAddScoped<IElasticSearchBaseDomainService<TCustomEntity>, ElasticSearchBaseDomainService<TCustomEntity>>();
 
-            services.TryAddScoped<IElasticSearchRepository<TErrorEntity, int>, ElasticSearchRepository<TErrorEntity, int>>();
-            services.TryAddScoped<IElasticSearchRepository<TOperationalEntity, int>, ElasticSearchRepository<TOperationalEntity, int>>();
-            services.TryAddScoped<IElasticSearchRepository<TCustomEntity, int>, ElasticSearchRepository<TCustomEntity, int>>();
+            services.TryAddScoped<IElasticSearchRepository<TErrorEntity>, ElasticSearchRepository<TErrorEntity>>();
+            services.TryAddScoped<IElasticSearchRepository<TOperationalEntity>, ElasticSearchRepository<TOperationalEntity>>();
+            services.TryAddScoped<IElasticSearchRepository<TCustomEntity>, ElasticSearchRepository<TCustomEntity>>();
 
             if (elasticSearchSetting != null)
             {
+                services.TryAddSingleton(elasticSearchSetting);
+
+                var client = CreateClient(elasticSearchSetting);
+                services.AddSingleton(typeof(IElasticClient), client);
+                CreateIndexes(client, elasticSearchSetting);
+            }
+        }
+
+        public static void ConfigureElasticSearchLogger(this IServiceCollection services, ElasticSearchSettingModel elasticSearchSetting = null)
+        {
+            services.ConfigurePrerequisites();
+
+            services.TryAddScoped<IElasticSearchBaseDomainService<ErrorLog>, ElasticSearchBaseDomainService<ErrorLog>>();
+            services.TryAddScoped<IElasticSearchBaseDomainService<OperationLog>, ElasticSearchBaseDomainService<OperationLog>>();
+            services.TryAddScoped<IElasticSearchBaseDomainService<CustomLog>, ElasticSearchBaseDomainService<CustomLog>>();
+
+            services.TryAddScoped<IElasticSearchRepository<ErrorLog>, ElasticSearchRepository<ErrorLog>>();
+            services.TryAddScoped<IElasticSearchRepository<OperationLog>, ElasticSearchRepository<OperationLog>>();
+            services.TryAddScoped<IElasticSearchRepository<CustomLog>, ElasticSearchRepository<CustomLog>>();
+
+            if (elasticSearchSetting != null)
+            {
+                services.TryAddSingleton(elasticSearchSetting);
+
                 var client = CreateClient(elasticSearchSetting);
                 services.AddSingleton(typeof(IElasticClient), client);
                 CreateIndexes(client, elasticSearchSetting);
@@ -60,32 +84,35 @@ static ElasticClient CreateClient(ElasticSearchSettingModel elasticSearchSetting
 
         static void CreateIndexes(ElasticClient client, ElasticSearchSettingModel elasticSearchSetting)
         {
-            List<Type> indexes = new();
-            if (elasticSearchSetting.RegisterDefaultLogModels)
-            {
-                indexes = typeof(BaseEntity<>).Assembly
-                    .GetTypes()
-                    .Where(x => x.GetCustomAttributes(typeof(EntityAttribute), false).Any(y => ((EntityAttribute)y).ElasticSearchEnabled))
-                    .ToList();
-            }
-            if (elasticSearchSetting.DiscoveryTypes != null && elasticSearchSetting.DiscoveryTypes.Any())
+            if (!elasticSearchSetting.CreateIndexPerPartition)
             {
-                foreach (var item in elasticSearchSetting.DiscoveryTypes)
+                List<Type> indexes = new();
+                if (elasticSearchSetting.RegisterDefaultLogModels)
                 {
-                    indexes.AddRange(item.Assembly.GetTypes()
+                    indexes = typeof(BaseEntity<>).Assembly
+                        .GetTypes()
                         .Where(x => x.GetCustomAttributes(typeof(EntityAttribute), false).Any(y => ((EntityAttribute)y).ElasticSearchEnabled))
-                        .ToList());
+                        .ToList();
+                }
+                if (elasticSearchSetting.DiscoveryTypes != null && elasticSearchSetting.DiscoveryTypes.Any())
+                {
+                    foreach (var item in elasticSearchSetting.DiscoveryTypes)
+                    {
+                        indexes.AddRange(item.Assembly.GetTypes()
+                            .Where(x => x.GetCustomAttributes(typeof(EntityAttribute), false).Any(y => ((EntityAttribute)y).ElasticSearchEnabled))
+                            .ToList());
+                    }
                 }
-            }
 
-            foreach (var item in indexes)
-            {
-                string indexName = $"{AppDomain.CurrentDomain.FriendlyName}.{item.Name}".ToLower();
-                if (!client.Indices.Exists(indexName).Exists)
+                foreach (var item in indexes)
                 {
-                    var result = client.Indices.Create(indexName, index => index.Map(m => m.AutoMap(item).NumericDetection(true)));
-                    if (!result.IsValid)
-                        throw new JingetException("Jinget Says: " + result.OriginalException);
+                    string indexName = $"{AppDomain.CurrentDomain.FriendlyName}.{item.Name}".ToLower();
+                    if (!client.Indices.Exists(indexName).Exists)
+                    {
+                        var result = client.Indices.Create(indexName, index => index.Map(m => m.AutoMap(item).NumericDetection(true)));
+                        if (!result.IsValid)
+                            throw new JingetException("Jinget Says: " + result.OriginalException);
+                    }
                 }
             }
         }

03-Infrastructure/Jinget.Logger/Configuration/ElasticSearch/ElasticSearchSettingModel.cs

@@ -35,5 +35,17 @@ public class ElasticSearchSettingModel
         /// should be registered in DI container or not
         /// </summary>
         public bool RegisterDefaultLogModels { get; set; }
+
+        /// <summary>
+        /// Create index per partition using HttpContext.Items["jinget.log.partitionkey"] value.
+        /// If this mode is selected, then <see cref="RegisterDefaultLogModels"/> and also <seealso cref="DiscoveryTypes"/> will not be used.
+        /// If this mode is selected, then index creation will be deferred until the first document insertion.
+        /// foeach partition key, a separated index will be created. 
+        /// all of the indexes will share the same data model. 
+        /// for request/response logs, <see cref="Entities.Log.OperationLog"/> will be used.
+        /// for errors, <see cref="Entities.Log.ErrorLog"/> will be used.
+        /// for custom logs, <see cref="Entities.Log.CustomLog"/> will be used.
+        /// </summary>
+        public bool CreateIndexPerPartition { get; set; }
     }
 }

03-Infrastructure/Jinget.Logger/Configuration/JingetProgram.cs

@@ -21,6 +21,13 @@ public static class JingetProgram
                 f.BlackListStrings = blackList;
             }));
 
+        public static IHostBuilder LogToElasticSearch(this IHostBuilder webHostBuilder, string[] blackList)
+            =>
+            webHostBuilder.ConfigureLogging(builder => builder.AddElasticSearch(f =>
+            {
+                f.BlackListStrings = blackList;
+            }));
+
         public static IHostBuilder LogToFile(this IHostBuilder webHostBuilder, string[] blackList, string fileNamePrefix = "Log", string logDirectory = "Logs", int retainFileCountLimit = 5, int fileSizeLimit = 10) =>
             webHostBuilder.ConfigureLogging(builder => builder.AddFile(f =>
             {

03-Infrastructure/Jinget.Logger/Entities/Log/Base/LogBaseEntity.cs

@@ -5,12 +5,15 @@ namespace Jinget.Logger.Entities.Log.Base
     public abstract class LogBaseEntity : BaseEntity<int>
     {
         public DateTime When { get; set; }
-
         public string Url { get; set; }
         public string Description { get; set; }
-
         public string SubSystem { get; set; }
 
+        /// <summary>
+        /// This is used when the CreateIndexPerPartition is set to true.
+        /// This property is specific to Elasticsearch logging
+        /// </summary>
+        public string ParitionKey { get; set; }
         public Guid RequestId { get; set; } = Guid.Empty;
     }
 }

03-Infrastructure/Jinget.Logger/ExceptionHandler/ExceptionHandler.cs

@@ -22,23 +22,29 @@ public void Handle(Exception ex, object details)
         {
             ErrorLog logEntity = new()
             {
-                Description = $"" +
-                    $"Error Message: {ex.Message + Environment.NewLine} " +
-                    $"Details: {JsonConvert.SerializeObject(details) + Environment.NewLine} " +
-                    $"StackTrace:{ex.StackTrace}"
+                Description = JsonConvert.SerializeObject(new
+                {
+                    ex.Message,
+                    Details = JsonConvert.SerializeObject(details),
+                    ex.StackTrace
+                }),
+
+                ParitionKey =
+                _accessor.HttpContext.Items["jinget.log.partitionkey"] != null ?
+                _accessor.HttpContext.Items["jinget.log.partitionkey"].ToString() :
+                "",
+                Severity = LogLevel.Error.ToString()
             };
             if (details is LogBaseEntity entity)
             {
                 logEntity.RequestId = entity.RequestId;
-                logEntity.Severity = LogLevel.Error.ToString();
                 logEntity.SubSystem = entity.SubSystem;
                 logEntity.When = entity.When;
                 logEntity.Url = entity.Url;
             }
             else
             {
                 logEntity.RequestId = new Guid(_accessor.HttpContext.Response.Headers["RequestId"].ToString());
-                logEntity.Severity = LogLevel.Error.ToString();
                 logEntity.SubSystem = AppDomain.CurrentDomain.FriendlyName;
                 logEntity.When = DateTime.Now;
             }

03-Infrastructure/Jinget.Logger/Handlers/CommandHandlers/ElasticSearchBaseDomainService.cs

@@ -1,26 +1,21 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using Jinget.Logger.Entities;
+using Jinget.Logger.Entities.Log.Base;
+using Jinget.Logger.ViewModels;
 
 namespace Jinget.Logger.Handlers.CommandHandlers
 {
-    public class ElasticSearchBaseDomainService<TModelType, TKeyType> :
-        IElasticSearchBaseDomainService<TModelType, TKeyType>
-        where TModelType : BaseEntity<TKeyType>
+    public class ElasticSearchBaseDomainService<TModelType> :
+        IElasticSearchBaseDomainService<TModelType>
+        where TModelType : LogBaseEntity
     {
-        protected readonly IElasticSearchRepository<TModelType, TKeyType> Repository;
+        protected readonly IElasticSearchRepository<TModelType> Repository;
 
-        public ElasticSearchBaseDomainService(IElasticSearchRepository<TModelType, TKeyType> repository) => Repository = repository;
+        public ElasticSearchBaseDomainService(IElasticSearchRepository<TModelType> repository) => Repository = repository;
 
         public virtual async Task<bool> CreateAsync(TModelType param) => await Repository.IndexAsync(param);
         public virtual async Task<bool> BulkCreateAsync(IList<TModelType> @params) => await Repository.BulkIndexAsync(@params);
-
         public virtual async Task<TModelType> FetchLatestAsync() => await Repository.GetLatestAsync();
-
-        //public virtual async Task<(IReadOnlyCollection<TModelType> Records, long TotalRecordCount)> FetchAllAsync(
-        //       Func<QueryContainerDescriptor<TModelType>, QueryContainer> filter = null,
-        //       Func<SortDescriptor<TModelType>, IPromise<IList<ISort>>> orderBy = null,
-        //       int? pageSize = 0,
-        //       int? pageIndex = -1) => await Repository.QueryAsync(filter, orderBy, pageSize, pageIndex);
+        public virtual async Task<List<LogSearchViewModel>> SearchAsync(string partitionKey, string searchString, int pageNumber, int pageSize, string username = "") => await Repository.SearchAsync(partitionKey, searchString, pageNumber, pageSize, username);
     }
 }

03-Infrastructure/Jinget.Logger/Handlers/CommandHandlers/IElasticSearchBaseDomainService.cs

@@ -1,22 +1,18 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using Jinget.Logger.Entities;
+using Jinget.Logger.Entities.Log.Base;
+using Jinget.Logger.ViewModels;
 
 namespace Jinget.Logger.Handlers.CommandHandlers
 {
-    public interface IElasticSearchBaseDomainService<TModelType, TKeyType>
-        where TModelType : BaseEntity<TKeyType>
+    public interface IElasticSearchBaseDomainService<TModelType>
+        where TModelType : LogBaseEntity
     {
         Task<TModelType> FetchLatestAsync();
 
         Task<bool> CreateAsync(TModelType param);
 
         Task<bool> BulkCreateAsync(IList<TModelType> @params);
-
-        //Task<(IReadOnlyCollection<TModelType> Records, long TotalRecordCount)> FetchAllAsync(
-        //    Func<QueryContainerDescriptor<TModelType>, QueryContainer> filter = null,
-        //    Func<SortDescriptor<TModelType>, IPromise<IList<ISort>>> orderBy = null,
-        //    int? pageSize = 0,
-        //    int? pageIndex = -1);
+        Task<List<LogSearchViewModel>> SearchAsync(string partitionKey, string searchString, int pageNumber, int pageSize, string username = "");
     }
 }