feat: apm kibana query with clickhouse

masastack · Mar 8, 2024 · 0b0bbff · 0b0bbff
1 parent cb5896b
commit 0b0bbff
Show file tree

Hide file tree

Showing 93 changed files with 4,975 additions and 56 deletions.
diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Masa.Tsc.ApiGateways.Caller.csproj
@@ -7,6 +7,7 @@
 	</PropertyGroup>
 
 	<ItemGroup>
+		<PackageReference Include="Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse" Version="1.0.0-local" />
 		<PackageReference Include="Masa.Contrib.Service.Caller.DaprClient" Version="$(MasaFrameworkPackageVersion)" />
 		<PackageReference Include="Masa.Contrib.StackSdks.Config" Version="$(MasaFrameworkPackageVersion)" />
 		<PackageReference Include="Masa.BuildingBlocks.Data.Contracts" Version="$(MasaFrameworkPackageVersion)" />

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/ApmService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/ApmService.cs
@@ -0,0 +1,30 @@
+// Copyright (c) MASA Stack All rights reserved.
+// Licensed under the MIT License. See LICENSE.txt in the project root for license information.
+
+using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Request;
+using Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse.Models.Response;
+
+namespace Masa.Tsc.ApiGateways.Caller.Services;
+
+public sealed class ApmService : BaseService
+{
+    internal ApmService(ICaller caller) : base(caller, "/api/apm") { }
+
+    public Task<PaginatedListBase<ServiceListDto>> GetServicePageAsync(BaseApmRequestDto query) => Caller.GetAsync<PaginatedListBase<ServiceListDto>>($"{RootPath}/services", data: query)!;
+
+    public Task<PaginatedListBase<EndpointListDto>> GetEndpointPageAsync(BaseApmRequestDto query) => Caller.GetAsync<PaginatedListBase<EndpointListDto>>($"{RootPath}/endpoints", data: query)!;
+
+    public Task<PaginatedListBase<ErrorMessageDto>> GetErrorsPageAsync(ApmEndpointRequestDto query) => Caller.GetAsync<PaginatedListBase<ErrorMessageDto>>($"{RootPath}/errors", data: query)!;
+
+    public Task<List<ChartPointDto>> GetSpanErrorsAsync(ApmEndpointRequestDto query) => Caller.GetAsync<List<ChartPointDto>>($"{RootPath}/spanErrors", data: query)!;
+
+    public Task<PaginatedListBase<TraceResponseDto>> GetTraceDetailAsync(ApmTraceLatencyRequestDto query) => Caller.GetAsync<PaginatedListBase<TraceResponseDto>>($"{RootPath}/traceDetail", data: query)!;
+
+    public Task<List<ChartLineDto>> GetChartsAsync(BaseApmRequestDto query) => Caller.GetAsync<List<ChartLineDto>>($"{RootPath}/charts", data: query)!;
+
+    public Task<List<ChartLineCountDto>> GetErrorChartAsync(ApmEndpointRequestDto query) => Caller.GetAsync<List<ChartLineCountDto>>($"{RootPath}/errorChart", data: query)!;
+
+    public Task<List<ChartLineCountDto>> GetLogChartAsync(ApmEndpointRequestDto query) => Caller.GetAsync<List<ChartLineCountDto>>($"{RootPath}/logChart", data: query)!;
+
+    public Task<EndpointLatencyDistributionDto> GetLatencyDistributionAsync(ApmEndpointRequestDto query) => Caller.GetAsync<EndpointLatencyDistributionDto>($"{RootPath}/latencyDistributions", data: query)!;
+}
diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/AppService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/AppService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class AppService : BaseService
+public sealed class AppService : BaseService
 {
-    public AppService(ICaller caller) : base(caller, "/api/app") { }
+    internal AppService(ICaller caller) : base(caller, "/api/app") { }
 
     public async Task<List<AppDto>> GetAppsAsync(string projectId) => await Caller.GetAsync<List<AppDto>>($"{RootPath}?projectId={projectId}") ?? default!;
 

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/DirectoryService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/DirectoryService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class DirectoryService : BaseService
+public sealed class DirectoryService : BaseService
 {
-    public DirectoryService(ICaller caller) : base(caller, "/api/Instrument/directory") { }
+    internal DirectoryService(ICaller caller) : base(caller, "/api/Instrument/directory") { }
 
     public async Task<IEnumerable<DirectoryTreeDto>> GetTreeAsync(bool isContainsInstrument = true) => (await Caller.GetAsync<IEnumerable<DirectoryTreeDto>>($"{RootPath}/tree/{isContainsInstrument}"))!;
 

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/InstrumentService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/InstrumentService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class InstrumentService : BaseService
+public sealed class InstrumentService : BaseService
 {
-    public InstrumentService(ICaller caller) : base(caller, "/api/Instrument") { }
+    internal InstrumentService(ICaller caller) : base(caller, "/api/Instrument") { }
 
     public async Task AddAsync(AddDashboardDto param) => await Caller.PostAsync($"{RootPath}", param);
 

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/LogService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/LogService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class LogService : BaseService
+public sealed class LogService : BaseService
 {
-    public LogService(ICaller caller) : base(caller, "/api/log") { }
+    internal LogService(ICaller caller) : base(caller, "/api/log") { }
 
     public async Task<TResult> AggregateAsync<TResult>(SimpleAggregateRequestDto model)
     {

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/MetricService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/MetricService.cs
@@ -3,11 +3,11 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class MetricService : BaseService
+public sealed class MetricService : BaseService
 {
     static JsonSerializerOptions options;
 
-    public MetricService(ICaller caller) : base(caller, "/api/metric")
+    internal MetricService(ICaller caller) : base(caller, "/api/metric")
     {
         if (options == null)
         {

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/ProjectService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/ProjectService.cs
@@ -3,11 +3,11 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class ProjectService : BaseService
+public sealed class ProjectService : BaseService
 {
     private readonly IDccClient _dccClient;
 
-    public ProjectService(ICaller caller, IDccClient dccClient) : base(caller, "/api/project")
+    internal ProjectService(ICaller caller, IDccClient dccClient) : base(caller, "/api/project")
     {
         _dccClient = dccClient;
     }

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/SettingService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/SettingService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class SettingService: BaseService
+public sealed class SettingService: BaseService
 {
-    public SettingService(ICaller caller) : base(caller, "/api/settings") { }
+    internal SettingService(ICaller caller) : base(caller, "/api/settings") { }
 
     public async Task<SettingDto> GetStorage() => await Caller.GetAsync<SettingDto>($"{RootPath}/storage") ?? default!;
 }
diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TeamService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TeamService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class TeamService : BaseService
+public sealed class TeamService : BaseService
 {
-    public TeamService(ICaller caller) : base(caller, "/api/team") { }
+    internal TeamService(ICaller caller) : base(caller, "/api/team") { }
 
     public async Task<TeamDto> GetTeamAsync(Guid teamId, string projectId) => (await Caller.GetAsync<TeamDto>($"{RootPath}/{teamId}/{projectId}"))!;
 }
diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TopologyService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TopologyService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class TopologyService : BaseService
+public sealed class TopologyService : BaseService
 {
-    public TopologyService(ICaller caller) : base(caller, "/api/topology") { }
+    internal TopologyService(ICaller caller) : base(caller, "/api/topology") { }
 
     public async Task<TopologyResultDto> GetAsync(string? serviceName, int level, DateTime start, DateTime end) => (await Caller.GetAsync<TopologyResultDto>($"{RootPath}", new { serviceName, level, start, end }))!;
 }
diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TraceService.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/Services/TraceService.cs
@@ -3,9 +3,9 @@
 
 namespace Masa.Tsc.ApiGateways.Caller.Services;
 
-public class TraceService : BaseService
+public sealed class TraceService : BaseService
 {
-    public TraceService(ICaller caller) : base(caller, "/api/trace") { }
+    internal TraceService(ICaller caller) : base(caller, "/api/trace") { }
 
     public async Task<IEnumerable<TraceResponseDto>> GetAsync(string traceId)
     {

diff --git a/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/TscCaller.cs b/src/ApiGateways/Masa.Tsc.ApiGateways.Caller/TscCaller.cs
@@ -17,10 +17,13 @@ internal TscCaller(IServiceProvider serviceProvider, ICaller caller)
         InstrumentService = new InstrumentService(caller);
         MetricService = new MetricService(caller);
         TopologyService = new TopologyService(caller);
+        ApmService=new ApmService(caller);
     }
 
     public AppService AppService { get; private init; }
 
+    public ApmService ApmService { get; private init; }
+
     public ProjectService ProjectService { get; private init; }
 
     public TeamService TeamService { get; private init; }

diff --git a/src/Contracts/Masa.Tsc.Contracts.Admin/Infrastructure/Const/StorageConst.cs b/src/Contracts/Masa.Tsc.Contracts.Admin/Infrastructure/Const/StorageConst.cs
@@ -36,9 +36,13 @@ public static string Timestimap(bool isElasticsearch = false, bool isClickhouse
 
     public static string TraceId { get; private set; } = "TraceId";
 
+    public static string SpanId { get; private set; } = "SpanId";
+
     public const string Environment = "Resource.service.namespace";
 
     public const string ExceptionMessage = "Attributes.exception.message";
 
+    public const string ExceptionType = "Attributes.exception.type";
+
     public const string TaskId = "Attributes.TaskId";
 }
diff --git a/src/Contracts/Masa.Tsc.Contracts.Admin/Logs/LogPageQueryDto.cs b/src/Contracts/Masa.Tsc.Contracts.Admin/Logs/LogPageQueryDto.cs
@@ -13,10 +13,14 @@ public class LogPageQueryDto : Pagination<LogPageQueryDto>
 
     public bool IsDesc { get; set; } = true;
 
+    public string SortField { get; set; }
+
     public string Duration { get; set; }
 
     public string SpanId { get; set; }
 
+    public string Env { get; set; }
+
     /// <summary>
     ///  scheduler job taskid
     /// </summary>

diff --git a/src/Contracts/Masa.Tsc.Contracts.Admin/Traces/RequestTraceListDto.cs b/src/Contracts/Masa.Tsc.Contracts.Admin/Traces/RequestTraceListDto.cs
@@ -22,4 +22,12 @@ public class RequestTraceListDto : Pagination<RequestTraceListDto>
     public bool IsDesc { get; set; } = true;
 
     public bool IsError { get; set; }
+
+    public string Env { get; set; }
+
+    public int? LatMin { get; set; }
+
+    public int? LatMax { get; set; }
+
+    public string SpanId { get; set; }
 }
diff --git a/src/Services/Masa.Tsc.Service.Admin/Application/Logs/Queries/LogsQuery.cs b/src/Services/Masa.Tsc.Service.Admin/Application/Logs/Queries/LogsQuery.cs
@@ -4,10 +4,12 @@
 namespace Masa.Tsc.Service.Admin.Application.Logs;
 
 public record LogsQuery(string Query, DateTime Start, DateTime End, int Page = 1, int Size = 10, bool IsDesc = true,
+    string? SortField = null,
     string Duration = "15m",
     string? JobTaskId = null,
     string? SpanId = null,
     string? Service = null,
+    string? Env = null,
     string? LogLevel = null) : Query<PaginatedListBase<LogResponseDto>>
 {
     public override PaginatedListBase<LogResponseDto> Result { get; set; }

diff --git a/src/Services/Masa.Tsc.Service.Admin/Application/Logs/QueryHandler.cs b/src/Services/Masa.Tsc.Service.Admin/Application/Logs/QueryHandler.cs
@@ -1,6 +1,8 @@
 // Copyright (c) MASA Stack All rights reserved.
 // Licensed under the MIT License. See LICENSE.txt in the project root for license information.
 
+using System;
+
 namespace Masa.Tsc.Service.Admin.Application.Logs;
 
 public class QueryHandler : EnvQueryHandler
@@ -31,22 +33,20 @@ public async Task GetLatestDataAsync(LatestLogQuery queryData)
             Keyword = queryData.Query,
             Page = 1,
             PageSize = 1,
-            Sort = new FieldOrderDto { Name = "@timestamp", IsDesc = !queryData.IsDesc }
+            Sort = new FieldOrderDto { Name = StorageConst.Timestimap(ConfigConst.IsElasticsearch, ConfigConst.IsClickhouse), IsDesc = !queryData.IsDesc }
         };
 
         var env = GetServiceEnvironmentName(string.Empty!);
         query.SetEnv(env);
         var data = await _logService.ListAsync(query);
-
         queryData.Result = data?.Result?.FirstOrDefault()!;
     }
 
     [EventHandler]
     public async Task GetMappingAsync(LogFieldQuery query)
     {
         query.Result = await _logService.GetMappingAsync();
-        if (query.Result == null)
-            query.Result = Array.Empty<MappingResponseDto>();
+        query.Result ??= Array.Empty<MappingResponseDto>();
     }
 
     [EventHandler]
@@ -95,13 +95,14 @@ public async Task GetPageListAsync(LogsQuery queryData)
             RawQuery = isRawQuery ? queryData.Query! : string.Empty,
             Page = queryData.Page,
             PageSize = queryData.Size,
-            Sort = new FieldOrderDto { Name = "@timestamp", IsDesc = queryData.IsDesc },
+            Sort = new FieldOrderDto { Name = string.IsNullOrEmpty(queryData.SortField) ? StorageConst.Timestimap(ConfigConst.IsElasticsearch, ConfigConst.IsClickhouse) : queryData.SortField, IsDesc = queryData.IsDesc },
             Conditions = conditions
         };
-        if (!isSkipEnv && !isRawQuery)
+        if (!isSkipEnv)
         {
-            var env = GetServiceEnvironmentName(queryData.Service!);
-            query.SetEnv(env);
+            var env = queryData.Env == null ? GetServiceEnvironmentName(queryData.Service!) : queryData.Env;
+            if (!string.IsNullOrEmpty(env))
+                query.SetEnv(env,true);
         }
 
         var data = await _logService.ListAsync(query);

diff --git a/src/Services/Masa.Tsc.Service.Admin/Application/Metrics/QueryHandler.cs b/src/Services/Masa.Tsc.Service.Admin/Application/Metrics/QueryHandler.cs
@@ -17,7 +17,7 @@ public class QueryHandler : EnvQueryHandler
         IMultiEnvironmentContext multiEnvironment) : base(masaStackConfig, environment, multiEnvironment)
     {
         _prometheusClient = masaPrometheusClient;
-        _multilevelCacheClient = multilevelCacheClientFactory.Create(masaStackConfig.GetServiceId(MasaStackProject.TSC));
+        _multilevelCacheClient = multilevelCacheClientFactory.Create($"{masaStackConfig.GetServiceId(MasaStackProject.TSC)}_{masaStackConfig.Environment}");
         _logger = logger;
     }
 
@@ -70,7 +70,7 @@ public async Task GetRangeValuesAsync(RangeValueQuery query)
             return;
         }
 
-        _logger.LogError("request failed {data.ErrorType} {data.Error}", data.ErrorType,data.Error);
+        _logger.LogError("request failed {data.ErrorType} {data.Error}", data.ErrorType, data.Error);
     }
 
     [EventHandler]

diff --git a/src/Services/Masa.Tsc.Service.Admin/Application/Traces/Queries/TraceListQuery.cs b/src/Services/Masa.Tsc.Service.Admin/Application/Traces/Queries/TraceListQuery.cs
@@ -3,7 +3,7 @@
 
 namespace Masa.Tsc.Service.Admin.Application.Traces;
 
-public record TraceListQuery(string Service, string Instance, string Endpoint, string TraceId, DateTime Start, DateTime End, int Page, int Size, bool IsDesc, string Keyword, bool IsError) : Query<PaginatedListBase<TraceResponseDto>>
+public record TraceListQuery(string Service, string Instance, string Endpoint, string TraceId, DateTime Start, DateTime End, int Page, int Size, bool IsDesc, string Keyword, bool IsError,string Env,int? LatMin,int? LatMax,string SpanId) : Query<PaginatedListBase<TraceResponseDto>>
 {
     public override PaginatedListBase<TraceResponseDto> Result { get; set; }
 }
diff --git a/src/Services/Masa.Tsc.Service.Admin/Application/Traces/QueryHandler.cs b/src/Services/Masa.Tsc.Service.Admin/Application/Traces/QueryHandler.cs
@@ -1,10 +1,6 @@
 // Copyright (c) MASA Stack All rights reserved.
 // Licensed under the MIT License. See LICENSE.txt in the project root for license information.
 
-using Google.Api;
-using Masa.Contrib.StackSdks.Pm;
-using System;
-
 namespace Masa.Tsc.Service.Admin.Application.Traces;
 
 public class QueryHandler : EnvQueryHandler
@@ -87,6 +83,16 @@ public async Task GetListAsync(TraceListQuery query)
             }
         }
 
+        if (!string.IsNullOrEmpty(query.SpanId))
+        {
+            list.Add(new FieldConditionDto
+            {
+                Name = StorageConst.SpanId,
+                Type = ConditionTypes.Equal,
+                Value = query.SpanId
+            });
+        }
+
         if (query.IsError)
         {
             list.Add(new FieldConditionDto
@@ -97,10 +103,46 @@ public async Task GetListAsync(TraceListQuery query)
             });
         }
 
-        queryDto.Conditions = list;
 
-        if (string.IsNullOrEmpty(query.TraceId) && !query.IsError && !isRawQuery)
-            queryDto.SetEnv(GetServiceEnvironmentName(query.Service));
+        var conditions = new List<FieldConditionDto>();
+        if (!string.IsNullOrEmpty(query.Env))
+        {
+            conditions.Add(new FieldConditionDto
+            {
+                Name = "Resource.service.namespace",
+                Type = ConditionTypes.Equal,
+                Value = query.Env
+            });
+        }
+        else
+        {
+            if (string.IsNullOrEmpty(query.TraceId) && !query.IsError && !isRawQuery)
+                queryDto.SetEnv(GetServiceEnvironmentName(query.Service));
+        }
+
+        if (query.LatMin.HasValue && query.LatMin.Value >= 0)
+        {
+            conditions.Add(new FieldConditionDto
+            {
+                Name = "Duration",
+                Type = ConditionTypes.GreatEqual,
+                Value = query.LatMin.Value,
+            });
+        }
+
+        if (query.LatMax.HasValue && query.LatMax.Value >= 0 && (
+            !query.LatMin.HasValue
+            || query.LatMin.HasValue && query.LatMax - query.LatMin.Value > 0))
+            conditions.Add(new FieldConditionDto
+            {
+                Name = "Duration",
+                Type = ConditionTypes.LessEqual,
+                Value = query.LatMax.Value,
+            });
+        if (conditions.Any())
+            queryDto.Conditions = conditions;
+
+        queryDto.Conditions = list;
 
         query.Result = await _traceService.ListAsync(queryDto);
         query.Result ??= new PaginatedListBase<TraceResponseDto>();