Skip to content
Permalink
Browse files

Background jobs should not get deadlocks.

Context properties will now display HTML as code
  • Loading branch information...
jgauffin committed Jul 10, 2019
1 parent 3202b68 commit cbff8960d6327dcb11a6ec97069a8fdb0fc22457
@@ -1,10 +1,13 @@
using System;
using System.Data;
using System.Diagnostics;
using System.Threading.Tasks;
using Coderr.Server.Abstractions.Boot;
using Coderr.Server.Abstractions.Config;
using Coderr.Server.App.Core.Reports.Config;
using Griffin.ApplicationServices;
using Griffin.Data;
using Griffin.Data.Mapper;
using log4net;

namespace Coderr.Server.App.Core.Incidents.Jobs
@@ -23,28 +26,58 @@ namespace Coderr.Server.App.Core.Incidents.Jobs
internal class DeleteEmptyIncidents : IBackgroundJobAsync
{
private readonly ILog _logger = LogManager.GetLogger(typeof(DeleteEmptyIncidents));
private readonly IAdoNetUnitOfWork _unitOfWork;
private readonly IDbConnection _connection;
private readonly IConfiguration<ReportConfig> _reportConfiguration;

/// <summary>
/// Creates a new instance of <see cref="DeleteEmptyIncidents" />.
/// </summary>
/// <param name="unitOfWork">Used for SQL queries</param>
public DeleteEmptyIncidents(IAdoNetUnitOfWork unitOfWork, IConfiguration<ReportConfig> reportConfiguration)
/// <param name="connection">Used for SQL queries</param>
public DeleteEmptyIncidents(IDbConnection connection, IConfiguration<ReportConfig> reportConfiguration)
{
if (unitOfWork == null) throw new ArgumentNullException("unitOfWork");
_unitOfWork = unitOfWork;
this._reportConfiguration = reportConfiguration;
_connection = connection;
_reportConfiguration = reportConfiguration;
}

/// <inheritdoc />
public async Task ExecuteAsync()
{
using (var cmd = _unitOfWork.CreateDbCommand())
using (var cmd = _connection.CreateDbCommand())
{
cmd.CommandText =
$@"DELETE TOP(500) Incidents
WHERE LastReportAtUtc < @retentionDays";
$@"CREATE TABLE #ItemsToDelete
(
Id int NOT NULL PRIMARY KEY
)
INSERT #ItemsToDelete (Id)
SELECT TOP(500) Id
FROM Incidents WITH (ReadUncommitted)
WHERE LastReportAtUtc < @retentionDays
declare @counter int = 0;
IF @@ROWCOUNT <> 0
BEGIN
DECLARE ItemsToDeleteCursor CURSOR LOCAL FORWARD_ONLY READ_ONLY
FOR SELECT Id FROM #ItemsToDelete
set @counter = 1
DECLARE @IdToDelete int
OPEN ItemsToDeleteCursor
FETCH NEXT FROM ItemsToDeleteCursor INTO @IdToDelete
WHILE @@FETCH_STATUS = 0
BEGIN
set @counter = @counter + 1
DELETE FROM Incidents WHERE Id = @IdToDelete
FETCH NEXT FROM ItemsToDeleteCursor INTO @IdToDelete
END
CLOSE ItemsToDeleteCursor
DEALLOCATE ItemsToDeleteCursor
END
DROP TABLE #ItemsToDelete
select @counter;";

// Wait until no reports have been received for the specified report save time
// and then make sure during another period that no new reports have been received.
@@ -1,10 +1,13 @@
using System;
using System.Data;
using System.Diagnostics;
using System.Threading.Tasks;
using Coderr.Server.Abstractions.Boot;
using Coderr.Server.Abstractions.Config;
using Coderr.Server.App.Core.Reports.Config;
using Griffin.ApplicationServices;
using Griffin.Data;
using Griffin.Data.Mapper;
using log4net;

namespace Coderr.Server.App.Core.Reports.Jobs
@@ -17,56 +20,89 @@ namespace Coderr.Server.App.Core.Reports.Jobs
public class DeleteOldReports : IBackgroundJobAsync
{
private readonly ILog _logger = LogManager.GetLogger(typeof(DeleteOldReports));
private readonly IAdoNetUnitOfWork _unitOfWork;
private readonly IDbConnection _connection;
private readonly IConfiguration<ReportConfig> _reportConfig;

/// <summary>
/// Creates a new instance of <see cref="DeleteOldReports" />.
/// </summary>
/// <param name="unitOfWork">Used for SQL queries</param>
/// <param name="connection">Used for SQL queries</param>
/// <param name="reportConfig"></param>
/// <exception cref="ArgumentNullException"></exception>
public DeleteOldReports(IAdoNetUnitOfWork unitOfWork, IConfiguration<ReportConfig> reportConfig)
public DeleteOldReports(IDbConnection connection, IConfiguration<ReportConfig> reportConfig)
{
_unitOfWork = unitOfWork ?? throw new ArgumentNullException(nameof(unitOfWork));
_connection = connection;
_reportConfig = reportConfig;
}

/// <summary>
/// Number of reports which can be stored per incident.
/// </summary>
public int MaxReportsPerIncident
{
get { return _reportConfig.Value.MaxReportsPerIncident; }
}

/// <summary>
/// Number of days to keep old reports.
/// </summary>
public int RetentionDays
{
get
{
return _reportConfig.Value.RetentionDays;
}
}
public int RetentionDays => _reportConfig.Value.RetentionDays;

/// <inheritdoc />
public async Task ExecuteAsync()
{
using (var cmd = _unitOfWork.CreateDbCommand())
using (var cmd = _connection.CreateDbCommand())
{
var sql = @"DELETE TOP(1000) FROM ErrorReports WHERE CreatedAtUtc < @date";
var sql = @"CREATE TABLE #OldReports
(
ReportId int NOT NULL PRIMARY KEY
)
INSERT #OldReports (ReportId)
SELECT TOP(1000) Id
FROM ErrorReports WITH (READUNCOMMITTED)
WHERE CreatedAtUtc < @date
declare @counter int = 0;
IF @@ROWCOUNT <> 0
BEGIN
DECLARE OldReportsCursor CURSOR LOCAL FORWARD_ONLY READ_ONLY
FOR SELECT ReportId FROM #OldReports
set @counter = 1
DECLARE @ReportId int
OPEN OldReportsCursor
FETCH NEXT FROM OldReportsCursor INTO @ReportId
WHILE @@FETCH_STATUS = 0
BEGIN
set @counter = @counter + 1
DELETE FROM ErrorReports WHERE Id = @ReportId
FETCH NEXT FROM OldReportsCursor INTO @ReportId
END
CLOSE OldReportsCursor
DEALLOCATE OldReportsCursor
END
DROP TABLE #OldReports
select @counter;
";
cmd.CommandText = sql;
cmd.AddParameter("date", DateTime.UtcNow.AddDays(-RetentionDays));
cmd.CommandTimeout = 90;
var rows = await cmd.ExecuteNonQueryAsync();
if (rows > 0)
try
{
_logger.Debug("Deleted the oldest " + rows + " reports.");
var rows = await cmd.ExecuteNonQueryAsync();
if (rows > 0)
{
_logger.Debug("Deleted the oldest " + rows + " reports.");
}
}
catch (Exception ex)
{
_logger.Error("Failed on connection " + cmd.Connection.ConnectionString, ex);
throw;
}
}
}

}
}
@@ -1,9 +1,9 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using Coderr.Server.Abstractions.Boot;
using Coderr.Server.Abstractions.Config;
using Coderr.Server.App.Core.Reports.Config;

using Griffin.ApplicationServices;
using Griffin.Data;
using log4net;
@@ -22,18 +22,17 @@ namespace Coderr.Server.App.Core.Reports.Jobs
public class DeleteReportsBelowReportLimit : IBackgroundJob
{
private readonly ILog _logger = LogManager.GetLogger(typeof(DeleteReportsBelowReportLimit));
private readonly IAdoNetUnitOfWork _unitOfWork;
private ConfigurationStore _configStore;
private readonly IDbConnection _connection;
private readonly IConfiguration<ReportConfig> _reportConfig;

/// <summary>
/// Creates a new instance of <see cref="DeleteReportsBelowReportLimit" />.
/// </summary>
/// <param name="unitOfWork">Used for SQL queries</param>
public DeleteReportsBelowReportLimit(IAdoNetUnitOfWork unitOfWork, ConfigurationStore configStore)
/// <param name="connection">Used for SQL queries</param>
public DeleteReportsBelowReportLimit(IDbConnection connection, IConfiguration<ReportConfig> reportConfig)
{
if (unitOfWork == null) throw new ArgumentNullException("unitOfWork");
_unitOfWork = unitOfWork;
_configStore = configStore;
_connection = connection;
_reportConfig = reportConfig;
}

/// <summary>
@@ -43,57 +42,73 @@ public int MaxReportsPerIncident
{
get
{
var config = _configStore.Load<ReportConfig>();
if (config == null)
return 100;
return config.MaxReportsPerIncident;
return _reportConfig?.Value?.MaxReportsPerIncident ?? 100;
}
}

/// <inheritdoc />
public void Execute()
{
// find incidents with too many reports.
var incidentsToTruncate = new List<Tuple<int, int>>();
using (var cmd = _unitOfWork.CreateCommand())
using (var cmd = _connection.CreateCommand())
{
cmd.CommandText =
@"SELECT TOP(5) IncidentId, count(Id)
FROM ErrorReports WITH (ReadPast)
GROUP BY IncidentId
HAVING Count(IncidentId) > @max
ORDER BY count(Id) DESC";
var sql = $@"CREATE TABLE #Incidents (Id int NOT NULL PRIMARY KEY, NumberOfItems int)
INSERT #Incidents (Id, NumberOfItems)
SELECT TOP(100) IncidentId, Count(Id) - @max
FROM ErrorReports WITH (READUNCOMMITTED)
GROUP BY IncidentId
HAVING Count(Id) > @max
ORDER BY count(Id) DESC
CREATE TABLE #ReportsToDelete (Id int not null primary key)
declare @counter int = 0;
DECLARE IncidentCursor CURSOR LOCAL FORWARD_ONLY READ_ONLY
FOR SELECT Id, NumberOfItems FROM #Incidents
DECLARE @IncidentId int
DECLARE @NumberOfItems int
OPEN IncidentCursor
FETCH NEXT FROM IncidentCursor INTO @IncidentId, @NumberOfItems
WHILE @@FETCH_STATUS = 0
BEGIN
INSERT INTO #ReportsToDelete (Id)
SELECT TOP(@NumberOfItems) Id
FROM ErrorReports WITH (READUNCOMMITTED)
WHERE IncidentId = @IncidentId
ORDER BY Id asc
FETCH NEXT FROM IncidentCursor INTO @IncidentId, @NumberOfItems
END
CLOSE IncidentCursor
DEALLOCATE IncidentCursor
DROP TABLE #Incidents
DECLARE ItemsToDeleteCursor CURSOR LOCAL FORWARD_ONLY READ_ONLY
FOR SELECT Id FROM #ReportsToDelete
DECLARE @IdToDelete int
OPEN ItemsToDeleteCursor
FETCH NEXT FROM ItemsToDeleteCursor INTO @IdToDelete
WHILE @@FETCH_STATUS = 0
BEGIN
set @counter = @counter + 1
DELETE FROM ErrorReports WHERE Id = @IdToDelete
FETCH NEXT FROM ItemsToDeleteCursor INTO @IdToDelete
END
CLOSE ItemsToDeleteCursor
DEALLOCATE ItemsToDeleteCursor
DROP TABLE #ReportsToDelete
select @counter;";

cmd.CommandText = sql;
cmd.CommandTimeout = 90;
cmd.AddParameter("max", MaxReportsPerIncident);
using (var reader = cmd.ExecuteReader())
var rows = (int)cmd.ExecuteScalar();
if (rows > 0)
{
while (reader.Read())
{
incidentsToTruncate.Add(new Tuple<int, int>((int)reader[0], (int)reader[1]));
}
_logger.Debug("Deleted the oldest " + rows + " reports.");
}
}

foreach (var incidentIdAndCount in incidentsToTruncate)
{
//do not delete more then 500 at a time.
var rowsToDelete = Math.Min(500, incidentIdAndCount.Item2 - MaxReportsPerIncident);
using (var cmd = _unitOfWork.CreateCommand())
{
var sql = $@"With RowsToDelete AS
(
SELECT TOP {rowsToDelete} Id
FROM ErrorReports WITH (ReadPast)
WHERE IncidentId = {incidentIdAndCount.Item1}
)
DELETE FROM RowsToDelete";
cmd.CommandText = sql;
cmd.CommandTimeout = 90;
var rows = cmd.ExecuteNonQuery();
if (rows > 0)
{
_logger.Debug("Deleted the oldest " + rows + " reports for incident " + incidentIdAndCount);
}
}
}
}
}
@@ -1,5 +1,6 @@
using System;
using System.Threading.Tasks;
using Coderr.Client;
using Coderr.Server.Domain.Modules.ApplicationVersions;
using Coderr.Server.ReportAnalyzer.Abstractions.Incidents;
using DotNetCqs;
@@ -29,6 +30,13 @@ public async Task HandleAsync(IMessageContext context, ReportAddedToIncident e)
if (version == null)
return;

if (version.Length > 20)
{
Err.ReportLogicError("Application version is too large.", new {version, e.Incident.ApplicationName},
"AppVersionLength");
return;
}

var isNewIncident = e.Incident.ReportCount <= 1;
var versionEntity = await _repository.FindVersionAsync(e.Incident.ApplicationId, version)
?? new ApplicationVersion(e.Incident.ApplicationId, e.Incident.ApplicationName,
@@ -19,7 +19,7 @@ public ActionResult Index()
public ActionResult NoInstallation()
{
if (Request.Path.Value.EndsWith("/setup/activate", StringComparison.OrdinalIgnoreCase))
return Redirect("~/?#/welcome/admin/");
return Redirect("~/");
return View();
}

0 comments on commit cbff896

Please sign in to comment.
You can’t perform that action at this time.