diff --git a/src/Raytha.Application/Common/Interfaces/IExecuteBackgroundTask.cs b/src/Raytha.Application/Common/Interfaces/IExecuteBackgroundTask.cs index 93c0cb07..91980331 100644 --- a/src/Raytha.Application/Common/Interfaces/IExecuteBackgroundTask.cs +++ b/src/Raytha.Application/Common/Interfaces/IExecuteBackgroundTask.cs @@ -1,5 +1,7 @@ -namespace Raytha.Application.Common.Interfaces; +using System.Text.Json; + +namespace Raytha.Application.Common.Interfaces; public interface IExecuteBackgroundTask { - Task Execute(Guid jobId, dynamic args, CancellationToken cancellationToken); + Task Execute(Guid jobId, JsonElement args, CancellationToken cancellationToken); } diff --git a/src/Raytha.Application/Common/Interfaces/IRaythaDbJsonQueryEngine.cs b/src/Raytha.Application/Common/Interfaces/IRaythaDbJsonQueryEngine.cs index cef8b81e..1d0ad388 100644 --- a/src/Raytha.Application/Common/Interfaces/IRaythaDbJsonQueryEngine.cs +++ b/src/Raytha.Application/Common/Interfaces/IRaythaDbJsonQueryEngine.cs @@ -1,4 +1,5 @@ using Raytha.Domain.Entities; +using System.Data; namespace Raytha.Application.Common.Interfaces; @@ -11,9 +12,14 @@ public interface IRaythaDbJsonQueryEngine string[] filters, int pageSize, int pageNumber, - string orderBy); + string orderBy, + IDbTransaction transaction = null); + + IEnumerable QueryAllContentItemsAsTransaction(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, string orderBy); + int CountContentItems(Guid contentTypeId, string[] searchOnColumns, string search, - string[] filters); + string[] filters, + IDbTransaction transaction = null); } diff --git a/src/Raytha.Application/ContentItems/Commands/BeginExportContentItemsToCsv.cs b/src/Raytha.Application/ContentItems/Commands/BeginExportContentItemsToCsv.cs index 4679cb49..472deeff 100644 --- a/src/Raytha.Application/ContentItems/Commands/BeginExportContentItemsToCsv.cs +++ b/src/Raytha.Application/ContentItems/Commands/BeginExportContentItemsToCsv.cs @@ -2,6 +2,16 @@ using Raytha.Application.Common.Models; using CSharpVitamins; using Raytha.Application.Common.Interfaces; +using FluentValidation; +using Raytha.Application.Common.Exceptions; +using Raytha.Domain.Entities; +using Microsoft.EntityFrameworkCore; +using Raytha.Domain.ValueObjects; +using Raytha.Application.Common.Utils; +using Csv; +using Raytha.Domain.Common; +using Raytha.Domain.ValueObjects.FieldValues; +using System.Text.Json; namespace Raytha.Application.ContentItems.Commands; @@ -9,18 +19,45 @@ public class BeginExportContentItemsToCsv { public record Command : LoggableRequest> { + public ShortGuid ViewId { get; init; } + public bool ExportOnlyColumnsFromView { get; init; } = true; + } + public class Validator : AbstractValidator + { + public Validator() + { + RuleFor(x => x.ViewId).NotEmpty(); + } } public class Handler : IRequestHandler> { private readonly IBackgroundTaskQueue _taskQueue; - public Handler(IBackgroundTaskQueue taskQueue) + private readonly IRaythaDbContext _entityFrameworkDb; + private readonly IContentTypeInRoutePath _contentTypeInRoutePath; + public Handler( + IBackgroundTaskQueue taskQueue, + IRaythaDbContext entityFrameworkDb, + IContentTypeInRoutePath contentTypeInRoutePath + ) { _taskQueue = taskQueue; + _entityFrameworkDb = entityFrameworkDb; + _contentTypeInRoutePath = contentTypeInRoutePath; } public async Task> Handle(Command request, CancellationToken cancellationToken) { + View view = _entityFrameworkDb.Views + .Include(p => p.ContentType) + .ThenInclude(p => p.ContentTypeFields) + .FirstOrDefault(p => p.Id == request.ViewId.Guid); + + if (view == null) + throw new NotFoundException("View", request.ViewId); + + _contentTypeInRoutePath.ValidateContentTypeInRoutePathMatchesValue(view.ContentType.DeveloperName); + var backgroundJobId = await _taskQueue.EnqueueAsync(request, cancellationToken); return new CommandResponseDto(backgroundJobId); @@ -29,31 +66,182 @@ public async Task> Handle(Command request, Cancell public class BackgroundTask : IExecuteBackgroundTask { - private readonly IRaythaDbContext _db; - public BackgroundTask(IRaythaDbContext db) + private readonly IRaythaDbJsonQueryEngine _db; + private readonly IRaythaDbContext _entityFrameworkDb; + private readonly IContentTypeInRoutePath _contentTypeInRoutePath; + private readonly ICurrentOrganization _currentOrganization; + private readonly IFileStorageProvider _fileStorageProvider; + + public BackgroundTask( + ICurrentOrganization currentOrganization, + IRaythaDbJsonQueryEngine db, + IRaythaDbContext entityFrameworkDb, + IFileStorageProvider fileStorageProvider) { _db = db; + _entityFrameworkDb = entityFrameworkDb; + _currentOrganization = currentOrganization; + _fileStorageProvider = fileStorageProvider; } - public async Task Execute(Guid jobId, dynamic args, CancellationToken cancellationToken) + public async Task Execute(Guid jobId, JsonElement args, CancellationToken cancellationToken) { - int delayLoop = 0; - var guid = Guid.NewGuid().ToString(); - Random rnd = new Random(); - var job = _db.BackgroundTasks.First(p => p.Id == jobId); - while (delayLoop < 3) + IEnumerable items; + + Guid viewId = args.GetProperty("ViewId").GetProperty("Guid").GetGuid(); + bool exportOnlyColumnsFromView = args.GetProperty("ExportOnlyColumnsFromView").GetBoolean(); + + View view = _entityFrameworkDb.Views + .Include(p => p.ContentType) + .ThenInclude(p => p.ContentTypeFields) + .FirstOrDefault(p => p.Id == viewId); + + if (view == null) + throw new NotFoundException("View", viewId); + + string[] filters = GetFiltersForView(view); + string finalOrderBy = GetSortForView(view); + + int count = _db.CountContentItems(view.ContentTypeId, new string[0], string.Empty, filters); + + var job = _entityFrameworkDb.BackgroundTasks.First(p => p.Id == jobId); + job.StatusInfo = $"Pulling {count} items for {view.ContentType.LabelPlural} - {view.Label}"; + job.PercentComplete = 0; + _entityFrameworkDb.BackgroundTasks.Update(job); + await _entityFrameworkDb.SaveChangesAsync(cancellationToken); + + var myExport = new CsvExport(); + int currentIndex = 0; + foreach (var item in _db.QueryAllContentItemsAsTransaction(view.ContentTypeId, + new string[0], + string.Empty, + filters, + finalOrderBy)) { - Thread.Sleep(rnd.Next(1000, 10000)); - delayLoop++; - job.StatusInfo = $"Made it here: {delayLoop}"; - job.PercentComplete = delayLoop * 10; - _db.BackgroundTasks.Update(job); - await _db.SaveChangesAsync(cancellationToken); + var contentItemAsDict = MapToListItemValues(ContentItemDto.GetProjection(item)); + + myExport.AddRow(); + if (exportOnlyColumnsFromView) + { + foreach (var column in view.Columns) + { + if (contentItemAsDict.ContainsKey(column)) + { + myExport[column] = contentItemAsDict[column]; + } + else + { + myExport[column] = string.Empty; + } + } + } + else + { + foreach (var column in BuiltInContentTypeField.ReservedContentTypeFields) + { + if (contentItemAsDict.ContainsKey(column)) + { + myExport[column] = contentItemAsDict[column]; + } + else + { + myExport[column] = string.Empty; + } + } + foreach (var column in view.ContentType.ContentTypeFields.Select(p => p.DeveloperName)) + { + if (contentItemAsDict.ContainsKey(column)) + { + myExport[column] = contentItemAsDict[column]; + } + else + { + myExport[column] = string.Empty; + } + } + } + + currentIndex++; + int percentDone = (currentIndex / count) * 100; + if (percentDone % 20 == 0) + { + job.PercentComplete = percentDone - 10; + _entityFrameworkDb.BackgroundTasks.Update(job); + await _entityFrameworkDb.SaveChangesAsync(cancellationToken); + } } + job.StatusInfo = $"Items pulled. Generating csv..."; + _entityFrameworkDb.BackgroundTasks.Update(job); + await _entityFrameworkDb.SaveChangesAsync(cancellationToken); + + var csvAsBytes = myExport.ExportToBytes(); + + string fileName = $"{_currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(DateTime.UtcNow)}-{view.DeveloperName}"; + string downloadUrl = await _fileStorageProvider.SaveAndGetDownloadUrlAsync(csvAsBytes, Guid.NewGuid().ToString(), fileName, "text/csv", DateTime.UtcNow.AddYears(999)); + job.PercentComplete = 100; - //job.DownloadUrl = "https://s3.amazonaws.com/vo-random/ShareX/2023/05/chrome_f7gOblEvFX.png"; - _db.BackgroundTasks.Update(job); - await _db.SaveChangesAsync(cancellationToken); + job.DownloadUrl = downloadUrl; + _entityFrameworkDb.BackgroundTasks.Update(job); + await _entityFrameworkDb.SaveChangesAsync(cancellationToken); + } + + protected string GetSortForView(View view) + { + string finalOrderBy = $"{BuiltInContentTypeField.CreationTime.DeveloperName} {SortOrder.DESCENDING}"; + var viewOrderBy = view.Sort.Select(p => $"{p.DeveloperName} {p.SortOrder.DeveloperName}").ToList(); + finalOrderBy = viewOrderBy.Any() ? string.Join(",", viewOrderBy) : finalOrderBy; + + return finalOrderBy; + } + + protected string[] GetFiltersForView(View view) + { + var conditionToODataUtility = new FilterConditionToODataUtility(view.ContentType); + var oDataFromFilter = conditionToODataUtility.ToODataFilter(view.Filter); + return new string[] { oDataFromFilter }; + } + + private Dictionary MapToListItemValues(ContentItemDto item) + { + var viewModel = new Dictionary + { + //Built in + { BuiltInContentTypeField.Id, item.Id }, + { "CreatorUser", item.CreatorUser != null ? item.CreatorUser.FullName : "N/A" }, + { "LastModifierUser", item.LastModifierUser != null ? item.LastModifierUser.FullName : "N/A" }, + { BuiltInContentTypeField.CreationTime, _currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(item.CreationTime) }, + { BuiltInContentTypeField.LastModificationTime, _currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(item.LastModificationTime) }, + { BuiltInContentTypeField.IsPublished, item.IsPublished.YesOrNo() }, + { BuiltInContentTypeField.IsDraft, item.IsDraft.YesOrNo() }, + { BuiltInContentTypeField.PrimaryField, item.PrimaryField }, + { "Template", item.WebTemplate.Label } + }; + + //Content type fields + foreach (var field in item.PublishedContent as Dictionary) + { + if (field.Value is DateTimeFieldValue dateTimeFieldValue) + { + viewModel.Add(field.Key, _currentOrganization.TimeZoneConverter.ToDateFormat(dateTimeFieldValue.Value)); + } + else if (field.Value is GuidFieldValue guidFieldValue) + { + if (guidFieldValue.HasValue) + viewModel.Add(field.Key, (ShortGuid)guidFieldValue.Value); + else + viewModel.Add(field.Key, string.Empty); + } + else if (field.Value is IBaseEntity) + { + viewModel.Add(field.Key, field.Value.PrimaryField); + } + else + { + viewModel.Add(field.Key, field.Value.ToString()); + } + } + + return viewModel; } } } diff --git a/src/Raytha.Application/Raytha.Application.csproj b/src/Raytha.Application/Raytha.Application.csproj index 7924e6e0..b1133cea 100644 --- a/src/Raytha.Application/Raytha.Application.csproj +++ b/src/Raytha.Application/Raytha.Application.csproj @@ -15,6 +15,7 @@ + diff --git a/src/Raytha.Infrastructure/BackgroundTasks/BackgroundTaskQueue.cs b/src/Raytha.Infrastructure/BackgroundTasks/BackgroundTaskQueue.cs index 490b2214..7b265206 100644 --- a/src/Raytha.Infrastructure/BackgroundTasks/BackgroundTaskQueue.cs +++ b/src/Raytha.Infrastructure/BackgroundTasks/BackgroundTaskQueue.cs @@ -1,5 +1,4 @@ -using Microsoft.EntityFrameworkCore; -using Raytha.Application.Common.Interfaces; +using Raytha.Application.Common.Interfaces; using Raytha.Domain.Entities; using System.Text.Json; diff --git a/src/Raytha.Infrastructure/BackgroundTasks/QueuedHostedService.cs b/src/Raytha.Infrastructure/BackgroundTasks/QueuedHostedService.cs index b9e08ac8..f3fe4f9e 100644 --- a/src/Raytha.Infrastructure/BackgroundTasks/QueuedHostedService.cs +++ b/src/Raytha.Infrastructure/BackgroundTasks/QueuedHostedService.cs @@ -5,6 +5,7 @@ using Raytha.Application.Common.Interfaces; using Raytha.Domain.Entities; using System.Text.Json; +using System.Text.Json.Serialization; namespace Raytha.Infrastructure.BackgroundTasks; @@ -50,7 +51,7 @@ private async Task BackgroundProcessing(CancellationToken stoppingToken) scope.ServiceProvider .GetRequiredService(Type.GetType(backgroundTask.Name)) as IExecuteBackgroundTask; - await scopedProcessingService.Execute(backgroundTask.Id, JsonSerializer.Deserialize(backgroundTask.Args), stoppingToken); + await scopedProcessingService.Execute(backgroundTask.Id, JsonSerializer.Deserialize(backgroundTask.Args), stoppingToken); backgroundTask.Status = BackgroundTaskStatus.Complete; } catch (Exception ex) diff --git a/src/Raytha.Infrastructure/JsonQueryEngine/RaythaDbJsonQueryEngine.cs b/src/Raytha.Infrastructure/JsonQueryEngine/RaythaDbJsonQueryEngine.cs index 77305914..a2f1209b 100644 --- a/src/Raytha.Infrastructure/JsonQueryEngine/RaythaDbJsonQueryEngine.cs +++ b/src/Raytha.Infrastructure/JsonQueryEngine/RaythaDbJsonQueryEngine.cs @@ -1,5 +1,6 @@ using CSharpVitamins; using Dapper; +using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Raytha.Application.Common.Interfaces; using Raytha.Application.Common.Utils; @@ -97,7 +98,7 @@ public ContentItem FirstOrDefault(Guid entityId) return contentItem; } - public IEnumerable QueryContentItems(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, int pageSize, int pageNumber, string orderBy) + public IEnumerable QueryContentItems(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, int pageSize, int pageNumber, string orderBy, IDbTransaction transaction = null) { var contentType = _entityFramework.ContentTypes .Include(p => p.ContentTypeFields) @@ -155,7 +156,7 @@ public IEnumerable QueryContentItems(Guid contentTypeId, string[] s var selector = sqlBuilder.AddTemplate($"SELECT /**select**/ {fromClause} /**where**/ /**orderby**/ OFFSET {skip} ROWS FETCH NEXT {pageSize} ROWS ONLY"); - var resultFromQuery = (IEnumerable>)_db.Query(selector.RawSql, new { search = $"%{search}%", exactsearch = $"{search}" }); + var resultFromQuery = (IEnumerable>)_db.Query(selector.RawSql, new { search = $"%{search}%", exactsearch = $"{search}" }, transaction: transaction); var items = new List(); if (resultFromQuery == null) @@ -209,7 +210,30 @@ public IEnumerable QueryContentItems(Guid contentTypeId, string[] s return items; } - public int CountContentItems(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters) + public IEnumerable QueryAllContentItemsAsTransaction(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, string orderBy) + { + if (_db.State == ConnectionState.Closed) + { + _db.Open(); + } + using (IDbTransaction transaction = _db.BeginTransaction(IsolationLevel.Snapshot)) + { + int count = CountContentItems(contentTypeId, searchOnColumns, search, filters, transaction: transaction); + + int totalPages = (int)Math.Ceiling((double)count / View.DEFAULT_MAX_ITEMS_PER_PAGE); + for (int pageIndex = 1; pageIndex <= totalPages; pageIndex++) + { + foreach (var item in QueryContentItems(contentTypeId, searchOnColumns, search, filters, View.DEFAULT_MAX_ITEMS_PER_PAGE, pageIndex, orderBy, transaction: transaction)) + { + yield return item; + } + } + + transaction.Commit(); + } + } + + public int CountContentItems(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, IDbTransaction transaction = null) { var contentType = _entityFramework.ContentTypes .Include(p => p.ContentTypeFields) @@ -255,7 +279,7 @@ public int CountContentItems(Guid contentTypeId, string[] searchOnColumns, strin var selector = sqlBuilder.AddTemplate($"SELECT COUNT(*) as Count {fromClause} /**where**/"); - var numResults = _db.Query(selector.RawSql, new { search = $"%{search}%", exactsearch = $"{search}" }).First().Count; + var numResults = _db.Query(selector.RawSql, new { search = $"%{search}%", exactsearch = $"{search}" }, transaction: transaction).First().Count; return numResults; } @@ -430,7 +454,7 @@ private string CreateSqlOrderByStatement(ContentType contentType, string orderBy try { var column = orderByElement.Split(" ")[0]; - var direction = SortOrder.From(orderByElement.Split(" ")[1]); + var direction = Raytha.Domain.ValueObjects.SortOrder.From(orderByElement.Split(" ")[1]); var columnAsContentTypeField = contentType.ContentTypeFields.FirstOrDefault(p => p.DeveloperName == column); if (columnAsContentTypeField != null) diff --git a/src/Raytha.Web/Areas/Admin/Controllers/ContentItemsController.cs b/src/Raytha.Web/Areas/Admin/Controllers/ContentItemsController.cs index 4283c56c..e0ccbe6b 100644 --- a/src/Raytha.Web/Areas/Admin/Controllers/ContentItemsController.cs +++ b/src/Raytha.Web/Areas/Admin/Controllers/ContentItemsController.cs @@ -490,7 +490,8 @@ public async Task BeginExportToCsv(ContentItemsExportToCsv_ViewMo { var input = new BeginExportContentItemsToCsv.Command { - + ViewId = CurrentView.Id, + ExportOnlyColumnsFromView = model.ViewColumnsOnly }; var response = await Mediator.Send(input);