Finishes core functionality for export to csv #79 #80

src/Raytha.Application/Common/Interfaces/IExecuteBackgroundTask.cs

@@ -1,5 +1,7 @@
-namespace Raytha.Application.Common.Interfaces;
+using System.Text.Json;
+
+namespace Raytha.Application.Common.Interfaces;
 public interface IExecuteBackgroundTask
 {
-    Task Execute(Guid jobId, dynamic args, CancellationToken cancellationToken);
+    Task Execute(Guid jobId, JsonElement args, CancellationToken cancellationToken);
 }

src/Raytha.Application/Common/Interfaces/IRaythaDbJsonQueryEngine.cs

@@ -1,4 +1,5 @@
 using Raytha.Domain.Entities;
+using System.Data;
 
 namespace Raytha.Application.Common.Interfaces;
 
@@ -11,9 +12,14 @@ public interface IRaythaDbJsonQueryEngine
                                                string[] filters,
                                                int pageSize,
                                                int pageNumber,
-                                               string orderBy);
+                                               string orderBy,
+                                               IDbTransaction transaction = null);
+
+    IEnumerable<ContentItem> QueryAllContentItemsAsTransaction(Guid contentTypeId, string[] searchOnColumns, string search, string[] filters, string orderBy);
+
     int CountContentItems(Guid contentTypeId,
                           string[] searchOnColumns,
                           string search,
-                          string[] filters);
+                          string[] filters,
+                          IDbTransaction transaction = null);
 }

src/Raytha.Application/ContentItems/Commands/BeginExportContentItemsToCsv.cs

@@ -2,25 +2,62 @@
 using Raytha.Application.Common.Models;
 using CSharpVitamins;
 using Raytha.Application.Common.Interfaces;
+using FluentValidation;
+using Raytha.Application.Common.Exceptions;
+using Raytha.Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using Raytha.Domain.ValueObjects;
+using Raytha.Application.Common.Utils;
+using Csv;
+using Raytha.Domain.Common;
+using Raytha.Domain.ValueObjects.FieldValues;
+using System.Text.Json;
 
 namespace Raytha.Application.ContentItems.Commands;
 
 public class BeginExportContentItemsToCsv
 {
     public record Command : LoggableRequest<CommandResponseDto<ShortGuid>>
     {
+        public ShortGuid ViewId { get; init; }
+        public bool ExportOnlyColumnsFromView { get; init; } = true;
+    }
 
+    public class Validator : AbstractValidator<Command>
+    {
+        public Validator()
+        {
+            RuleFor(x => x.ViewId).NotEmpty();
+        }
     }
 
     public class Handler : IRequestHandler<Command, CommandResponseDto<ShortGuid>>
     {
         private readonly IBackgroundTaskQueue _taskQueue;
-        public Handler(IBackgroundTaskQueue taskQueue)
+        private readonly IRaythaDbContext _entityFrameworkDb;
+        private readonly IContentTypeInRoutePath _contentTypeInRoutePath;
+        public Handler(
+            IBackgroundTaskQueue taskQueue,
+            IRaythaDbContext entityFrameworkDb,
+            IContentTypeInRoutePath contentTypeInRoutePath
+            )
         {
             _taskQueue = taskQueue;
+            _entityFrameworkDb = entityFrameworkDb;
+            _contentTypeInRoutePath = contentTypeInRoutePath;
         }
         public async Task<CommandResponseDto<ShortGuid>> Handle(Command request, CancellationToken cancellationToken)
         {
+            View view = _entityFrameworkDb.Views
+                .Include(p => p.ContentType)
+                .ThenInclude(p => p.ContentTypeFields)
+                .FirstOrDefault(p => p.Id == request.ViewId.Guid);
+
+            if (view == null)
+                throw new NotFoundException("View", request.ViewId);
+
+            _contentTypeInRoutePath.ValidateContentTypeInRoutePathMatchesValue(view.ContentType.DeveloperName);
+
             var backgroundJobId = await _taskQueue.EnqueueAsync<BackgroundTask>(request, cancellationToken);
 
             return new CommandResponseDto<ShortGuid>(backgroundJobId);
@@ -29,31 +66,182 @@ public async Task<CommandResponseDto<ShortGuid>> Handle(Command request, Cancell
 
     public class BackgroundTask : IExecuteBackgroundTask
     {
-        private readonly IRaythaDbContext _db;
-        public BackgroundTask(IRaythaDbContext db)
+        private readonly IRaythaDbJsonQueryEngine _db;
+        private readonly IRaythaDbContext _entityFrameworkDb;
+        private readonly IContentTypeInRoutePath _contentTypeInRoutePath;
+        private readonly ICurrentOrganization _currentOrganization;
+        private readonly IFileStorageProvider _fileStorageProvider;
+
+        public BackgroundTask(
+            ICurrentOrganization currentOrganization, 
+            IRaythaDbJsonQueryEngine db, 
+            IRaythaDbContext entityFrameworkDb, 
+            IFileStorageProvider fileStorageProvider)
         {
             _db = db;
+            _entityFrameworkDb = entityFrameworkDb;
+            _currentOrganization = currentOrganization;
+            _fileStorageProvider = fileStorageProvider;
         }
-        public async Task Execute(Guid jobId, dynamic args, CancellationToken cancellationToken)
+        public async Task Execute(Guid jobId, JsonElement args, CancellationToken cancellationToken)
         {
-            int delayLoop = 0;
-            var guid = Guid.NewGuid().ToString();
-            Random rnd = new Random();
-            var job = _db.BackgroundTasks.First(p => p.Id == jobId);
-            while (delayLoop < 3)
+            IEnumerable<ContentItemDto> items;
+
+            Guid viewId = args.GetProperty("ViewId").GetProperty("Guid").GetGuid();
+            bool exportOnlyColumnsFromView = args.GetProperty("ExportOnlyColumnsFromView").GetBoolean();
+
+            View view = _entityFrameworkDb.Views
+                .Include(p => p.ContentType)
+                .ThenInclude(p => p.ContentTypeFields)
+                .FirstOrDefault(p => p.Id == viewId);
+
+            if (view == null)
+                throw new NotFoundException("View", viewId);
+
+            string[] filters = GetFiltersForView(view);
+            string finalOrderBy = GetSortForView(view);
+
+            int count = _db.CountContentItems(view.ContentTypeId, new string[0], string.Empty, filters);
+
+            var job = _entityFrameworkDb.BackgroundTasks.First(p => p.Id == jobId);
+            job.StatusInfo = $"Pulling {count} items for {view.ContentType.LabelPlural} - {view.Label}";
+            job.PercentComplete = 0;
+            _entityFrameworkDb.BackgroundTasks.Update(job);
+            await _entityFrameworkDb.SaveChangesAsync(cancellationToken);
+
+            var myExport = new CsvExport();
+            int currentIndex = 0;
+            foreach (var item in _db.QueryAllContentItemsAsTransaction(view.ContentTypeId,
+                                                new string[0],
+                                                string.Empty,
+                                                filters,
+                                                finalOrderBy))
             {
-                Thread.Sleep(rnd.Next(1000, 10000));
-                delayLoop++;
-                job.StatusInfo = $"Made it here: {delayLoop}";
-                job.PercentComplete = delayLoop * 10;
-                _db.BackgroundTasks.Update(job);
-                await _db.SaveChangesAsync(cancellationToken);
+                var contentItemAsDict = MapToListItemValues(ContentItemDto.GetProjection(item));
+
+                myExport.AddRow();
+                if (exportOnlyColumnsFromView)
+                {
+                    foreach (var column in view.Columns)
+                    {
+                        if (contentItemAsDict.ContainsKey(column))
+                        {
+                            myExport[column] = contentItemAsDict[column];
+                        }
+                        else
+                        {
+                            myExport[column] = string.Empty;
+                        }
+                    }
+                }
+                else
+                {
+                    foreach (var column in BuiltInContentTypeField.ReservedContentTypeFields)
+                    {
+                        if (contentItemAsDict.ContainsKey(column))
+                        {
+                            myExport[column] = contentItemAsDict[column];
+                        }
+                        else
+                        {
+                            myExport[column] = string.Empty;
+                        }
+                    }
+                    foreach (var column in view.ContentType.ContentTypeFields.Select(p => p.DeveloperName))
+                    {
+                        if (contentItemAsDict.ContainsKey(column))
+                        {
+                            myExport[column] = contentItemAsDict[column];
+                        }
+                        else
+                        {
+                            myExport[column] = string.Empty;
+                        }
+                    }
+                }
+
+                currentIndex++;
+                int percentDone = (currentIndex / count) * 100;
+                if (percentDone % 20 == 0) 
+                {
+                    job.PercentComplete = percentDone - 10;
+                    _entityFrameworkDb.BackgroundTasks.Update(job);
+                    await _entityFrameworkDb.SaveChangesAsync(cancellationToken);
+                }
             }
 
+            job.StatusInfo = $"Items pulled. Generating csv...";
+            _entityFrameworkDb.BackgroundTasks.Update(job);
+            await _entityFrameworkDb.SaveChangesAsync(cancellationToken);
+
+            var csvAsBytes = myExport.ExportToBytes();
+
+            string fileName = $"{_currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(DateTime.UtcNow)}-{view.DeveloperName}";
+            string downloadUrl = await _fileStorageProvider.SaveAndGetDownloadUrlAsync(csvAsBytes, Guid.NewGuid().ToString(), fileName, "text/csv", DateTime.UtcNow.AddYears(999));
+
             job.PercentComplete = 100;
-            //job.DownloadUrl = "https://s3.amazonaws.com/vo-random/ShareX/2023/05/chrome_f7gOblEvFX.png";
-            _db.BackgroundTasks.Update(job);
-            await _db.SaveChangesAsync(cancellationToken);
+            job.DownloadUrl = downloadUrl;
+            _entityFrameworkDb.BackgroundTasks.Update(job);
+            await _entityFrameworkDb.SaveChangesAsync(cancellationToken);
+        }
+
+        protected string GetSortForView(View view)
+        {
+            string finalOrderBy = $"{BuiltInContentTypeField.CreationTime.DeveloperName} {SortOrder.DESCENDING}";
+            var viewOrderBy = view.Sort.Select(p => $"{p.DeveloperName} {p.SortOrder.DeveloperName}").ToList();
+            finalOrderBy = viewOrderBy.Any() ? string.Join(",", viewOrderBy) : finalOrderBy;
+
+            return finalOrderBy;
+        }
+
+        protected string[] GetFiltersForView(View view)
+        {
+            var conditionToODataUtility = new FilterConditionToODataUtility(view.ContentType);
+            var oDataFromFilter = conditionToODataUtility.ToODataFilter(view.Filter);
+            return new string[] { oDataFromFilter };
+        }
+
+        private Dictionary<string, string> MapToListItemValues(ContentItemDto item)
+        {
+            var viewModel = new Dictionary<string, string>
+            {
+                //Built in
+                { BuiltInContentTypeField.Id, item.Id },
+                { "CreatorUser", item.CreatorUser != null ? item.CreatorUser.FullName : "N/A" },
+                { "LastModifierUser", item.LastModifierUser != null ? item.LastModifierUser.FullName : "N/A" },
+                { BuiltInContentTypeField.CreationTime, _currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(item.CreationTime) },
+                { BuiltInContentTypeField.LastModificationTime, _currentOrganization.TimeZoneConverter.UtcToTimeZoneAsDateTimeFormat(item.LastModificationTime) },
+                { BuiltInContentTypeField.IsPublished, item.IsPublished.YesOrNo() },
+                { BuiltInContentTypeField.IsDraft, item.IsDraft.YesOrNo() },
+                { BuiltInContentTypeField.PrimaryField, item.PrimaryField },
+                { "Template", item.WebTemplate.Label }
+            };
+
+            //Content type fields
+            foreach (var field in item.PublishedContent as Dictionary<string, dynamic>)
+            {
+                if (field.Value is DateTimeFieldValue dateTimeFieldValue)
+                {
+                    viewModel.Add(field.Key, _currentOrganization.TimeZoneConverter.ToDateFormat(dateTimeFieldValue.Value));
+                }
+                else if (field.Value is GuidFieldValue guidFieldValue)
+                {
+                    if (guidFieldValue.HasValue)
+                        viewModel.Add(field.Key, (ShortGuid)guidFieldValue.Value);
+                    else
+                        viewModel.Add(field.Key, string.Empty);
+                }
+                else if (field.Value is IBaseEntity)
+                {
+                    viewModel.Add(field.Key, field.Value.PrimaryField);
+                }
+                else
+                {
+                    viewModel.Add(field.Key, field.Value.ToString());
+                }
+            }
+
+            return viewModel;
         }
     }
 }

src/Raytha.Application/Raytha.Application.csproj

@@ -15,6 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="CSharpVitamins.ShortGuid" Version="2.0.0" />
+    <PackageReference Include="CsvExport" Version="2.0.0" />
     <PackageReference Include="MediatR" Version="11.1.0" />
 	<PackageReference Include="MediatR.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />

src/Raytha.Infrastructure/BackgroundTasks/BackgroundTaskQueue.cs

@@ -1,5 +1,4 @@
-using Microsoft.EntityFrameworkCore;
-using Raytha.Application.Common.Interfaces;
+using Raytha.Application.Common.Interfaces;
 using Raytha.Domain.Entities;
 using System.Text.Json;
 

src/Raytha.Infrastructure/BackgroundTasks/QueuedHostedService.cs

@@ -5,6 +5,7 @@
 using Raytha.Application.Common.Interfaces;
 using Raytha.Domain.Entities;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Raytha.Infrastructure.BackgroundTasks;
 
@@ -50,7 +51,7 @@ private async Task BackgroundProcessing(CancellationToken stoppingToken)
                         scope.ServiceProvider
                             .GetRequiredService(Type.GetType(backgroundTask.Name)) as IExecuteBackgroundTask;
 
-                    await scopedProcessingService.Execute(backgroundTask.Id, JsonSerializer.Deserialize<dynamic>(backgroundTask.Args), stoppingToken);
+                    await scopedProcessingService.Execute(backgroundTask.Id, JsonSerializer.Deserialize<JsonElement>(backgroundTask.Args), stoppingToken);
                     backgroundTask.Status = BackgroundTaskStatus.Complete;
                 }
                 catch (Exception ex)