From 566ed1cce21643ca1d3c7ac2a67c8395ca507a58 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Wed, 24 Feb 2021 15:36:33 -0600 Subject: [PATCH 01/14] Direct upload to azure To validate file sizes in the event of a rogue client, Azure event webhooks will be hooked up to AzureValidateFile. Sends outside of a grace size will be deleted as non-compliant. TODO: LocalSendFileStorageService direct upload method/endpoint. --- src/Api/Api.csproj | 1 + src/Api/Controllers/SendsController.cs | 93 ++++++++++++++---- src/Api/Utilities/ApiHelpers.cs | 42 ++++++++ src/Core/Enums/FileUploadType.cs | 8 ++ .../Models/Api/Request/SendRequestModel.cs | 1 + .../SendFileUploadDataResponseModel.cs | 13 +++ src/Core/Services/ISendService.cs | 4 +- src/Core/Services/ISendStorageService.cs | 10 +- .../AzureSendFileStorageService.cs | 69 +++++++++++-- .../LocalSendStorageService.cs | 11 ++- .../Services/Implementations/SendService.cs | 96 +++++++++++-------- .../NoopSendFileStorageService.cs | 17 +++- 12 files changed, 291 insertions(+), 74 deletions(-) create mode 100644 src/Core/Enums/FileUploadType.cs create mode 100644 src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs diff --git a/src/Api/Api.csproj b/src/Api/Api.csproj index 5ee677cbeef8..1373497c7128 100644 --- a/src/Api/Api.csproj +++ b/src/Api/Api.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 0fee3e19b4f9..dc187787b2c6 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -7,11 +7,13 @@ using Bit.Core.Models.Api; using Bit.Core.Exceptions; using Bit.Core.Services; -using Bit.Api.Utilities; -using Bit.Core.Models.Table; using Bit.Core.Utilities; using Bit.Core.Settings; using Bit.Core.Models.Api.Response; +using Bit.Core.Enums; +using Microsoft.Azure.EventGrid.Models; +using Bit.Api.Utilities; +using System.Collections.Generic; namespace Bit.Api.Controllers { @@ -64,13 +66,20 @@ public async Task Access(string id, [FromBody] SendAccessRequestM } [AllowAnonymous] - [HttpGet("access/file/{id}")] - public async Task GetSendFileDownloadData(string id) + [HttpGet("{sendId}/access/file/{fileId}")] + public async Task GetSendFileDownloadData(string sendId, string fileId) { + var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); + + if (send == null) + { + throw new BadRequestException("Could not locate send"); + } + return new SendFileDownloadDataResponseModel() { - Id = id, - Url = await _sendFileStorageService.GetSendFileDownloadUrlAsync(id), + Id = fileId, + Url = await _sendFileStorageService.GetSendFileDownloadUrlAsync(send, fileId), }; } @@ -107,31 +116,73 @@ public async Task Post([FromBody] SendRequestModel model) } [HttpPost("file")] - [RequestSizeLimit(105_906_176)] - [DisableFormValueModelBinding] - public async Task PostFile() + public async Task PostFile([FromBody] SendRequestModel model) { - if (!Request?.ContentType.Contains("multipart/") ?? true) + if (model.Type != SendType.File) { throw new BadRequestException("Invalid content."); } - if (Request.ContentLength > 105906176) // 101 MB, give em' 1 extra MB for cushion + if (!model.FileLength.HasValue) { - throw new BadRequestException("Max file size is 100 MB."); + throw new BadRequestException("Invalid content. File size hint is required."); } - Send send = null; - await Request.GetSendFileAsync(async (stream, fileName, model) => + var userId = _userService.GetProperUserId(User).Value; + var (send, data) = model.ToSend(userId, model.File.FileName, _sendService); + var uploadUrl = await _sendService.SaveFileSendAsync(send, data, model.FileLength.Value); + return new SendFileUploadDataResponseModel { - model.ValidateCreation(); - var userId = _userService.GetProperUserId(User).Value; - var (madeSend, madeData) = model.ToSend(userId, fileName, _sendService); - send = madeSend; - await _sendService.CreateSendAsync(send, madeData, stream, Request.ContentLength.GetValueOrDefault(0)); - }); + Url = uploadUrl, + FileUploadType = _sendFileStorageService.FileUploadType, + SendResponse = new SendResponseModel(send, _globalSettings) + }; + } - return new SendResponseModel(send, _globalSettings); + [AllowAnonymous] + [HttpPost("file/validate/azure")] + [HttpGet("file/validate/azure")] + public async Task AzureValidateFile() + { + return await ApiHelpers.HandleAzureEvents(Request, new Dictionary> + { + {"Microsoft.Storage.BlobCreated", async (eventGridEvent) => { + try + { + var blobName = eventGridEvent.Subject.Split($"{AzureSendFileStorageService.FilesContainerName}/blobs/")[1]; + var sendId = AzureSendFileStorageService.SendIdFromBlobName(blobName); + var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); + if (send == null) + { + return; + } + await _sendService.ValidateSendFile(send); + } + catch + { + return; + } + }} + }); + } + + [HttpPost("{id}/validate")] + public async Task PostValidateFile(string id) + { + var userId = _userService.GetProperUserId(User).Value; + var send = await _sendRepository.GetByIdAsync(new Guid(id)); + + if (send == null || send.UserId != userId) + { + throw new NotFoundException(); + } + + if (send.Type != SendType.File) + { + throw new BadRequestException("Invalid content."); + } + + await _sendService.ValidateSendFile(send); } [HttpPut("{id}")] diff --git a/src/Api/Utilities/ApiHelpers.cs b/src/Api/Utilities/ApiHelpers.cs index 8aef098b52b5..cb8868debb98 100644 --- a/src/Api/Utilities/ApiHelpers.cs +++ b/src/Api/Utilities/ApiHelpers.cs @@ -1,5 +1,10 @@ using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.EventGrid; +using Microsoft.Azure.EventGrid.Models; using Newtonsoft.Json; +using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -29,5 +34,42 @@ public async static Task ReadJsonFileFromBody(HttpContext httpContext, IFo return obj; } + + /// + /// Validates Azure event subscription and calls the appropriate event handler. Responds HttpOk. + /// + /// HttpRequest received from Azure + /// Dictionary of eventType strings and their associated handlers. + /// OkObjectResult + // Reference https://docs.microsoft.com/en-us/azure/event-grid/receive-events + public async static Task HandleAzureEvents(HttpRequest request, + Dictionary> eventTypeHandlers) + { + var response = string.Empty; + var requestContent = await new StreamReader(request.Body).ReadToEndAsync(); + var eventGridSubscriber = new EventGridSubscriber(); + var eventGridEvents = eventGridSubscriber.DeserializeEventGridEvents(requestContent); + + foreach (var eventGridEvent in eventGridEvents) + { + if (eventGridEvent.Data is SubscriptionValidationEventData eventData) + { + // Might want to enable additional validation: subject, topic etc. + + var responseData = new SubscriptionValidationResponse() + { + ValidationResponse = eventData.ValidationCode + }; + + return new OkObjectResult(responseData); + } + else if (eventTypeHandlers.ContainsKey(eventGridEvent.EventType)) + { + await eventTypeHandlers[eventGridEvent.EventType](eventGridEvent); + } + } + + return new OkObjectResult(response); + } } } diff --git a/src/Core/Enums/FileUploadType.cs b/src/Core/Enums/FileUploadType.cs new file mode 100644 index 000000000000..dc50eb669620 --- /dev/null +++ b/src/Core/Enums/FileUploadType.cs @@ -0,0 +1,8 @@ +namespace Bit.Core.Enums +{ + public enum FileUploadType + { + Direct = 0, + Azure = 1, + } +} diff --git a/src/Core/Models/Api/Request/SendRequestModel.cs b/src/Core/Models/Api/Request/SendRequestModel.cs index ccb27b0cc56b..d64faef176a8 100644 --- a/src/Core/Models/Api/Request/SendRequestModel.cs +++ b/src/Core/Models/Api/Request/SendRequestModel.cs @@ -13,6 +13,7 @@ namespace Bit.Core.Models.Api public class SendRequestModel { public SendType Type { get; set; } + public long? FileLength { get; set; } = null; [EncryptedString] [EncryptedStringLength(1000)] public string Name { get; set; } diff --git a/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs b/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs new file mode 100644 index 000000000000..7e4b95ddbf30 --- /dev/null +++ b/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs @@ -0,0 +1,13 @@ +using Bit.Core.Enums; + +namespace Bit.Core.Models.Api.Response +{ + public class SendFileUploadDataResponseModel : ResponseModel + { + public string Url { get; set; } + public FileUploadType FileUploadType { get; set; } + public SendResponseModel SendResponse { get; set; } + + public SendFileUploadDataResponseModel() : base("send-fileUpload") { } + } +} diff --git a/src/Core/Services/ISendService.cs b/src/Core/Services/ISendService.cs index 6b44a3824f00..bac3bd4ad199 100644 --- a/src/Core/Services/ISendService.cs +++ b/src/Core/Services/ISendService.cs @@ -10,8 +10,10 @@ public interface ISendService { Task DeleteSendAsync(Send send); Task SaveSendAsync(Send send); - Task CreateSendAsync(Send send, SendFileData data, Stream stream, long requestLength); + Task SaveFileSendAsync(Send send, SendFileData data, long fileLength); Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password); string HashPassword(string password); + Task ValidateSendFile(Send send); + } } diff --git a/src/Core/Services/ISendStorageService.cs b/src/Core/Services/ISendStorageService.cs index 1b4c5cc3718c..00fd16a29ff0 100644 --- a/src/Core/Services/ISendStorageService.cs +++ b/src/Core/Services/ISendStorageService.cs @@ -1,4 +1,5 @@ -using Bit.Core.Models.Table; +using Bit.Core.Enums; +using Bit.Core.Models.Table; using System; using System.IO; using System.Threading.Tasks; @@ -7,10 +8,13 @@ namespace Bit.Core.Services { public interface ISendFileStorageService { + FileUploadType FileUploadType { get; } Task UploadNewFileAsync(Stream stream, Send send, string fileId); - Task DeleteFileAsync(string fileId); + Task DeleteFileAsync(Send send, string fileId); Task DeleteFilesForOrganizationAsync(Guid organizationId); Task DeleteFilesForUserAsync(Guid userId); - Task GetSendFileDownloadUrlAsync(string fileId); + Task GetSendFileDownloadUrlAsync(Send send, string fileId); + Task GetSendFileUploadUrlAsync(Send send, string fileId); + Task ValidateFile(Send send, string fileId, long expectedFileSize); } } diff --git a/src/Core/Services/Implementations/AzureSendFileStorageService.cs b/src/Core/Services/Implementations/AzureSendFileStorageService.cs index 50edc58cb3fa..1a12bb4e5243 100644 --- a/src/Core/Services/Implementations/AzureSendFileStorageService.cs +++ b/src/Core/Services/Implementations/AzureSendFileStorageService.cs @@ -5,17 +5,24 @@ using System; using Bit.Core.Models.Table; using Bit.Core.Settings; +using Bit.Core.Enums; namespace Bit.Core.Services { public class AzureSendFileStorageService : ISendFileStorageService { - private const string FilesContainerName = "sendfiles"; + public const string FilesContainerName = "sendfiles"; + private const long FileSizeValidationGrace = 1024L; private static readonly TimeSpan _downloadLinkLiveTime = TimeSpan.FromMinutes(1); private readonly CloudBlobClient _blobClient; private CloudBlobContainer _sendFilesContainer; + public FileUploadType FileUploadType => FileUploadType.Azure; + + public static string SendIdFromBlobName(string blobName) => blobName.Split('/')[0]; + public static string BlobName(Send send, string fileId) => $"{send.Id}/{fileId}"; + public AzureSendFileStorageService( GlobalSettings globalSettings) { @@ -26,7 +33,7 @@ public AzureSendFileStorageService( public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) { await InitAsync(); - var blob = _sendFilesContainer.GetBlockBlobReference(fileId); + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); if (send.UserId.HasValue) { blob.Metadata.Add("userId", send.UserId.Value.ToString()); @@ -39,10 +46,10 @@ public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) await blob.UploadFromStreamAsync(stream); } - public async Task DeleteFileAsync(string fileId) + public async Task DeleteFileAsync(Send send, string fileId) { await InitAsync(); - var blob = _sendFilesContainer.GetBlockBlobReference(fileId); + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); await blob.DeleteIfExistsAsync(); } @@ -56,19 +63,67 @@ public async Task DeleteFilesForUserAsync(Guid userId) await InitAsync(); } - public async Task GetSendFileDownloadUrlAsync(string fileId) + public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) { await InitAsync(); - var blob = _sendFilesContainer.GetBlockBlobReference(fileId); + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); var accessPolicy = new SharedAccessBlobPolicy() { SharedAccessExpiryTime = DateTime.UtcNow.Add(_downloadLinkLiveTime), - Permissions = SharedAccessBlobPermissions.Read + Permissions = SharedAccessBlobPermissions.Read, }; return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); } + public async Task GetSendFileUploadUrlAsync(Send send, string fileId) + { + await InitAsync(); + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); + + var accessPolicy = new SharedAccessBlobPolicy() + { + SharedAccessExpiryTime = DateTime.UtcNow.Add(_downloadLinkLiveTime), + Permissions = SharedAccessBlobPermissions.Create, + }; + + return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); + } + + public async Task ValidateFile(Send send, string fileId, long expectedFileSize) + { + await InitAsync(); + + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); + + if (!blob.Exists()) + { + return false; + } + + blob.FetchAttributes(); + + if (send.UserId.HasValue) + { + blob.Metadata["userId"] = send.UserId.Value.ToString(); + } + else + { + blob.Metadata["organizationId"] = send.OrganizationId.Value.ToString(); + } + blob.Properties.ContentDisposition = $"attachment; filename=\"{fileId}\""; + blob.SetMetadata(); + blob.SetProperties(); + + var length = blob.Properties.Length; + if (length < expectedFileSize - FileSizeValidationGrace || length > expectedFileSize + FileSizeValidationGrace) + { + return false; + } + + return true; + } + private async Task InitAsync() { if (_sendFilesContainer == null) diff --git a/src/Core/Services/Implementations/LocalSendStorageService.cs b/src/Core/Services/Implementations/LocalSendStorageService.cs index a015149e8a29..ddeefd318514 100644 --- a/src/Core/Services/Implementations/LocalSendStorageService.cs +++ b/src/Core/Services/Implementations/LocalSendStorageService.cs @@ -3,6 +3,7 @@ using System; using Bit.Core.Models.Table; using Bit.Core.Settings; +using Bit.Core.Enums; namespace Bit.Core.Services { @@ -11,6 +12,8 @@ public class LocalSendStorageService : ISendFileStorageService private readonly string _baseDirPath; private readonly string _baseSendUrl; + public FileUploadType FileUploadType => FileUploadType.Direct; + public LocalSendStorageService( GlobalSettings globalSettings) { @@ -28,7 +31,7 @@ public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) } } - public async Task DeleteFileAsync(string fileId) + public async Task DeleteFileAsync(Send send, string fileId) { await InitAsync(); DeleteFileIfExists($"{_baseDirPath}/{fileId}"); @@ -44,7 +47,7 @@ public async Task DeleteFilesForUserAsync(Guid userId) await InitAsync(); } - public async Task GetSendFileDownloadUrlAsync(string fileId) + public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) { await InitAsync(); return $"{_baseSendUrl}/{fileId}"; @@ -67,5 +70,9 @@ private Task InitAsync() return Task.FromResult(0); } + + public Task GetSendFileUploadUrlAsync(Send send, string fileId) => throw new NotImplementedException(); + public Task ValidateFile(Send send, string fileId, long expectedFileSize) => throw new NotImplementedException(); + } } diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index 29592617168d..113cbd8ead86 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -69,81 +69,64 @@ public async Task SaveSendAsync(Send send) } } - public async Task CreateSendAsync(Send send, SendFileData data, Stream stream, long requestLength) + public async Task SaveFileSendAsync(Send send, SendFileData data, long fileLength) { if (send.Type != SendType.File) { throw new BadRequestException("Send is not of type \"file\"."); } - if (requestLength < 1) + if (fileLength < 1) { throw new BadRequestException("No file data."); } - var storageBytesRemaining = 0L; - if (send.UserId.HasValue) - { - var user = await _userRepository.GetByIdAsync(send.UserId.Value); - if (!(await _userService.CanAccessPremium(user))) - { - throw new BadRequestException("You must have premium status to use file sends."); - } - - if (user.Premium) - { - storageBytesRemaining = user.StorageBytesRemaining(); - } - else - { - // Users that get access to file storage/premium from their organization get the default - // 1 GB max storage. - storageBytesRemaining = user.StorageBytesRemaining( - _globalSettings.SelfHosted ? (short)10240 : (short)1); - } - } - else if (send.OrganizationId.HasValue) - { - var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value); - if (!org.MaxStorageGb.HasValue) - { - throw new BadRequestException("This organization cannot use file sends."); - } + var storageBytesRemaining = await StorageRemainingForSendAsync(send); - storageBytesRemaining = org.StorageBytesRemaining(); - } - - if (storageBytesRemaining < requestLength) + if (storageBytesRemaining < fileLength) { throw new BadRequestException("Not enough storage available."); } var fileId = Utilities.CoreHelpers.SecureRandomString(32, upper: false, special: false); - await _sendFileStorageService.UploadNewFileAsync(stream, send, fileId); try { data.Id = fileId; - data.Size = stream.Length; + data.Size = fileLength; send.Data = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); await SaveSendAsync(send); + return await _sendFileStorageService.GetSendFileUploadUrlAsync(send, fileId); } catch { // Clean up since this is not transactional - await _sendFileStorageService.DeleteFileAsync(fileId); + await _sendFileStorageService.DeleteFileAsync(send, fileId); throw; } } + public async Task ValidateSendFile(Send send) + { + var fileData = JsonConvert.DeserializeObject(send.Data); + + var valid = await _sendFileStorageService.ValidateFile(send, fileData.Id, fileData.Size); + + if (!valid) + { + // File reported differs in size from that promised. Must be a rogue client. Delete Send + await DeleteSendAsync(send); + } + } + public async Task DeleteSendAsync(Send send) { await _sendRepository.DeleteAsync(send); if (send.Type == Enums.SendType.File) { var data = JsonConvert.DeserializeObject(send.Data); - await _sendFileStorageService.DeleteFileAsync(data.Id); + await _sendFileStorageService.DeleteFileAsync(send, data.Id); } await _pushService.PushSyncSendDeleteAsync(send); } @@ -208,5 +191,42 @@ private async Task ValidateUserCanSaveAsync(Guid? userId) } } } + + private async Task StorageRemainingForSendAsync(Send send) + { + var storageBytesRemaining = 0L; + if (send.UserId.HasValue) + { + var user = await _userRepository.GetByIdAsync(send.UserId.Value); + if (!await _userService.CanAccessPremium(user)) + { + throw new BadRequestException("You must have premium status to use file sends."); + } + + if (user.Premium) + { + storageBytesRemaining = user.StorageBytesRemaining(); + } + else + { + // Users that get access to file storage/premium from their organization get the default + // 1 GB max storage. + storageBytesRemaining = user.StorageBytesRemaining( + _globalSettings.SelfHosted ? (short)10240 : (short)1); + } + } + else if (send.OrganizationId.HasValue) + { + var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value); + if (!org.MaxStorageGb.HasValue) + { + throw new BadRequestException("This organization cannot use file sends."); + } + + storageBytesRemaining = org.StorageBytesRemaining(); + } + + return storageBytesRemaining; + } } } diff --git a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs index 2b174ce36fbb..8f8cab44452b 100644 --- a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs +++ b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs @@ -2,17 +2,20 @@ using System.IO; using System; using Bit.Core.Models.Table; +using Bit.Core.Enums; namespace Bit.Core.Services { public class NoopSendFileStorageService : ISendFileStorageService { + public FileUploadType FileUploadType => FileUploadType.Direct; + public Task UploadNewFileAsync(Stream stream, Send send, string attachmentId) { return Task.FromResult(0); } - public Task DeleteFileAsync(string fileId) + public Task DeleteFileAsync(Send send, string fileId) { return Task.FromResult(0); } @@ -27,9 +30,19 @@ public Task DeleteFilesForUserAsync(Guid userId) return Task.FromResult(0); } - public Task GetSendFileDownloadUrlAsync(string fileId) + public Task GetSendFileDownloadUrlAsync(Send send, string fileId) + { + return Task.FromResult((string)null); + } + + public Task GetSendFileUploadUrlAsync(Send send, string fileId) { return Task.FromResult((string)null); } + + public Task ValidateFile(Send send, string fileId, long expectedFileSize) + { + return Task.FromResult(false); + } } } From b87866f758222c5343b0bcb7ca86ee3d5ca0d070 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 09:11:26 -0600 Subject: [PATCH 02/14] Decode send Id from accessId --- src/Api/Controllers/SendsController.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index dc187787b2c6..d6fc2fcea5ba 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -66,10 +66,11 @@ public async Task Access(string id, [FromBody] SendAccessRequestM } [AllowAnonymous] - [HttpGet("{sendId}/access/file/{fileId}")] - public async Task GetSendFileDownloadData(string sendId, string fileId) + [HttpGet("{encodedSendId}/access/file/{fileId}")] + public async Task GetSendFileDownloadData(string encodedSendId, string fileId) { - var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); + var sendId = new Guid(CoreHelpers.Base64UrlDecode(encodedSendId)); + var send = await _sendRepository.GetByIdAsync(sendId); if (send == null) { From dc930653290d4893480ff26dc1d2e11a9a9ae839 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 09:12:14 -0600 Subject: [PATCH 03/14] Quick respond to no-body event calls These shouldn't happen, but might if some errant get requests occur --- src/Api/Utilities/ApiHelpers.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Api/Utilities/ApiHelpers.cs b/src/Api/Utilities/ApiHelpers.cs index cb8868debb98..2469868603d6 100644 --- a/src/Api/Utilities/ApiHelpers.cs +++ b/src/Api/Utilities/ApiHelpers.cs @@ -47,6 +47,11 @@ public async static Task HandleAzureEvents(HttpRequest request, { var response = string.Empty; var requestContent = await new StreamReader(request.Body).ReadToEndAsync(); + if (string.IsNullOrWhiteSpace(requestContent)) + { + return new OkObjectResult(response); + } + var eventGridSubscriber = new EventGridSubscriber(); var eventGridEvents = eventGridSubscriber.DeserializeEventGridEvents(requestContent); From 8af6a466e9e8750c2df36c226a52efae85cbbf99 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 09:46:28 -0600 Subject: [PATCH 04/14] Event Grid only POSTS to webhook --- src/Api/Controllers/SendsController.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index d6fc2fcea5ba..57343726add1 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -142,7 +142,6 @@ public async Task PostFile([FromBody] SendReque [AllowAnonymous] [HttpPost("file/validate/azure")] - [HttpGet("file/validate/azure")] public async Task AzureValidateFile() { return await ApiHelpers.HandleAzureEvents(Request, new Dictionary> From 41217c71040bd565ef7949b5110645e45e888716 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 11:50:09 -0600 Subject: [PATCH 05/14] Enable local storage direct file upload --- src/Api/Controllers/SendsController.cs | 40 ++++++++++--------- src/Api/Utilities/MultipartFormDataHelper.cs | 38 +++++------------- src/Core/Services/ISendService.cs | 1 + src/Core/Services/ISendStorageService.cs | 2 +- .../AzureSendFileStorageService.cs | 5 +-- .../LocalSendStorageService.cs | 8 +++- .../Services/Implementations/SendService.cs | 25 +++++++++++- .../NoopSendFileStorageService.cs | 2 +- 8 files changed, 66 insertions(+), 55 deletions(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 57343726add1..98d57a05b3a7 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -140,6 +140,27 @@ public async Task PostFile([FromBody] SendReque }; } + [HttpPost("{id}/file/{fileId}")] + [DisableFormValueModelBinding] + public async Task PostFileForExistingSend(string id, string fileId) + { + if (!Request?.ContentType.Contains("multipart/") ?? true) + { + throw new BadRequestException("Invalid content."); + } + + if (Request.ContentLength > 105906176) // 101 MB, give em' 1 extra MB for cushion + { + throw new BadRequestException("Max file size for direct upload is 100 MB."); + } + + var send = await _sendRepository.GetByIdAsync(new Guid(id)); + await Request.GetSendFileAsync(async (stream) => + { + await _sendFileStorageService.UploadNewFileAsync(stream, send, fileId); + }); + } + [AllowAnonymous] [HttpPost("file/validate/azure")] public async Task AzureValidateFile() @@ -166,25 +187,6 @@ public async Task AzureValidateFile() }); } - [HttpPost("{id}/validate")] - public async Task PostValidateFile(string id) - { - var userId = _userService.GetProperUserId(User).Value; - var send = await _sendRepository.GetByIdAsync(new Guid(id)); - - if (send == null || send.UserId != userId) - { - throw new NotFoundException(); - } - - if (send.Type != SendType.File) - { - throw new BadRequestException("Invalid content."); - } - - await _sendService.ValidateSendFile(send); - } - [HttpPut("{id}")] public async Task Put(string id, [FromBody] SendRequestModel model) { diff --git a/src/Api/Utilities/MultipartFormDataHelper.cs b/src/Api/Utilities/MultipartFormDataHelper.cs index 03ed0f1ae722..be81efcc8726 100644 --- a/src/Api/Utilities/MultipartFormDataHelper.cs +++ b/src/Api/Utilities/MultipartFormDataHelper.cs @@ -66,46 +66,28 @@ public static async Task GetFileAsync(this HttpRequest request, Func callback) + public static async Task GetSendFileAsync(this HttpRequest request, Func callback) { var boundary = GetBoundary(MediaTypeHeaderValue.Parse(request.ContentType), _defaultFormOptions.MultipartBoundaryLengthLimit); var reader = new MultipartReader(boundary, request.Body); - var firstSection = await reader.ReadNextSectionAsync(); - if (firstSection != null) + + var dataSection = await reader.ReadNextSectionAsync(); + if (dataSection != null) { - if (ContentDispositionHeaderValue.TryParse(firstSection.ContentDisposition, out _)) + if (ContentDispositionHeaderValue.TryParse(dataSection.ContentDisposition, + out var thirdContent) && HasFileContentDisposition(thirdContent)) { - // Request model json, then data - string requestModelJson = null; - using (var sr = new StreamReader(firstSection.Body)) + using (dataSection.Body) { - requestModelJson = await sr.ReadToEndAsync(); - } - - var secondSection = await reader.ReadNextSectionAsync(); - if (secondSection != null) - { - if (ContentDispositionHeaderValue.TryParse(secondSection.ContentDisposition, - out var secondContent) && HasFileContentDisposition(secondContent)) - { - var fileName = HeaderUtilities.RemoveQuotes(secondContent.FileName).ToString(); - using (secondSection.Body) - { - var model = JsonConvert.DeserializeObject(requestModelJson); - await callback(secondSection.Body, fileName, model); - } - } - - secondSection = null; + await callback(dataSection.Body); } - } - firstSection = null; } + + dataSection = null; } diff --git a/src/Core/Services/ISendService.cs b/src/Core/Services/ISendService.cs index bac3bd4ad199..8ef12caf01ee 100644 --- a/src/Core/Services/ISendService.cs +++ b/src/Core/Services/ISendService.cs @@ -11,6 +11,7 @@ public interface ISendService Task DeleteSendAsync(Send send); Task SaveSendAsync(Send send); Task SaveFileSendAsync(Send send, SendFileData data, long fileLength); + Task UploadFileToExistingSendAsync(Stream stream, Send send); Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password); string HashPassword(string password); Task ValidateSendFile(Send send); diff --git a/src/Core/Services/ISendStorageService.cs b/src/Core/Services/ISendStorageService.cs index 00fd16a29ff0..e225077619c1 100644 --- a/src/Core/Services/ISendStorageService.cs +++ b/src/Core/Services/ISendStorageService.cs @@ -15,6 +15,6 @@ public interface ISendFileStorageService Task DeleteFilesForUserAsync(Guid userId); Task GetSendFileDownloadUrlAsync(Send send, string fileId); Task GetSendFileUploadUrlAsync(Send send, string fileId); - Task ValidateFile(Send send, string fileId, long expectedFileSize); + Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway); } } diff --git a/src/Core/Services/Implementations/AzureSendFileStorageService.cs b/src/Core/Services/Implementations/AzureSendFileStorageService.cs index 1a12bb4e5243..480cecb61ebf 100644 --- a/src/Core/Services/Implementations/AzureSendFileStorageService.cs +++ b/src/Core/Services/Implementations/AzureSendFileStorageService.cs @@ -12,7 +12,6 @@ namespace Bit.Core.Services public class AzureSendFileStorageService : ISendFileStorageService { public const string FilesContainerName = "sendfiles"; - private const long FileSizeValidationGrace = 1024L; private static readonly TimeSpan _downloadLinkLiveTime = TimeSpan.FromMinutes(1); private readonly CloudBlobClient _blobClient; @@ -90,7 +89,7 @@ public async Task GetSendFileUploadUrlAsync(Send send, string fileId) return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); } - public async Task ValidateFile(Send send, string fileId, long expectedFileSize) + public async Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) { await InitAsync(); @@ -116,7 +115,7 @@ public async Task ValidateFile(Send send, string fileId, long expectedFile blob.SetProperties(); var length = blob.Properties.Length; - if (length < expectedFileSize - FileSizeValidationGrace || length > expectedFileSize + FileSizeValidationGrace) + if (length < expectedFileSize - leeway || length > expectedFileSize + leeway) { return false; } diff --git a/src/Core/Services/Implementations/LocalSendStorageService.cs b/src/Core/Services/Implementations/LocalSendStorageService.cs index ddeefd318514..373648a2e1a1 100644 --- a/src/Core/Services/Implementations/LocalSendStorageService.cs +++ b/src/Core/Services/Implementations/LocalSendStorageService.cs @@ -71,8 +71,12 @@ private Task InitAsync() return Task.FromResult(0); } - public Task GetSendFileUploadUrlAsync(Send send, string fileId) => throw new NotImplementedException(); - public Task ValidateFile(Send send, string fileId, long expectedFileSize) => throw new NotImplementedException(); + public Task GetSendFileUploadUrlAsync(Send send, string fileId) + => Task.FromResult($"/sends/{send.Id}/file/{fileId}"); + + // Validation of local files is handled when they are direct uploaded + public Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) => + Task.FromResult(true); } } diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index 113cbd8ead86..f4df15e4143d 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -26,6 +26,7 @@ public class SendService : ISendService private readonly IPushNotificationService _pushService; private readonly GlobalSettings _globalSettings; private readonly ICurrentContext _currentContext; + private const long _fileSizeLeeway = 1024; public SendService( ISendRepository sendRepository, @@ -107,11 +108,33 @@ public async Task SaveFileSendAsync(Send send, SendFileData data, long f } } + public async Task UploadFileToExistingSendAsync(Stream stream, Send send) + { + if (send?.Data == null) + { + throw new BadRequestException("Send does not have file data"); + } + + if (send.Type != SendType.File) + { + throw new BadRequestException("Not a File Type Send."); + } + + var data = JsonConvert.DeserializeObject(send.Data); + + if (stream.Length < data.Size - _fileSizeLeeway || stream.Length > data.Size + _fileSizeLeeway) + { + throw new BadRequestException("Stream size does not match expected size."); + } + + await _sendFileStorageService.UploadNewFileAsync(stream, send, data.Id); + } + public async Task ValidateSendFile(Send send) { var fileData = JsonConvert.DeserializeObject(send.Data); - var valid = await _sendFileStorageService.ValidateFile(send, fileData.Id, fileData.Size); + var valid = await _sendFileStorageService.ValidateFile(send, fileData.Id, fileData.Size, _fileSizeLeeway); if (!valid) { diff --git a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs index 8f8cab44452b..3b61d330bb1e 100644 --- a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs +++ b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs @@ -40,7 +40,7 @@ public Task GetSendFileUploadUrlAsync(Send send, string fileId) return Task.FromResult((string)null); } - public Task ValidateFile(Send send, string fileId, long expectedFileSize) + public Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) { return Task.FromResult(false); } From 93963f2d317463dfc8adb4fc17237235e3bab776 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 15:39:30 -0600 Subject: [PATCH 06/14] Increase file size difference leeway --- src/Core/Services/Implementations/SendService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index f4df15e4143d..5d0cb1627097 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -26,7 +26,7 @@ public class SendService : ISendService private readonly IPushNotificationService _pushService; private readonly GlobalSettings _globalSettings; private readonly ICurrentContext _currentContext; - private const long _fileSizeLeeway = 1024; + private const long _fileSizeLeeway = 1024L * 1024L; // 1MB public SendService( ISendRepository sendRepository, From 4861006fb8b0b02777804981af6cdaa8b08b0905 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Thu, 25 Feb 2021 16:20:45 -0600 Subject: [PATCH 07/14] Upload through service --- src/Api/Controllers/SendsController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 98d57a05b3a7..9a1e209f6c2c 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -157,7 +157,7 @@ public async Task PostFileForExistingSend(string id, string fileId) var send = await _sendRepository.GetByIdAsync(new Guid(id)); await Request.GetSendFileAsync(async (stream) => { - await _sendFileStorageService.UploadNewFileAsync(stream, send, fileId); + await _sendService.UploadFileToExistingSendAsync(stream, send); }); } From 197755c41fa5e7868f33c338390c3dc4baa1dfb8 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 09:07:15 -0600 Subject: [PATCH 08/14] Fix LocalFileSendStorage It turns out that multipartHttpStreams do not have a length until read. this causes all long files to be "invalid". We need to write the entire stream, then validate length, just like Azure. the difference is, We can return an exception to local storage admonishing the client for lying --- src/Api/Utilities/MultipartFormDataHelper.cs | 9 ++--- src/Core/Services/ISendService.cs | 2 +- src/Core/Services/ISendStorageService.cs | 2 +- .../AzureSendFileStorageService.cs | 2 +- .../LocalSendStorageService.cs | 38 ++++++++++++++++--- .../Services/Implementations/SendService.cs | 14 ++++--- .../NoopSendFileStorageService.cs | 2 +- 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/Api/Utilities/MultipartFormDataHelper.cs b/src/Api/Utilities/MultipartFormDataHelper.cs index be81efcc8726..261b8992f3b5 100644 --- a/src/Api/Utilities/MultipartFormDataHelper.cs +++ b/src/Api/Utilities/MultipartFormDataHelper.cs @@ -72,22 +72,19 @@ public static async Task GetSendFileAsync(this HttpRequest request, Func AccessAsync(Guid sendId, string password); string HashPassword(string password); - Task ValidateSendFile(Send send); + Task ValidateSendFile(Send send); } } diff --git a/src/Core/Services/ISendStorageService.cs b/src/Core/Services/ISendStorageService.cs index e225077619c1..91829070e623 100644 --- a/src/Core/Services/ISendStorageService.cs +++ b/src/Core/Services/ISendStorageService.cs @@ -15,6 +15,6 @@ public interface ISendFileStorageService Task DeleteFilesForUserAsync(Guid userId); Task GetSendFileDownloadUrlAsync(Send send, string fileId); Task GetSendFileUploadUrlAsync(Send send, string fileId); - Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway); + Task ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway); } } diff --git a/src/Core/Services/Implementations/AzureSendFileStorageService.cs b/src/Core/Services/Implementations/AzureSendFileStorageService.cs index 480cecb61ebf..2b646764669d 100644 --- a/src/Core/Services/Implementations/AzureSendFileStorageService.cs +++ b/src/Core/Services/Implementations/AzureSendFileStorageService.cs @@ -89,7 +89,7 @@ public async Task GetSendFileUploadUrlAsync(Send send, string fileId) return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); } - public async Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) + public async Task ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) { await InitAsync(); diff --git a/src/Core/Services/Implementations/LocalSendStorageService.cs b/src/Core/Services/Implementations/LocalSendStorageService.cs index 373648a2e1a1..57173a3c8d19 100644 --- a/src/Core/Services/Implementations/LocalSendStorageService.cs +++ b/src/Core/Services/Implementations/LocalSendStorageService.cs @@ -13,6 +13,8 @@ public class LocalSendStorageService : ISendFileStorageService private readonly string _baseSendUrl; public FileUploadType FileUploadType => FileUploadType.Direct; + private string RelativeFilePath(Send send, string fileID) => $"{send.Id}/{fileID}"; + private string FilePath(Send send, string fileID) => $"{_baseDirPath}/{RelativeFilePath(send, fileID)}"; public LocalSendStorageService( GlobalSettings globalSettings) @@ -24,7 +26,9 @@ public LocalSendStorageService( public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) { await InitAsync(); - using (var fs = File.Create($"{_baseDirPath}/{fileId}")) + var path = FilePath(send, fileId); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + using (var fs = File.Create(path)) { stream.Seek(0, SeekOrigin.Begin); await stream.CopyToAsync(fs); @@ -34,7 +38,9 @@ public async Task UploadNewFileAsync(Stream stream, Send send, string fileId) public async Task DeleteFileAsync(Send send, string fileId) { await InitAsync(); - DeleteFileIfExists($"{_baseDirPath}/{fileId}"); + var path = FilePath(send, fileId); + DeleteFileIfExists(path); + DeleteDirectoryIfExists(Path.GetDirectoryName(path)); } public async Task DeleteFilesForOrganizationAsync(Guid organizationId) @@ -50,7 +56,7 @@ public async Task DeleteFilesForUserAsync(Guid userId) public async Task GetSendFileDownloadUrlAsync(Send send, string fileId) { await InitAsync(); - return $"{_baseSendUrl}/{fileId}"; + return $"{_baseSendUrl}/{RelativeFilePath(send, fileId)}"; } private void DeleteFileIfExists(string path) @@ -61,6 +67,14 @@ private void DeleteFileIfExists(string path) } } + private void DeleteDirectoryIfExists(string path) + { + if (Directory.Exists(path)) + { + Directory.Delete(path); + } + } + private Task InitAsync() { if (!Directory.Exists(_baseDirPath)) @@ -74,9 +88,21 @@ private Task InitAsync() public Task GetSendFileUploadUrlAsync(Send send, string fileId) => Task.FromResult($"/sends/{send.Id}/file/{fileId}"); - // Validation of local files is handled when they are direct uploaded - public Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) => - Task.FromResult(true); + public Task ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) + { + var path = FilePath(send, fileId); + if (!File.Exists(path)) + { + return Task.FromResult(false); + } + + var fileInfo = new FileInfo(path); + if (expectedFileSize < fileInfo.Length - leeway || expectedFileSize > fileInfo.Length + leeway) + { + return Task.FromResult(false); + } + return Task.FromResult(true); + } } } diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index 5d0cb1627097..37d6605e9e15 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -122,25 +122,27 @@ public async Task UploadFileToExistingSendAsync(Stream stream, Send send) var data = JsonConvert.DeserializeObject(send.Data); - if (stream.Length < data.Size - _fileSizeLeeway || stream.Length > data.Size + _fileSizeLeeway) + await _sendFileStorageService.UploadNewFileAsync(stream, send, data.Id); + + if (!await ValidateSendFile(send)) { - throw new BadRequestException("Stream size does not match expected size."); + throw new BadRequestException("File received does not match expected file length."); } - - await _sendFileStorageService.UploadNewFileAsync(stream, send, data.Id); } - public async Task ValidateSendFile(Send send) + public async Task ValidateSendFile(Send send) { var fileData = JsonConvert.DeserializeObject(send.Data); - var valid = await _sendFileStorageService.ValidateFile(send, fileData.Id, fileData.Size, _fileSizeLeeway); + var valid = await _sendFileStorageService.ValidateFileAsync(send, fileData.Id, fileData.Size, _fileSizeLeeway); if (!valid) { // File reported differs in size from that promised. Must be a rogue client. Delete Send await DeleteSendAsync(send); } + + return valid; } public async Task DeleteSendAsync(Send send) diff --git a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs index 3b61d330bb1e..b0d694dcdb68 100644 --- a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs +++ b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs @@ -40,7 +40,7 @@ public Task GetSendFileUploadUrlAsync(Send send, string fileId) return Task.FromResult((string)null); } - public Task ValidateFile(Send send, string fileId, long expectedFileSize, long leeway) + public Task ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) { return Task.FromResult(false); } From e3aee91c48e9e87dee651f7eff20f75d9fa31a75 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 13:35:58 -0600 Subject: [PATCH 09/14] Update src/Api/Utilities/ApiHelpers.cs Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com> --- src/Api/Utilities/ApiHelpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Api/Utilities/ApiHelpers.cs b/src/Api/Utilities/ApiHelpers.cs index 2469868603d6..20ed178762fe 100644 --- a/src/Api/Utilities/ApiHelpers.cs +++ b/src/Api/Utilities/ApiHelpers.cs @@ -41,7 +41,7 @@ public async static Task ReadJsonFileFromBody(HttpContext httpContext, IFo /// HttpRequest received from Azure /// Dictionary of eventType strings and their associated handlers. /// OkObjectResult - // Reference https://docs.microsoft.com/en-us/azure/event-grid/receive-events + /// Reference https://docs.microsoft.com/en-us/azure/event-grid/receive-events public async static Task HandleAzureEvents(HttpRequest request, Dictionary> eventTypeHandlers) { From 4b186b3f06766974e968c1951f64dbee6cb5e28f Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 15:02:17 -0600 Subject: [PATCH 10/14] Do not delete directory if it has files --- .../Services/Implementations/LocalSendStorageService.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Core/Services/Implementations/LocalSendStorageService.cs b/src/Core/Services/Implementations/LocalSendStorageService.cs index 57173a3c8d19..ae4c20cf2ef8 100644 --- a/src/Core/Services/Implementations/LocalSendStorageService.cs +++ b/src/Core/Services/Implementations/LocalSendStorageService.cs @@ -4,6 +4,7 @@ using Bit.Core.Models.Table; using Bit.Core.Settings; using Bit.Core.Enums; +using System.Linq; namespace Bit.Core.Services { @@ -40,7 +41,7 @@ public async Task DeleteFileAsync(Send send, string fileId) await InitAsync(); var path = FilePath(send, fileId); DeleteFileIfExists(path); - DeleteDirectoryIfExists(Path.GetDirectoryName(path)); + DeleteDirectoryIfExistsAndEmpty(Path.GetDirectoryName(path)); } public async Task DeleteFilesForOrganizationAsync(Guid organizationId) @@ -67,9 +68,9 @@ private void DeleteFileIfExists(string path) } } - private void DeleteDirectoryIfExists(string path) + private void DeleteDirectoryIfExistsAndEmpty(string path) { - if (Directory.Exists(path)) + if (Directory.Exists(path) && !Directory.EnumerateFiles(path).Any()) { Directory.Delete(path); } From 8aa423f95c8ddbe0884908dfebb3bde1eb958851 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 15:03:24 -0600 Subject: [PATCH 11/14] Allow large uploads for self hosted instances --- src/Api/Controllers/SendsController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 9a1e209f6c2c..a7a0637106ca 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; @@ -149,7 +149,7 @@ public async Task PostFileForExistingSend(string id, string fileId) throw new BadRequestException("Invalid content."); } - if (Request.ContentLength > 105906176) // 101 MB, give em' 1 extra MB for cushion + if (Request.ContentLength > 105906176 && !_globalSettings.SelfHosted) // 101 MB, give em' 1 extra MB for cushion { throw new BadRequestException("Max file size for direct upload is 100 MB."); } From 135d1e24b4d6a66525eab9d33700ca08d1efd3ab Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 15:03:39 -0600 Subject: [PATCH 12/14] Fix formatting --- src/Api/Controllers/SendsController.cs | 39 ++++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index a7a0637106ca..77499cb97730 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -166,25 +166,28 @@ await Request.GetSendFileAsync(async (stream) => public async Task AzureValidateFile() { return await ApiHelpers.HandleAzureEvents(Request, new Dictionary> + { { - {"Microsoft.Storage.BlobCreated", async (eventGridEvent) => { - try - { - var blobName = eventGridEvent.Subject.Split($"{AzureSendFileStorageService.FilesContainerName}/blobs/")[1]; - var sendId = AzureSendFileStorageService.SendIdFromBlobName(blobName); - var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); - if (send == null) - { - return; - } - await _sendService.ValidateSendFile(send); - } - catch - { - return; - } - }} - }); + "Microsoft.Storage.BlobCreated", async (eventGridEvent) => + { + try + { + var blobName = eventGridEvent.Subject.Split($"{AzureSendFileStorageService.FilesContainerName}/blobs/")[1]; + var sendId = AzureSendFileStorageService.SendIdFromBlobName(blobName); + var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); + if (send == null) + { + return; + } + await _sendService.ValidateSendFile(send); + } + catch + { + return; + } + } + } + }); } [HttpPut("{id}")] From aefe17ce771b0e4067c6f56cd73bed83ed1d091c Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 15:04:19 -0600 Subject: [PATCH 13/14] Re-verfiy access and increment access count on download of Send File --- src/Api/Controllers/SendsController.cs | 30 ++++++++--- src/Core/Services/ISendService.cs | 2 +- .../Services/Implementations/SendService.cs | 53 ++++++++++++++++--- 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 77499cb97730..9d4873e090c7 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; @@ -66,8 +66,9 @@ public async Task Access(string id, [FromBody] SendAccessRequestM } [AllowAnonymous] - [HttpGet("{encodedSendId}/access/file/{fileId}")] - public async Task GetSendFileDownloadData(string encodedSendId, string fileId) + [HttpPost("{encodedSendId}/access/file/{fileId}")] + public async Task GetSendFileDownloadData(string encodedSendId, + string fileId, [FromBody] SendAccessRequestModel model) { var sendId = new Guid(CoreHelpers.Base64UrlDecode(encodedSendId)); var send = await _sendRepository.GetByIdAsync(sendId); @@ -77,11 +78,28 @@ public async Task GetSendFileDownloadData(str throw new BadRequestException("Could not locate send"); } - return new SendFileDownloadDataResponseModel() + var (url, passwordRequired, passwordInvalid) = await _sendService.GetSendFileDownloadUrlAsync(send, fileId, + model.Password); + + if (passwordRequired) + { + return new UnauthorizedResult(); + } + if (passwordInvalid) + { + await Task.Delay(2000); + throw new BadRequestException("Invalid password."); + } + if (send == null) + { + throw new NotFoundException(); + } + + return new ObjectResult(new SendFileDownloadDataResponseModel() { Id = fileId, - Url = await _sendFileStorageService.GetSendFileDownloadUrlAsync(send, fileId), - }; + Url = url, + }); } [HttpGet("{id}")] diff --git a/src/Core/Services/ISendService.cs b/src/Core/Services/ISendService.cs index f05837a08ed0..b4ea58971d98 100644 --- a/src/Core/Services/ISendService.cs +++ b/src/Core/Services/ISendService.cs @@ -15,6 +15,6 @@ public interface ISendService Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password); string HashPassword(string password); Task ValidateSendFile(Send send); - + Task<(string, bool, bool)> GetSendFileDownloadUrlAsync(Send send, string fileId, string password); } } diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index 37d6605e9e15..124e0a1b7b7c 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -156,22 +156,21 @@ public async Task DeleteSendAsync(Send send) await _pushService.PushSyncSendDeleteAsync(send); } - // Response: Send, password required, password invalid - public async Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password) + public (bool grant, bool passwordRequiredError, bool passwordInvalidError) SendCanBeAccessed(Send send, + string password) { - var send = await _sendRepository.GetByIdAsync(sendId); var now = DateTime.UtcNow; if (send == null || send.MaxAccessCount.GetValueOrDefault(int.MaxValue) <= send.AccessCount || send.ExpirationDate.GetValueOrDefault(DateTime.MaxValue) < now || send.Disabled || send.DeletionDate < now) { - return (null, false, false); + return (false, false, false); } if (!string.IsNullOrWhiteSpace(send.Password)) { if (string.IsNullOrWhiteSpace(password)) { - return (null, true, false); + return (false, true, false); } var passwordResult = _passwordHasher.VerifyHashedPassword(new User(), send.Password, password); if (passwordResult == PasswordVerificationResult.SuccessRehashNeeded) @@ -180,11 +179,51 @@ public async Task DeleteSendAsync(Send send) } if (passwordResult == PasswordVerificationResult.Failed) { - return (null, false, true); + return (false, false, true); } } - // TODO: maybe move this to a simple ++ sproc? + + return (true, false, false); + } + + // Response: Send, password required, password invalid + public async Task<(string, bool, bool)> GetSendFileDownloadUrlAsync(Send send, string fileId, string password) + { + if (send.Type != SendType.File) + { + throw new BadRequestException("Can only get a download URL for file type a Send"); + } + + var (grantAccess, passwordRequired, passwordInvalid) = SendCanBeAccessed(send, password); + + if (!grantAccess) + { + return (null, passwordRequired, passwordInvalid); + } + send.AccessCount++; + await _sendRepository.ReplaceAsync(send); + return (await _sendFileStorageService.GetSendFileDownloadUrlAsync(send, fileId), false, false); + } + + // Response: Send, password required, password invalid + public async Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password) + { + var send = await _sendRepository.GetByIdAsync(sendId); + var (grantAccess, passwordRequired, passwordInvalid) = SendCanBeAccessed(send, password); + + if (!grantAccess) + { + return (null, passwordRequired, passwordInvalid); + } + + // TODO: maybe move this to a simple ++ sproc? + if (send.Type != SendType.File) + { + // File sends are incremented during file download + send.AccessCount++; + } + await _sendRepository.ReplaceAsync(send); return (send, false, false); } From 995c5a4c2ac490ff0552d01877511fbfd0df51f6 Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 26 Feb 2021 16:29:22 -0600 Subject: [PATCH 14/14] Update src/Core/Services/Implementations/SendService.cs Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com> --- src/Core/Services/Implementations/SendService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index ab86481aa5c7..cb0fa4cf2d8b 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -196,7 +196,7 @@ public async Task DeleteSendAsync(Send send) { if (send.Type != SendType.File) { - throw new BadRequestException("Can only get a download URL for file type a Send"); + throw new BadRequestException("Can only get a download URL for a file type of Send"); } var (grantAccess, passwordRequired, passwordInvalid) = SendCanBeAccessed(send, password);