diff --git a/src/Ryujinx.Graphics.GAL/BufferRange.cs b/src/Ryujinx.Graphics.GAL/BufferRange.cs index ad9ebee48dbf..f4fcd3492a09 100644 --- a/src/Ryujinx.Graphics.GAL/BufferRange.cs +++ b/src/Ryujinx.Graphics.GAL/BufferRange.cs @@ -10,12 +10,14 @@ namespace Ryujinx.Graphics.GAL public int Offset { get; } public int Size { get; } + public bool Write { get; } - public BufferRange(BufferHandle handle, int offset, int size) + public BufferRange(BufferHandle handle, int offset, int size, bool write = false) { Handle = handle; Offset = offset; Size = size; + Write = write; } } } \ No newline at end of file diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs index 24b0af2d7de8..4e0d0ac6cdbd 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs @@ -116,7 +116,7 @@ internal BufferHandle MapBufferBlocking(BufferHandle handle) internal BufferRange MapBufferRange(BufferRange range) { - return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size); + return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size, range.Write); } internal Span MapBufferRanges(Span ranges) @@ -135,7 +135,7 @@ internal Span MapBufferRanges(Span ranges) result = BufferHandle.Null; } - range = new BufferRange(result, range.Offset, range.Size); + range = new BufferRange(result, range.Offset, range.Size, range.Write); } } @@ -159,7 +159,7 @@ internal Span MapBufferRanges(Span ranges) result = BufferHandle.Null; } - assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size)); + assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size, range.Write)); } } @@ -182,7 +182,7 @@ internal Span MapBufferRanges(Span - /// Gets a sub-range from the buffer, from a start address till the end of the buffer. + /// Gets a sub-range from the buffer, from a start address til a page boundary after the given size. /// /// /// This can be used to bind and use sub-ranges of the buffer on the host API. /// /// Start address of the sub-range, must be greater than or equal to the buffer address + /// Size in bytes of the sub-range, must be less than or equal to the buffer size + /// Whether the buffer will be written to by this use /// The buffer sub-range - public BufferRange GetRange(ulong address) + public BufferRange GetRangeAligned(ulong address, ulong size, bool write) { + ulong end = ((address + size + MemoryManager.PageMask) & ~MemoryManager.PageMask) - Address; ulong offset = address - Address; - return new BufferRange(Handle, (int)offset, (int)(Size - offset)); + return new BufferRange(Handle, (int)offset, (int)(end - offset), write); } /// @@ -162,12 +165,13 @@ public BufferRange GetRange(ulong address) /// /// Start address of the sub-range, must be greater than or equal to the buffer address /// Size in bytes of the sub-range, must be less than or equal to the buffer size + /// Whether the buffer will be written to by this use /// The buffer sub-range - public BufferRange GetRange(ulong address, ulong size) + public BufferRange GetRange(ulong address, ulong size, bool write) { int offset = (int)(address - Address); - return new BufferRange(Handle, offset, (int)size); + return new BufferRange(Handle, offset, (int)size, write); } /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index a5a9b75e908a..9aa0a332692c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -372,15 +372,27 @@ public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, ui } /// - /// Gets a buffer sub-range starting at a given memory address. + /// Gets a buffer sub-range for a given GPU memory range. + /// + /// GPU memory manager where the buffer is mapped + /// Start GPU virtual address of the buffer + /// Size in bytes of the buffer + /// The buffer sub-range for the given range + public BufferRange GetGpuBufferRange(MemoryManager memoryManager, ulong gpuVa, ulong size) + { + return GetBufferRange(TranslateAndCreateBuffer(memoryManager, gpuVa, size), size); + } + + /// + /// Gets a buffer sub-range from a start address til a page boundary after the given size. /// /// Start address of the memory range /// Size in bytes of the memory range /// Whether the buffer will be written to by this use /// The buffer sub-range starting at the given memory address - public BufferRange GetBufferRangeTillEnd(ulong address, ulong size, bool write = false) + public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address); + return GetBuffer(address, size, write).GetRangeAligned(address, size, write); } /// @@ -392,7 +404,7 @@ public BufferRange GetBufferRangeTillEnd(ulong address, ulong size, bool write = /// The buffer sub-range for the given range public BufferRange GetBufferRange(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address, size); + return GetBuffer(address, size, write).GetRange(address, size, write); } /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index e20e1bb68164..df44ffb47090 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -623,7 +623,7 @@ private void BindBuffers(BufferCache bufferCache, BuffersPerStage[] bindings, bo { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); @@ -660,7 +660,7 @@ private void BindBuffers(BufferCache bufferCache, BuffersPerStage buffers, bool { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); diff --git a/src/Ryujinx.Graphics.Vulkan/Auto.cs b/src/Ryujinx.Graphics.Vulkan/Auto.cs index fdce7232ca25..83d52a9b7f65 100644 --- a/src/Ryujinx.Graphics.Vulkan/Auto.cs +++ b/src/Ryujinx.Graphics.Vulkan/Auto.cs @@ -18,6 +18,12 @@ interface IAutoPrivate : IAuto void AddCommandBufferDependencies(CommandBufferScoped cbs); } + interface IMirrorable where T : IDisposable + { + Auto GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored); + void ClearMirrors(CommandBufferScoped cbs, int offset, int size); + } + class Auto : IAutoPrivate, IDisposable where T : IDisposable { private int _referenceCount; @@ -26,6 +32,7 @@ class Auto : IAutoPrivate, IDisposable where T : IDisposable private readonly BitMap _cbOwnership; private readonly MultiFenceHolder _waitable; private readonly IAutoPrivate[] _referencedObjs; + private readonly IMirrorable _mirrorable; private bool _disposed; private bool _destroyed; @@ -37,6 +44,11 @@ public Auto(T value) _cbOwnership = new BitMap(CommandBufferPool.MaxCommandBuffers); } + public Auto(T value, IMirrorable mirrorable, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value, waitable, referencedObjs) + { + _mirrorable = mirrorable; + } + public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value) { _waitable = waitable; @@ -48,9 +60,18 @@ public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referenced } } - public T Get(CommandBufferScoped cbs, int offset, int size) + public T GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + mirrored = false; + var mirror = _mirrorable?.GetMirrorable(cbs, ref offset, size, out mirrored); + mirror._waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, false); + return mirror.Get(cbs); + } + + public T Get(CommandBufferScoped cbs, int offset, int size, bool write = false) { - _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size); + _mirrorable?.ClearMirrors(cbs, offset, size); + _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, write); return Get(cbs); } diff --git a/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs new file mode 100644 index 000000000000..da99f1c85a95 --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs @@ -0,0 +1,263 @@ +using Ryujinx.Common.Memory; +using System; +using System.Numerics; + +namespace Ryujinx.Graphics.Vulkan +{ + interface IBitMapListener + { + void BitMapSignal(int index, int count); + } + + struct BitMapStruct where T : IArray + { + public const int IntSize = 64; + + private const int IntShift = 6; + private const int IntMask = IntSize - 1; + + private T _masks; + + public BitMapStruct() + { + _masks = default; + } + + public bool BecomesUnsetFrom(in BitMapStruct from, ref BitMapStruct into) + { + bool result = false; + + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long fromMask = from._masks[i]; + long unsetMask = (~fromMask) & (fromMask ^ _masks[i]); + into._masks[i] = unsetMask; + + result |= unsetMask != 0; + } + + return result; + } + + public void SetAndSignalUnset(in BitMapStruct from, ref T2 listener) where T2 : struct, IBitMapListener + { + BitMapStruct result = new(); + + if (BecomesUnsetFrom(from, ref result)) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + ref T resultMasks = ref result._masks; + for (int i = 0; i < masks; i++) + { + long value = resultMasks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + listener.BitMapSignal(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + _masks = from._masks; + } + + public void SignalSet(Action action) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long value = _masks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + action(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + public bool AnySet() + { + for (int i = 0; i < _masks.Length; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + return false; + } + + public bool IsSet(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + return (_masks[wordIndex] & wordMask) != 0; + } + + public bool IsSet(int start, int end) + { + if (start == end) + { + return IsSet(start); + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + return (_masks[startIndex] & startMask & endMask) != 0; + } + + if ((_masks[startIndex] & startMask) != 0) + { + return true; + } + + for (int i = startIndex + 1; i < endIndex; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + if ((_masks[endIndex] & endMask) != 0) + { + return true; + } + + return false; + } + + public bool Set(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + if ((_masks[wordIndex] & wordMask) != 0) + { + return false; + } + + _masks[wordIndex] |= wordMask; + + return true; + } + + public void Set(int bit, bool value) + { + if (value) + { + Set(bit); + } + else + { + Clear(bit); + } + } + + public void SetRange(int start, int end) + { + if (start == end) + { + Set(start); + return; + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + _masks[startIndex] |= startMask & endMask; + } + else + { + _masks[startIndex] |= startMask; + + for (int i = startIndex + 1; i < endIndex; i++) + { + _masks[i] |= -1; + } + + _masks[endIndex] |= endMask; + } + } + + public BitMapStruct Union(BitMapStruct other) + { + var result = new BitMapStruct(); + + ref var masks = ref _masks; + ref var otherMasks = ref other._masks; + ref var newMasks = ref result._masks; + + for (int i = 0; i < masks.Length; i++) + { + newMasks[i] = masks[i] | otherMasks[i]; + } + + return result; + } + + public void Clear(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + _masks[wordIndex] &= ~wordMask; + } + + public void Clear() + { + for (int i = 0; i < _masks.Length; i++) + { + _masks[i] = 0; + } + } + + public void ClearInt(int start, int end) + { + for (int i = start; i <= end; i++) + { + _masks[i] = 0; + } + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 6e10fad00a37..5e65e609ca5e 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -10,7 +10,7 @@ namespace Ryujinx.Graphics.Vulkan { - class BufferHolder : IDisposable + class BufferHolder : IDisposable, IMirrorable, IMirrorable { private const int MaxUpdateBufferSize = 0x10000; @@ -64,6 +64,10 @@ class BufferHolder : IDisposable private List _swapActions; + private byte[] _pendingData; + private BufferRangeList _pendingDataRanges; + private Dictionary _mirrors; + public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType) { _gd = gd; @@ -71,7 +75,7 @@ public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAll _allocation = allocation; _allocationAuto = new Auto(allocation); _waitable = new MultiFenceHolder(size); - _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = allocation.HostPointer; @@ -91,7 +95,7 @@ public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = _allocation.HostPointer + offset; @@ -110,7 +114,7 @@ public bool TryBackingSwap(ref CommandBufferScoped? cbs) // Only swap if the buffer is not used in any queued command buffer. bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); - if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld) + if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld && (_pendingData == null || cbs != null)) { var currentAllocation = _allocationAuto; var currentBuffer = _buffer; @@ -120,6 +124,11 @@ public bool TryBackingSwap(ref CommandBufferScoped? cbs) if (buffer.Handle != 0) { + if (cbs != null) + { + ClearMirrors(cbs.Value, 0, Size); + } + _flushLock.AcquireWriterLock(Timeout.Infinite); ClearFlushFence(); @@ -128,7 +137,7 @@ public bool TryBackingSwap(ref CommandBufferScoped? cbs) _allocation = allocation; _allocationAuto = new Auto(allocation); - _buffer = new Auto(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(_gd.Api, _device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; _map = allocation.HostPointer; @@ -264,7 +273,7 @@ public unsafe Auto CreateView(VkFormat format, int offset, (_swapActions ??= new List()).Add(invalidateView); - return new Auto(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); + return new Auto(new DisposableBufferView(_gd.Api, _device, bufferView), this, _waitable, _buffer); } public void InheritMetrics(BufferHolder other) @@ -309,6 +318,80 @@ public unsafe void InsertBarrier(CommandBuffer commandBuffer, bool isWrite) } } + private ulong ToMirrorKey(int offset, int size) + { + return ((ulong)offset << 32) | (uint)size; + } + + private (int offset, int size) FromMirrorKey(ulong key) + { + return ((int)(key >> 32), (int)key); + } + + private unsafe bool TryGetMirror(CommandBufferScoped cbs, ref int offset, int size, out Auto buffer) + { + // Does this binding need to be mirrored? + + if (!_pendingDataRanges.OverlapsWith(offset, size)) + { + buffer = null; + return false; + } + + var key = ToMirrorKey(offset, size); + + if (_mirrors.TryGetValue(key, out StagingBufferReserved reserved)) + { + buffer = reserved.Buffer.GetBuffer(); + offset = reserved.Offset; + + return true; + } + + // Is this mirror allowed to exist? Can't be used for write in any in-flight write. + if (_waitable.IsBufferRangeInUse(offset, size, true)) + { + // Some of the data is not mirrorable, so upload the whole range. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + + // Build data for the new mirror. + + var baseData = new Span((void*)(_map + offset), size); + var modData = _pendingData.AsSpan(offset, size); + + StagingBufferReserved? newMirror = _gd.BufferManager.StagingBuffer.TryReserveData(cbs, size); + + if (newMirror != null) + { + var mirror = newMirror.Value; + _pendingDataRanges.FillData(baseData, modData, offset, new Span((void*)(mirror.Buffer._map + mirror.Offset), size)); + + if (_mirrors.Count == 0) + { + _gd.PipelineInternal.RegisterActiveMirror(this); + } + + _mirrors.Add(key, mirror); + + buffer = mirror.Buffer.GetBuffer(); + offset = mirror.Offset; + + return true; + } + else + { + // Data could not be placed on the mirror, likely out of space. Force the data to flush. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + } + public Auto GetBuffer() { return _buffer; @@ -346,6 +429,86 @@ public Auto GetBuffer(CommandBuffer commandBuffer, int offset, return _buffer; } + public Auto GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + if (_pendingData != null && TryGetMirror(cbs, ref offset, size, out Auto result)) + { + mirrored = true; + return result; + } + + mirrored = false; + return _buffer; + } + + Auto IMirrorable.GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + // Cannot mirror buffer views right now. + + throw new NotImplementedException(); + } + + public void ClearMirrors() + { + // Clear mirrors without forcing a flush. This happens when the command buffer is switched, + // as all reserved areas on the staging buffer are released. + + if (_pendingData != null) + { + _mirrors.Clear(); + }; + } + + public void ClearMirrors(CommandBufferScoped cbs, int offset, int size) + { + // Clear mirrors in the given range, and submit overlapping pending data. + + // TODO: within range. for now it just clears everything. + + if (_pendingData != null) + { + bool hadMirrors = _mirrors.Count > 0; + + _mirrors.Clear(); + + if (_pendingDataRanges.Count() != 0) + { + UploadPendingData(cbs, offset, size); + } + + if (hadMirrors) + { + _gd.PipelineInternal.Rebind(_buffer, 0, Size); + } + }; + } + + private void UploadPendingData(CommandBufferScoped cbs, int offset, int size) + { + var ranges = _pendingDataRanges.All(); + + // TODO: within range. for now it just clears everything. + + if (ranges != null) + { + var rangeCopy = ranges.ToArray(); + + _pendingDataRanges.Clear(); + + foreach (var range in rangeCopy) + { + if (_gd.PipelineInternal.CurrentCommandBuffer.CommandBuffer.Handle == cbs.CommandBuffer.Handle) + { + SetData(range.Offset, _pendingData.AsSpan(range.Offset, range.Size), cbs, _gd.PipelineInternal.EndRenderPass, false); + } + else + { + SetData(range.Offset, _pendingData.AsSpan(range.Offset, range.Size), cbs, null, false); + } + } + } + } + public void SignalWrite(int offset, int size) { ConsiderBackingSwap(); @@ -481,7 +644,37 @@ public unsafe Span GetDataStorage(int offset, int size) throw new InvalidOperationException("The buffer is not host mapped."); } - public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferScoped? cbs = null, Action endRenderPass = null) + public bool RemoveOverlappingMirrors(int offset, int size) + { + List toRemove = null; + foreach (var key in _mirrors.Keys) + { + (int keyOffset, int keySize) = FromMirrorKey(key); + if (!(offset + size <= keyOffset || offset >= keyOffset + keySize)) + { + if (toRemove == null) + { + toRemove = new List(); + } + + toRemove.Add(key); + } + } + + if (toRemove != null) + { + foreach (var key in toRemove) + { + _mirrors.Remove(key); + } + + return true; + } + + return false; + } + + public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferScoped? cbs = null, Action endRenderPass = null, bool allowCbsWait = true) { int dataSize = Math.Min(data.Length, Size - offset); if (dataSize == 0) @@ -490,6 +683,7 @@ public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferSco } _setCount++; + bool allowMirror = allowCbsWait && cbs != null && _currentType <= BufferAllocationType.HostMapped; if (_map != IntPtr.Zero) { @@ -497,7 +691,7 @@ public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferSco bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); // If the buffer is rented, take a little more time and check if the use overlaps this handle. - bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize); + bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize, false); if (!needsFlush) { @@ -505,12 +699,48 @@ public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferSco data.Slice(0, dataSize).CopyTo(new Span((void*)(_map + offset), dataSize)); + if (_pendingData != null) + { + bool removed = _pendingDataRanges.Remove(offset, dataSize); + if (RemoveOverlappingMirrors(offset, dataSize) || removed) + { + // If any mirrors were removed, rebind the buffer range. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + } + } + SignalWrite(offset, dataSize); return; } } + // If the buffer does not have an in-flight write (including an inline update), then upload data to a pendingCopy. + if (allowMirror && !_waitable.IsBufferRangeInUse(offset, dataSize, true)) + { + if (_pendingData == null) + { + _pendingData = new byte[Size]; + _mirrors = new Dictionary(); + } + + data.Slice(0, dataSize).CopyTo(_pendingData.AsSpan(offset, dataSize)); + _pendingDataRanges.Add(offset, dataSize); + + // Remove any overlapping mirrors. + RemoveOverlappingMirrors(offset, dataSize); + + // Tell the graphics device to rebind any constant buffer that overlaps the newly modified range, as it should access a mirror. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + + return; + } + + if (_pendingData != null) + { + _pendingDataRanges.Remove(offset, dataSize); + } + if (cbs != null && _gd.PipelineInternal.RenderPassActive && !(_buffer.HasCommandBufferDependency(cbs.Value) && @@ -528,7 +758,37 @@ public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferSco data.Length > MaxUpdateBufferSize || !TryPushData(cbs.Value, endRenderPass, offset, data)) { - _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + if (allowCbsWait) + { + _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + } + else + { + bool rentCbs = cbs == null; + if (rentCbs) + { + cbs = _gd.CommandBufferPool.Rent(); + } + + if (!_gd.BufferManager.StagingBuffer.TryPushData(cbs.Value, endRenderPass, this, offset, data)) + { + // Need to do a slow upload. + BufferHolder srcHolder = _gd.BufferManager.Create(_gd, dataSize, baseType: BufferAllocationType.HostMapped); + srcHolder.SetDataUnchecked(0, data); + + var srcBuffer = srcHolder.GetBuffer(); + var dstBuffer = this.GetBuffer(cbs.Value.CommandBuffer, true); + + Copy(_gd, cbs.Value, srcBuffer, dstBuffer, 0, offset, dataSize); + + srcHolder.Dispose(); + } + + if (rentCbs) + { + cbs.Value.Dispose(); + } + } } } @@ -567,7 +827,7 @@ private unsafe bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, i endRenderPass?.Invoke(); - var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; + var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length, true).Value; _writeCount--; @@ -617,7 +877,7 @@ private unsafe bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, i bool registerSrcUsage = true) { var srcBuffer = registerSrcUsage ? src.Get(cbs, srcOffset, size).Value : src.GetUnsafe().Value; - var dstBuffer = dst.Get(cbs, dstOffset, size).Value; + var dstBuffer = dst.Get(cbs, dstOffset, size, true).Value; InsertBufferBarrier( gd, diff --git a/src/Ryujinx.Graphics.Vulkan/BufferRangeList.cs b/src/Ryujinx.Graphics.Vulkan/BufferRangeList.cs new file mode 100644 index 000000000000..5c0fff8e2e84 --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BufferRangeList.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; + +namespace Ryujinx.Graphics.Vulkan +{ + struct BufferRangeList + { + internal struct Range + { + public int Offset { get; } + public int Size { get; } + + public int End => Offset + Size; + + public Range(int offset, int size) + { + Offset = offset; + Size = size; + } + + public bool OverlapsWith(int offset, int size) + { + return Offset < offset + size && offset < Offset + Size; + } + } + + private List _ranges; + + public List All() + { + return _ranges; + } + + public bool Remove(int offset, int size) + { + var list = _ranges; + bool removedAny = false; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + + if (overlapIndex >= 0) + { + // Overlaps with a range. Search back to find the first one it doesn't overlap with. + + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + var currentOverlap = list[overlapIndex]; + + // Orphan the start of the overlap. + if (currentOverlap.Offset < offset) + { + list[overlapIndex] = new Range(currentOverlap.Offset, offset - currentOverlap.Offset); + currentOverlap = new Range(offset, currentOverlap.End - offset); + list.Insert(++overlapIndex, currentOverlap); + startIndex++; + + removedAny = true; + } + + // Remove any middle overlaps. + while (currentOverlap.Offset < endOffset) + { + if (currentOverlap.End > endOffset) + { + // Update the end overlap instead of removing it, if it spans beyond the removed range. + list[overlapIndex] = new Range(endOffset, currentOverlap.End - endOffset); + + removedAny = true; + break; + } + + if (++overlapIndex >= list.Count) + { + break; + } + + currentOverlap = list[overlapIndex]; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + removedAny |= count > 0; + } + } + + return removedAny; + } + + public void Add(int offset, int size) + { + var list = _ranges; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + if (overlapIndex >= 0) + { + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + while (overlapIndex < list.Count && list[overlapIndex].OverlapsWith(offset, size)) + { + var currentOverlap = list[overlapIndex]; + var currentOverlapEndOffset = currentOverlap.Offset + currentOverlap.Size; + + if (offset > currentOverlap.Offset) + { + offset = currentOverlap.Offset; + } + + if (endOffset < currentOverlapEndOffset) + { + endOffset = currentOverlapEndOffset; + } + + overlapIndex++; + size = endOffset - offset; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + overlapIndex = startIndex; + } + else + { + overlapIndex = ~overlapIndex; + } + + list.Insert(overlapIndex, new Range(offset, size)); + } + else + { + _ranges = new List + { + new Range(offset, size) + }; + } + } + + public bool OverlapsWith(int offset, int size) + { + var list = _ranges; + if (list == null) + { + return false; + } + + return BinarySearch(list, offset, size) >= 0; + } + + private static int BinarySearch(List list, int offset, int size) + { + int left = 0; + int right = list.Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + var item = list[middle]; + + if (item.OverlapsWith(offset, size)) + { + return middle; + } + + if (offset < item.Offset) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + public void FillData(Span baseData, Span modData, int offset, Span result) + { + int size = baseData.Length; + int endOffset = offset + size; + + var list = _ranges; + if (list == null) + { + baseData.CopyTo(result); + } + + int srcOffset = offset; + int dstOffset = 0; + bool activeRange = false; + + for (int i = 0; i < list.Count; i++) + { + var range = list[i]; + + int rangeEnd = range.Offset + range.Size; + + if (activeRange) + { + if (range.Offset >= endOffset) + { + break; + } + } + else + { + if (rangeEnd <= offset) + { + continue; + } + + activeRange = true; + } + + int baseSize = range.Offset - srcOffset; + + if (baseSize > 0) + { + baseData.Slice(dstOffset, baseSize).CopyTo(result.Slice(dstOffset, baseSize)); + srcOffset += baseSize; + dstOffset += baseSize; + } + + int modSize = Math.Min(rangeEnd - srcOffset, endOffset - srcOffset); + if (modSize != 0) + { + modData.Slice(dstOffset, modSize).CopyTo(result.Slice(dstOffset, modSize)); + srcOffset += modSize; + dstOffset += modSize; + } + } + + int baseSizeEnd = endOffset - srcOffset; + + if (baseSizeEnd > 0) + { + baseData.Slice(dstOffset, baseSizeEnd).CopyTo(result.Slice(dstOffset, baseSizeEnd)); + } + } + + public int Count() + { + return _ranges?.Count ?? 0; + } + + public void Clear() + { + _ranges = null; + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/BufferState.cs b/src/Ryujinx.Graphics.Vulkan/BufferState.cs index 6829f83339a7..330dd7c6f875 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferState.cs @@ -23,7 +23,7 @@ public void BindTransformFeedbackBuffer(VulkanRenderer gd, CommandBufferScoped c { if (_buffer != null) { - var buffer = _buffer.Get(cbs, _offset, _size).Value; + var buffer = _buffer.Get(cbs, _offset, _size, true).Value; gd.TransformFeedbackApi.CmdBindTransformFeedbackBuffers(cbs.CommandBuffer, binding, 1, buffer, (ulong)_offset, (ulong)_size); } @@ -40,6 +40,11 @@ public void Swap(Auto from, Auto to) } } + public bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public void Dispose() { _buffer?.DecrementReferenceCount(); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs index 920501d3a3d4..094b20c2f969 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs @@ -6,6 +6,7 @@ internal class BufferUsageBitmap private int _size; private int _granularity; private int _bits; + private int _writeBitOffset; private int _intsPerCb; private int _bitsPerCb; @@ -14,7 +15,11 @@ public BufferUsageBitmap(int size, int granularity) { _size = size; _granularity = granularity; - _bits = (size + (granularity - 1)) / granularity; + + // There are two sets of bits - one for read tracking, and the other for write. + int bits = (size + (granularity - 1)) / granularity; + _writeBitOffset = bits; + _bits = bits << 1; _intsPerCb = (_bits + (BitMap.IntSize - 1)) / BitMap.IntSize; _bitsPerCb = _intsPerCb * BitMap.IntSize; @@ -22,7 +27,7 @@ public BufferUsageBitmap(int size, int granularity) _bitmap = new BitMap(_bitsPerCb * CommandBufferPool.MaxCommandBuffers); } - public void Add(int cbIndex, int offset, int size) + public void Add(int cbIndex, int offset, int size, bool write) { if (size == 0) { @@ -35,32 +40,32 @@ public void Add(int cbIndex, int offset, int size) size = _size - offset; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; _bitmap.SetRange(start, end); } - public bool OverlapsWith(int cbIndex, int offset, int size) + public bool OverlapsWith(int cbIndex, int offset, int size, bool write = false) { if (size == 0) { return false; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; return _bitmap.IsSet(start, end); } - public bool OverlapsWith(int offset, int size) + public bool OverlapsWith(int offset, int size, bool write = false) { for (int i = 0; i < CommandBufferPool.MaxCommandBuffers; i++) { - if (OverlapsWith(i, offset, size)) + if (OverlapsWith(i, offset, size, write)) { return true; } diff --git a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs index f3ac36e1385d..4b8697033787 100644 --- a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs +++ b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs @@ -1,4 +1,6 @@ -using Ryujinx.Graphics.GAL; +using Ryujinx.Common.Memory; +using Ryujinx.Common.Utilities; +using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Shader; using Silk.NET.Vulkan; using System; @@ -9,13 +11,39 @@ namespace Ryujinx.Graphics.Vulkan { class DescriptorSetUpdater { + private static int StorageBufferMaxMirrorable = 8192; + private struct BufferRef : IEquatable + { + public Auto Buffer; + public int Offset; + public bool Write; + + public BufferRef(Auto buffer) + { + Buffer = buffer; + Offset = 0; + Write = true; + } + + public BufferRef(Auto buffer, ref BufferRange range) + { + Buffer = buffer; + Offset = range.Offset; + Write = range.Write; + } + + public bool Equals(BufferRef other) + { + return Buffer == other.Buffer && Offset == other.Offset && Write == other.Write; + } + } + private readonly VulkanRenderer _gd; private readonly PipelineBase _pipeline; - private ShaderCollection _program; - private Auto[] _uniformBufferRefs; - private Auto[] _storageBufferRefs; + private BufferRef[] _uniformBufferRefs; + private BufferRef[] _storageBufferRefs; private Auto[] _textureRefs; private Auto[] _samplerRefs; private Auto[] _imageRefs; @@ -30,10 +58,13 @@ class DescriptorSetUpdater private BufferView[] _bufferTextures; private BufferView[] _bufferImages; - private bool[] _uniformSet; - private bool[] _storageSet; private Silk.NET.Vulkan.Buffer _cachedSupportBuffer; + private BitMapStruct> _uniformSet; + private BitMapStruct> _storageSet; + private BitMapStruct> _uniformMirrored; + private BitMapStruct> _storageMirrored; + [Flags] private enum DirtyFlags { @@ -59,8 +90,8 @@ public DescriptorSetUpdater(VulkanRenderer gd, PipelineBase pipeline) // Some of the bindings counts needs to be multiplied by 2 because we have buffer and // regular textures/images interleaved on the same descriptor set. - _uniformBufferRefs = new Auto[Constants.MaxUniformBufferBindings]; - _storageBufferRefs = new Auto[Constants.MaxStorageBufferBindings]; + _uniformBufferRefs = new BufferRef[Constants.MaxUniformBufferBindings]; + _storageBufferRefs = new BufferRef[Constants.MaxStorageBufferBindings]; _textureRefs = new Auto[Constants.MaxTextureBindings * 2]; _samplerRefs = new Auto[Constants.MaxTextureBindings * 2]; _imageRefs = new Auto[Constants.MaxImageBindings * 2]; @@ -83,9 +114,6 @@ public DescriptorSetUpdater(VulkanRenderer gd, PipelineBase pipeline) _textures.AsSpan().Fill(initialImageInfo); _images.AsSpan().Fill(initialImageInfo); - _uniformSet = new bool[Constants.MaxUniformBufferBindings]; - _storageSet = new bool[Constants.MaxStorageBufferBindings]; - if (gd.Capabilities.SupportsNullDescriptors) { // If null descriptors are supported, we can pass null as the handle. @@ -136,6 +164,71 @@ public void Initialize() _dummyTexture.SetData(dummyTextureData); } + private bool BindingOverlaps(ref DescriptorBufferInfo info, int bindingOffset, int offset, int size) + { + return offset < bindingOffset + (int)info.Range && (offset + size) > bindingOffset; + } + + internal void Rebind(Auto buffer, int offset, int size) + { + if (_program == null) + { + return; + } + + // Explicitly check the support buffer. + + if (_uniformBufferRefs[0].Buffer == buffer) + { + _uniformSet.Clear(0); + SignalDirty(DirtyFlags.Uniform); + } + + // Check stage bindings + + _uniformMirrored.Union(_uniformSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _uniformBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _uniformBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _uniformSet.Clear(binding); + SignalDirty(DirtyFlags.Uniform); + } + } + + binding++; + } + }); + + _storageMirrored.Union(_storageSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _storageBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _storageBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _storageSet.Clear(binding); + SignalDirty(DirtyFlags.Storage); + } + } + + binding++; + } + }); + } + public void SetProgram(ShaderCollection program) { _program = program; @@ -178,22 +271,28 @@ public void SetStorageBuffers(CommandBuffer commandBuffer, ReadOnlySpan vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true); - ref Auto currentVkBuffer = ref _storageBufferRefs[index]; + Auto vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, buffer.Write, isSSBO: true); + + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new DescriptorBufferInfo() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size }; + + var newRef = new BufferRef(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -207,21 +306,24 @@ public void SetStorageBuffers(CommandBuffer commandBuffer, int first, ReadOnlySp var vkBuffer = buffers[i]; int index = first + i; - ref Auto currentVkBuffer = ref _storageBufferRefs[index]; + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new DescriptorBufferInfo() { Offset = 0, Range = Vk.WholeSize }; + + BufferRef newRef = new BufferRef(vkBuffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -286,22 +388,28 @@ public void SetUniformBuffers(CommandBuffer commandBuffer, ReadOnlySpan vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); - ref Auto currentVkBuffer = ref _uniformBufferRefs[index]; + Auto vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); + + ref BufferRef currentBufferRef = ref _uniformBufferRefs[index]; DescriptorBufferInfo info = new DescriptorBufferInfo() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size }; + + BufferRef newRef = new BufferRef(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _uniformBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _uniformSet[index] = false; + _uniformSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -351,13 +459,26 @@ public void UpdateAndBindDescriptorSets(CommandBufferScoped cbs, PipelineBindPoi } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void UpdateBuffer( + private static bool UpdateBuffer( CommandBufferScoped cbs, ref DescriptorBufferInfo info, - Auto buffer, - Auto dummyBuffer) + ref BufferRef buffer, + Auto dummyBuffer, + bool mirrorable) { - info.Buffer = buffer?.Get(cbs, (int)info.Offset, (int)info.Range).Value ?? default; + int offset = buffer.Offset; + bool mirrored = false; + + if (mirrorable) + { + info.Buffer = buffer.Buffer?.GetMirrorable(cbs, ref offset, (int)info.Range, out mirrored).Value ?? default; + } + else + { + info.Buffer = buffer.Buffer?.Get(cbs, offset, (int)info.Range, buffer.Write).Value ?? default; + } + + info.Offset = (ulong)offset; // The spec requires that buffers with null handle have offset as 0 and range as VK_WHOLE_SIZE. if (info.Buffer.Handle == 0) @@ -366,6 +487,8 @@ public void UpdateAndBindDescriptorSets(CommandBufferScoped cbs, PipelineBindPoi info.Offset = 0; info.Range = Vk.WholeSize; } + + return mirrored; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -394,15 +517,24 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo { Span uniformBuffer = stackalloc DescriptorBufferInfo[1]; - if (!_uniformSet[0]) + int supportBufferOffset = 0; + ref var supportRef = ref _uniformBufferRefs[0]; + + if (_uniformSet.Set(0)) + { + var buffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, _pipeline.SupportBufferUpdater.Handle, false); + _cachedSupportBuffer = buffer.GetMirrorable(cbs, ref supportBufferOffset, SupportBuffer.RequiredSize, out _).Value; + supportRef.Buffer = buffer; + supportRef.Offset = supportBufferOffset; + } + else { - _cachedSupportBuffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, _pipeline.SupportBufferUpdater.Handle, false).Get(cbs, 0, SupportBuffer.RequiredSize).Value; - _uniformSet[0] = true; + supportBufferOffset = supportRef.Offset; } uniformBuffer[0] = new DescriptorBufferInfo() { - Offset = 0, + Offset = (ulong)supportBufferOffset, Range = (ulong)SupportBuffer.RequiredSize, Buffer = _cachedSupportBuffer }; @@ -422,11 +554,13 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); + ref BufferRef buffer = ref _uniformBufferRefs[index]; + + bool mirrored = UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); - _uniformSet[index] = true; + _uniformMirrored.Set(index, mirrored); } } @@ -439,11 +573,29 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo { int index = binding + i; - if (!_storageSet[index]) + ref BufferRef buffer = ref _storageBufferRefs[index]; + + /* + if (isMoltenVk) + { + if (buffer.Buffer?.MVKStageChanged(mvkStages, renderPassId) ?? false) + { + _pipeline.EndRenderPass(); + } + } + */ + + if (_storageSet.Set(index)) { - UpdateBuffer(cbs, ref _storageBuffers[index], _storageBufferRefs[index], dummyBuffer); + ref var info = ref _storageBuffers[index]; - _storageSet[index] = true; + bool mirrored = UpdateBuffer(cbs, + ref info, + ref _storageBufferRefs[index], + dummyBuffer, + !buffer.Write && info.Range <= (ulong)StorageBufferMaxMirrorable); + + _storageMirrored.Set(index, mirrored); } } @@ -489,7 +641,7 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo for (int i = 0; i < count; i++) { - bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs) ?? default; + bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs, false) ?? default; } dsc.UpdateBufferImages(0, binding, bufferTextures.Slice(0, count), DescriptorType.UniformTexelBuffer); @@ -514,7 +666,7 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo for (int i = 0; i < count; i++) { - bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i]) ?? default; + bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i], true) ?? default; } dsc.UpdateBufferImages(0, binding, bufferImages.Slice(0, count), DescriptorType.StorageTexelBuffer); @@ -557,7 +709,7 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateAndBindUniformBufferPd(CommandBufferScoped cbs, PipelineBindPoint pbp) { - if (!_uniformSet[0]) + if (_uniformSet.Set(0)) { Span uniformBuffer = stackalloc DescriptorBufferInfo[1]; @@ -568,8 +720,6 @@ private void UpdateAndBindUniformBufferPd(CommandBufferScoped cbs, PipelineBindP Buffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, _pipeline.SupportBufferUpdater.Handle, false).Get(cbs, 0, SupportBuffer.RequiredSize).Value }; - _uniformSet[0] = true; - UpdateBuffers(cbs, pbp, 0, uniformBuffer, DescriptorType.UniformBuffer); } @@ -587,10 +737,10 @@ private void UpdateAndBindUniformBufferPd(CommandBufferScoped cbs, PipelineBindP { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); - _uniformSet[index] = true; + ref BufferRef buffer = ref _uniformBufferRefs[index]; + UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); doUpdate = true; } } @@ -640,17 +790,17 @@ public void SignalCommandBufferChange() { _dirty = DirtyFlags.All; - Array.Clear(_uniformSet); - Array.Clear(_storageSet); + _uniformSet.Clear(); + _storageSet.Clear(); } - private void SwapBuffer(Auto[] list, Auto from, Auto to) + private void SwapBuffer(BufferRef[] list, Auto from, Auto to) { for (int i = 0; i < list.Length; i++) { - if (list[i] == from) + if (list[i].Buffer == from) { - list[i] = to; + list[i].Buffer = to; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs index 75b18456acd1..ece044af7112 100644 --- a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs @@ -157,5 +157,10 @@ public void Swap(Auto from, Auto to) _buffer = to; } } + + public bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } } } diff --git a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs index 13a4f4c14fd0..24a2002b1368 100644 --- a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs @@ -39,7 +39,17 @@ public MultiFenceHolder(int size) /// Size of the buffer region being used, in bytes public void AddBufferUse(int cbIndex, int offset, int size) { - _bufferUsageBitmap.Add(cbIndex, offset, size); + _bufferUsageBitmap.Add(cbIndex, offset, size, false); + } + + public void AddBufferUse(int cbIndex, int offset, int size, bool write) + { + _bufferUsageBitmap.Add(cbIndex, offset, size, false); + + if (write) + { + _bufferUsageBitmap.Add(cbIndex, offset, size, true); + } } /// @@ -74,6 +84,11 @@ public bool IsBufferRangeInUse(int offset, int size) return _bufferUsageBitmap.OverlapsWith(offset, size); } + public bool IsBufferRangeInUse(int offset, int size, bool write) + { + return _bufferUsageBitmap.OverlapsWith(offset, size, write); + } + /// /// Adds a fence to the holder. /// diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs index ce6148e2f7cd..876c3fb51981 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -184,7 +184,7 @@ public void ClearBuffer(BufferHandle destination, int offset, int size, uint val { EndRenderPass(); - var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size).Value; + var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size, true).Value; BufferHolder.InsertBufferBarrier( Gd, @@ -453,6 +453,10 @@ public void DrawIndexedIndirect(BufferRange indirectBuffer) return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); @@ -482,10 +486,6 @@ public void DrawIndexedIndirect(BufferRange indirectBuffer) } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); Gd.Api.CmdDrawIndexedIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); @@ -499,15 +499,19 @@ public void DrawIndexedIndirectCount(BufferRange indirectBuffer, BufferRange par return; } + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; + + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); DrawCount++; - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - if (_indexBufferPattern != null) { // Convert the index buffer into a supported topology. @@ -551,14 +555,9 @@ public void DrawIndexedIndirectCount(BufferRange indirectBuffer, BufferRange par (uint)maxDrawCount, (uint)stride); } - } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); if (Gd.Capabilities.SupportsIndirectParameters) @@ -594,15 +593,15 @@ public void DrawIndirect(BufferRange indirectBuffer) // TODO: Support quads and other unsupported topologies. + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - Gd.Api.CmdDrawIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); } @@ -619,6 +618,14 @@ public void DrawIndirectCount(BufferRange indirectBuffer, BufferRange parameterB return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size, false).Value; + // TODO: Support quads and other unsupported topologies. RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); @@ -626,14 +633,6 @@ public void DrawIndirectCount(BufferRange indirectBuffer, BufferRange parameterB ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - Gd.DrawIndirectCountApi.CmdDrawIndirectCount( CommandBuffer, buffer, @@ -706,6 +705,28 @@ public bool IsCommandBufferActive(CommandBuffer cb) return CommandBuffer.Handle == cb.Handle; } + internal void Rebind(Auto buffer, int offset, int size) + { + _descriptorSetUpdater.Rebind(buffer, offset, size); + + /* + if (_indexBuffer.Overlaps(buffer, offset, size)) + { + _indexBuffer.BindIndexBuffer(Gd, Cbs); + } + */ + + for (int i = 0; i < _vertexBuffers.Length; i++) + { + if (_vertexBuffers[i].Overlaps(buffer, offset, size)) + { + _vertexBuffers[i].BindVertexBuffer(Gd, Cbs, (uint)i, ref _newState, _vertexBufferUpdater); + } + } + + _vertexBufferUpdater.Commit(Cbs); + } + public void SetAlphaTest(bool enable, float reference, GAL.CompareOp op) { // This is currently handled using shader specialization, as Vulkan does not support alpha test. diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs index 8026103e7f4e..43261aa39a36 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs @@ -14,6 +14,7 @@ class PipelineFull : PipelineBase, IPipeline private CounterQueueEvent _activeConditionalRender; private readonly List _pendingQueryCopies; + private readonly List _activeBufferMirrors; private ulong _byteWeight; @@ -24,6 +25,7 @@ public PipelineFull(VulkanRenderer gd, Device device) : base(gd, device) _activeQueries = new List<(QueryPool, bool)>(); _pendingQueryCopies = new(); _backingSwaps = new(); + _activeBufferMirrors = new List(); CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; } @@ -236,6 +238,12 @@ public void FlushCommandsImpl() Gd.RegisterFlush(); // Restore per-command buffer state. + foreach (BufferHolder buffer in _activeBufferMirrors) + { + buffer.ClearMirrors(); + } + + _activeBufferMirrors.Clear(); foreach ((var queryPool, var isOcclusion) in _activeQueries) { @@ -252,6 +260,11 @@ public void FlushCommandsImpl() Restore(); } + public void RegisterActiveMirror(BufferHolder buffer) + { + _activeBufferMirrors.Add(buffer); + } + public void BeginQuery(BufferedQuery query, QueryPool pool, bool needsReset, bool isOcclusion, bool fromSamplePool) { if (needsReset) diff --git a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs index 861155a30c00..59f14ba8d3aa 100644 --- a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs +++ b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs @@ -183,7 +183,7 @@ public void PoolReset(CommandBuffer cmd, int resetSequence) public void PoolCopy(CommandBufferScoped cbs) { - var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long)).Value; + var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long), true).Value; QueryResultFlags flags = QueryResultFlags.ResultWaitBit; diff --git a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs index 4e3c1deeafb7..7b060853b0e1 100644 --- a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs @@ -1,12 +1,27 @@ +using Ryujinx.Common; using System; using System.Collections.Generic; using System.Diagnostics; namespace Ryujinx.Graphics.Vulkan { + struct StagingBufferReserved + { + public readonly BufferHolder Buffer; + public readonly int Offset; + public readonly int Size; + + public StagingBufferReserved(BufferHolder buffer, int offset, int size) + { + Buffer = buffer; + Offset = offset; + Size = size; + } + } + class StagingBuffer : IDisposable { - private const int BufferSize = 16 * 1024 * 1024; + private const int BufferSize = 32 * 1024 * 1024; private int _freeOffset; private int _freeSize; @@ -130,13 +145,83 @@ public unsafe bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, Bu } } - endRenderPass(); + endRenderPass?.Invoke(); PushDataImpl(cbs, dst, dstOffset, data); return true; } + private StagingBufferReserved ReserveDataImpl(CommandBufferScoped cbs, int size, int alignment) + { + // Assumes the caller has already determined that there is enough space. + int offset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = offset - _freeOffset; + + int capacity = Math.Min(_freeSize, BufferSize - offset); + int reservedLength = size + padding; + if (capacity < size) + { + offset = 0; // Place at start. + reservedLength += capacity; + } + + _freeOffset = (_freeOffset + reservedLength) & (BufferSize - 1); + _freeSize -= reservedLength; + Debug.Assert(_freeSize >= 0); + + _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), reservedLength)); + + return new StagingBufferReserved(_buffer, offset, size); + } + + private int GetContiguousFreeSize(int alignment) + { + int alignedFreeOffset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = alignedFreeOffset - _freeOffset; + + // Free regions: + // - Aligned free offset to end (minimum free size - padding) + // - 0 to _freeOffset + freeSize wrapped (only if free area contains 0) + + int endOffset = (_freeOffset + _freeSize) & (BufferSize - 1); + + return Math.Max( + Math.Min(_freeSize - padding, BufferSize - alignedFreeOffset), + endOffset < _freeOffset ? Math.Min(_freeSize, endOffset) : 0 + ); + } + + /// + /// Reserve a range on the staging buffer for the current command buffer and upload data to it. + /// + /// Command buffer to reserve the data on + /// The data to upload + /// The required alignment for the buffer offset + /// The reserved range of the staging buffer + public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size, int alignment = 256) + { + if (size > BufferSize) + { + return null; + } + + // Temporary reserved data cannot be fragmented. + + if (GetContiguousFreeSize(alignment) < size) + { + FreeCompleted(); + + if (GetContiguousFreeSize(alignment) < size) + { + Console.WriteLine("no space :("); + return null; + } + } + + return ReserveDataImpl(cbs, size, alignment); + } + private bool WaitFreeCompleted(CommandBufferPool cbp) { if (_pendingCopies.TryPeek(out var pc)) diff --git a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs index 66951153dc60..0843dad4a556 100644 --- a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs @@ -129,27 +129,27 @@ public void SetStorage(BufferRange buffer) ReleaseImpl(); } - public BufferView GetBufferView(CommandBufferScoped cbs) + public BufferView GetBufferView(CommandBufferScoped cbs, bool write) { if (_bufferView == null) { _bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl); } - return _bufferView?.Get(cbs, _offset, _size).Value ?? default; + return _bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } - public BufferView GetBufferView(CommandBufferScoped cbs, GAL.Format format) + public BufferView GetBufferView(CommandBufferScoped cbs, GAL.Format format, bool write) { var vkFormat = FormatTable.GetFormat(format); if (vkFormat == VkFormat) { - return GetBufferView(cbs); + return GetBufferView(cbs, write); } if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var bufferView)) { - return bufferView.Get(cbs, _offset, _size).Value; + return bufferView.Get(cbs, _offset, _size, write).Value; } bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl); @@ -159,7 +159,7 @@ public BufferView GetBufferView(CommandBufferScoped cbs, GAL.Format format) (_selfManagedViews ??= new Dictionary>()).Add(format, bufferView); } - return bufferView?.Get(cbs, _offset, _size).Value ?? default; + return bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs index 2118bfaebadc..1529ab52b1e3 100644 --- a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs @@ -90,9 +90,10 @@ public void BindVertexBuffer(VulkanRenderer gd, CommandBufferScoped cbs, uint bi if (autoBuffer != null) { - var buffer = autoBuffer.Get(cbs, _offset, _size).Value; + int offset = _offset; + var buffer = autoBuffer.GetMirrorable(cbs, ref offset, _size, out _).Value; - updater.BindVertexBuffer(cbs, binding, buffer, (ulong)_offset, (ulong)_size, (ulong)_stride); + updater.BindVertexBuffer(cbs, binding, buffer, (ulong)offset, (ulong)_size, (ulong)_stride); } } @@ -101,6 +102,11 @@ public bool BoundEquals(Auto buffer) return _buffer == buffer; } + public bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public bool Matches(Auto buffer, int descriptorIndex, int offset, int size, int stride = 0) { return _buffer == buffer && DescriptorIndex == descriptorIndex && _offset == offset && _size == size && _stride == stride;