Skip to content

Commit

Permalink
Reduce the amount of descriptor pool allocations on Vulkan (#5673)
Browse files Browse the repository at this point in the history 
* Reduce the amount of descriptor pool allocations on Vulkan

* Formatting

* Slice can be simplified

* Make GetDescriptorPoolSizes static

* Adjust CanFit calculation so that TryAllocateDescriptorSets never fails

* Remove unused field
  • Loading branch information
@gdkchan
gdkchan authored Sep 26, 2023
1 parent 4a835bb commit 4744bde
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 57 deletions.
7 changes: 7 additions & 0 deletions src/Ryujinx.Graphics.Vulkan/CommandBufferPool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ private struct ReservedCommandBuffer
{
public bool InUse;
public bool InConsumption;
public int SubmissionCount;
public CommandBuffer CommandBuffer;
public FenceHolder Fence;
public SemaphoreHolder Semaphore;
Expand Down Expand Up @@ -193,6 +194,11 @@ public FenceHolder GetFence(int cbIndex)
return _commandBuffers[cbIndex].Fence;
}

public int GetSubmissionCount(int cbIndex)
{
return _commandBuffers[cbIndex].SubmissionCount;
}

private int FreeConsumed(bool wait)
{
int freeEntry = 0;
Expand Down Expand Up @@ -282,6 +288,7 @@ public unsafe void Return(
Debug.Assert(entry.CommandBuffer.Handle == cbs.CommandBuffer.Handle);
entry.InUse = false;
entry.InConsumption = true;
entry.SubmissionCount++;
_inUseCount--;

var commandBuffer = entry.CommandBuffer;
Expand Down
103 changes: 67 additions & 36 deletions src/Ryujinx.Graphics.Vulkan/DescriptorSetManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,44 +6,36 @@ namespace Ryujinx.Graphics.Vulkan
{
class DescriptorSetManager : IDisposable
{
private const uint DescriptorPoolMultiplier = 16;
public const uint MaxSets = 16;

public class DescriptorPoolHolder : IDisposable
{
public Vk Api { get; }
public Device Device { get; }

private readonly DescriptorPool _pool;
private readonly uint _capacity;
private int _freeDescriptors;
private int _totalSets;
private int _setsInUse;
private bool _done;

public unsafe DescriptorPoolHolder(Vk api, Device device)
public unsafe DescriptorPoolHolder(Vk api, Device device, ReadOnlySpan<DescriptorPoolSize> poolSizes, bool updateAfterBind)
{
Api = api;
Device = device;

var poolSizes = new[]
foreach (var poolSize in poolSizes)
{
new DescriptorPoolSize(DescriptorType.UniformBuffer, (1 + Constants.MaxUniformBufferBindings) * DescriptorPoolMultiplier),
new DescriptorPoolSize(DescriptorType.StorageBuffer, Constants.MaxStorageBufferBindings * DescriptorPoolMultiplier),
new DescriptorPoolSize(DescriptorType.CombinedImageSampler, Constants.MaxTextureBindings * DescriptorPoolMultiplier),
new DescriptorPoolSize(DescriptorType.StorageImage, Constants.MaxImageBindings * DescriptorPoolMultiplier),
new DescriptorPoolSize(DescriptorType.UniformTexelBuffer, Constants.MaxTextureBindings * DescriptorPoolMultiplier),
new DescriptorPoolSize(DescriptorType.StorageTexelBuffer, Constants.MaxImageBindings * DescriptorPoolMultiplier),
};

uint maxSets = (uint)poolSizes.Length * DescriptorPoolMultiplier;

_capacity = maxSets;
_freeDescriptors += (int)poolSize.DescriptorCount;
}

fixed (DescriptorPoolSize* pPoolsSize = poolSizes)
{
var descriptorPoolCreateInfo = new DescriptorPoolCreateInfo
{
SType = StructureType.DescriptorPoolCreateInfo,
MaxSets = maxSets,
Flags = updateAfterBind ? DescriptorPoolCreateFlags.UpdateAfterBindBit : DescriptorPoolCreateFlags.None,
MaxSets = MaxSets,
PoolSizeCount = (uint)poolSizes.Length,
PPoolSizes = pPoolsSize,
};
Expand All @@ -52,18 +44,22 @@ public unsafe DescriptorPoolHolder(Vk api, Device device)
}
}

public DescriptorSetCollection AllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout> layouts)
public unsafe DescriptorSetCollection AllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout> layouts, int consumedDescriptors)
{
TryAllocateDescriptorSets(layouts, isTry: false, out var dsc);
TryAllocateDescriptorSets(layouts, consumedDescriptors, isTry: false, out var dsc);
return dsc;
}

public bool TryAllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout> layouts, out DescriptorSetCollection dsc)
public bool TryAllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout> layouts, int consumedDescriptors, out DescriptorSetCollection dsc)
{
return TryAllocateDescriptorSets(layouts, isTry: true, out dsc);
return TryAllocateDescriptorSets(layouts, consumedDescriptors, isTry: true, out dsc);
}

private unsafe bool TryAllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout> layouts, bool isTry, out DescriptorSetCollection dsc)
private unsafe bool TryAllocateDescriptorSets(
ReadOnlySpan<DescriptorSetLayout> layouts,
int consumedDescriptors,
bool isTry,
out DescriptorSetCollection dsc)
{
Debug.Assert(!_done);

Expand All @@ -84,7 +80,7 @@ private unsafe bool TryAllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout>
var result = Api.AllocateDescriptorSets(Device, &descriptorSetAllocateInfo, pDescriptorSets);
if (isTry && result == Result.ErrorOutOfPoolMemory)
{
_totalSets = (int)_capacity;
_totalSets = (int)MaxSets;
_done = true;
DestroyIfDone();
dsc = default;
Expand All @@ -95,6 +91,7 @@ private unsafe bool TryAllocateDescriptorSets(ReadOnlySpan<DescriptorSetLayout>
}
}

_freeDescriptors -= consumedDescriptors;
_totalSets += layouts.Length;
_setsInUse += layouts.Length;

Expand All @@ -109,9 +106,15 @@ public void FreeDescriptorSets(DescriptorSetCollection dsc)
DestroyIfDone();
}

public bool CanFit(int count)
public bool CanFit(int setsCount, int descriptorsCount)
{
if (_totalSets + count <= _capacity)
// Try to determine if an allocation with the given parameters will succeed.
// An allocation may fail if the sets count or descriptors count exceeds the available counts
// of the pool.
// Not getting that right is not fatal, it will just create a new pool and try again,
// but it is less efficient.

if (_totalSets + setsCount <= MaxSets && _freeDescriptors >= descriptorsCount)
{
return true;
}
Expand Down Expand Up @@ -148,46 +151,74 @@ public void Dispose()
}

private readonly Device _device;
private DescriptorPoolHolder _currentPool;
private readonly DescriptorPoolHolder[] _currentPools;

public DescriptorSetManager(Device device)
public DescriptorSetManager(Device device, int poolCount)
{
_device = device;
_currentPools = new DescriptorPoolHolder[poolCount];
}

public Auto<DescriptorSetCollection> AllocateDescriptorSet(Vk api, DescriptorSetLayout layout)
public Auto<DescriptorSetCollection> AllocateDescriptorSet(
Vk api,
DescriptorSetLayout layout,
ReadOnlySpan<DescriptorPoolSize> poolSizes,
int poolIndex,
int consumedDescriptors,
bool updateAfterBind)
{
Span<DescriptorSetLayout> layouts = stackalloc DescriptorSetLayout[1];
layouts[0] = layout;
return AllocateDescriptorSets(api, layouts);
return AllocateDescriptorSets(api, layouts, poolSizes, poolIndex, consumedDescriptors, updateAfterBind);
}

public Auto<DescriptorSetCollection> AllocateDescriptorSets(Vk api, ReadOnlySpan<DescriptorSetLayout> layouts)
public Auto<DescriptorSetCollection> AllocateDescriptorSets(
Vk api,
ReadOnlySpan<DescriptorSetLayout> layouts,
ReadOnlySpan<DescriptorPoolSize> poolSizes,
int poolIndex,
int consumedDescriptors,
bool updateAfterBind)
{
// If we fail the first time, just create a new pool and try again.
if (!GetPool(api, layouts.Length).TryAllocateDescriptorSets(layouts, out var dsc))

var pool = GetPool(api, poolSizes, poolIndex, layouts.Length, consumedDescriptors, updateAfterBind);
if (!pool.TryAllocateDescriptorSets(layouts, consumedDescriptors, out var dsc))
{
dsc = GetPool(api, layouts.Length).AllocateDescriptorSets(layouts);
pool = GetPool(api, poolSizes, poolIndex, layouts.Length, consumedDescriptors, updateAfterBind);
dsc = pool.AllocateDescriptorSets(layouts, consumedDescriptors);
}

return new Auto<DescriptorSetCollection>(dsc);
}

private DescriptorPoolHolder GetPool(Vk api, int requiredCount)
private DescriptorPoolHolder GetPool(
Vk api,
ReadOnlySpan<DescriptorPoolSize> poolSizes,
int poolIndex,
int setsCount,
int descriptorsCount,
bool updateAfterBind)
{
if (_currentPool == null || !_currentPool.CanFit(requiredCount))
ref DescriptorPoolHolder currentPool = ref _currentPools[poolIndex];

if (currentPool == null || !currentPool.CanFit(setsCount, descriptorsCount))
{
_currentPool = new DescriptorPoolHolder(api, _device);
currentPool = new DescriptorPoolHolder(api, _device, poolSizes, updateAfterBind);
}

return _currentPool;
return currentPool;
}

protected virtual void Dispose(bool disposing)
{
if (disposing)
{
_currentPool?.Dispose();
for (int index = 0; index < _currentPools.Length; index++)
{
_currentPools[index]?.Dispose();
_currentPools[index] = null;
}
}
}

Expand Down
12 changes: 11 additions & 1 deletion src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ public BufferRef(Auto<DisposableBuffer> buffer, ref BufferRange range)
private BitMapStruct<Array2<long>> _uniformMirrored;
private BitMapStruct<Array2<long>> _storageMirrored;

private bool _updateDescriptorCacheCbIndex;

[Flags]
private enum DirtyFlags
{
Expand Down Expand Up @@ -218,6 +220,7 @@ internal void Rebind(Auto<DisposableBuffer> buffer, int offset, int size)
public void SetProgram(ShaderCollection program)
{
_program = program;
_updateDescriptorCacheCbIndex = true;
_dirty = DirtyFlags.All;
}

Expand Down Expand Up @@ -490,7 +493,13 @@ private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPo

var dummyBuffer = _dummyBuffer?.GetBuffer();

var dsc = program.GetNewDescriptorSetCollection(_gd, cbs.CommandBufferIndex, setIndex, out var isNew).Get(cbs);
if (_updateDescriptorCacheCbIndex)
{
_updateDescriptorCacheCbIndex = false;
program.UpdateDescriptorCacheCommandBufferIndex(cbs.CommandBufferIndex);
}

var dsc = program.GetNewDescriptorSetCollection(setIndex, out var isNew).Get(cbs);

if (!program.HasMinimalLayout)
{
Expand Down Expand Up @@ -697,6 +706,7 @@ private void Initialize(CommandBufferScoped cbs, int setIndex, DescriptorSetColl

public void SignalCommandBufferChange()
{
_updateDescriptorCacheCbIndex = true;
_dirty = DirtyFlags.All;

_uniformSet.Clear();
Expand Down
Loading

0 comments on commit 4744bde

Please sign in to comment.