using Silk.NET.Vulkan;
using Buffer = Silk.NET.Vulkan.Buffer;
namespace AcDream.App.Rendering.Gpu.Vk;
///
/// Campaign V slice V6a, plan §4.3 and §4.8: every transfer that has been asked
/// for but not yet recorded, plus the staging memory those transfers read from.
///
/// Why a queue rather than an immediate copy. Uploads arrive from
/// the streaming and texture-cache code at arbitrary moments, including with no
/// frame open at all. Vulkan copies must be recorded into a command buffer, and
/// they must be recorded OUTSIDE a dynamic-rendering block. So requests
/// accumulate here and are drained into the frame's command buffer at the one
/// moment both conditions hold: immediately before a pass begins. That is the
/// direct analogue of the GL backend's "flush immediately before every draw"
/// discipline, moved to the coarser granularity Vulkan actually needs.
///
/// One batched barrier, not one per copy. The drain emits a single
/// buffer memory barrier covering every copy it recorded, moving the whole batch
/// from transfer writes to the vertex/index/indirect/shader reads that follow.
/// Plan §4.8 budgets four to six barriers per frame; this is one of them.
///
/// Staging exhaustion is not an error. When the ring cannot serve a
/// request — the payload is larger than the whole ring, or unretired frames hold
/// the space — the upload takes a temporary dedicated staging buffer that is
/// retired through the ledger. The transfer is equally correct and equally
/// ordered; it just costs one allocation. See
/// for why that is a policy rather than a
/// workaround.
///
internal sealed unsafe class VulkanUploadQueue : IDisposable
{
/// Plan §4.3: a 48 MiB persistently mapped staging ring.
internal const ulong DefaultStagingCapacityBytes = 48UL * 1024 * 1024;
private readonly Silk.NET.Vulkan.Vk _vk;
private readonly Device _device;
private readonly VulkanDeviceMemoryAllocator _allocator;
private readonly VulkanFrameFlightController _flights;
private readonly VulkanStagingRingState _ringState;
private readonly VulkanDebugNames _debugNames;
private readonly Buffer _stagingBuffer;
private readonly VulkanAllocation _stagingAllocation;
private readonly List _bufferCopies = [];
private readonly List _imageCopies = [];
private readonly List _temporaries = [];
private bool _disposed;
private readonly record struct BufferCopy2(Buffer Source, Buffer Destination, ulong SourceOffset, ulong DestinationOffset, ulong SizeBytes);
private readonly record struct ImageCopy2(
Buffer Source,
ulong SourceOffset,
Image Destination,
uint MipLevel,
uint Layer,
uint Width,
uint Height);
private readonly record struct TemporaryStaging(Buffer Buffer, VulkanAllocation Allocation);
internal VulkanUploadQueue(
Silk.NET.Vulkan.Vk vk,
Device device,
VulkanDeviceMemoryAllocator allocator,
VulkanFrameFlightController flights,
VulkanDebugNames debugNames,
ulong stagingCapacityBytes = DefaultStagingCapacityBytes)
{
_vk = vk ?? throw new ArgumentNullException(nameof(vk));
_device = device;
_allocator = allocator ?? throw new ArgumentNullException(nameof(allocator));
_flights = flights ?? throw new ArgumentNullException(nameof(flights));
_debugNames = debugNames ?? throw new ArgumentNullException(nameof(debugNames));
_ringState = new VulkanStagingRingState(stagingCapacityBytes);
(_stagingBuffer, _stagingAllocation) = CreateHostBuffer(
stagingCapacityBytes,
BufferUsageFlags.TransferSrcBit,
"vk-staging-ring");
}
///
/// Images touched by this batch, and the layout each is in on entry.
///
/// The entry layout is not always UNDEFINED, and that
/// distinction is load-bearing. UNDEFINED lets the driver discard the
/// existing contents, which is exactly right for the first upload into a
/// fresh image and exactly wrong for the incremental array-layer fills that
/// mirror ManagedGLTextureArray — discarding there would erase every
/// layer uploaded earlier. The first writer of a batch records the layout it
/// found the image in, and that is what the barrier names.
///
private readonly Dictionary _imageEntryLayouts = [];
/// Mip chains to generate with vkCmdBlitImage after this batch's copies land.
private readonly List _mipBlits = [];
private readonly record struct MipBlitRequest(
Image Image,
int Width,
int Height,
int MipLevelCount,
int LayerCount);
internal int PendingBufferCopyCount => _bufferCopies.Count;
internal int PendingImageCopyCount => _imageCopies.Count;
internal ulong StagingLiveBytes => _ringState.LiveBytes;
/// Stages and queues a copy into .
internal void StageBufferWrite(
Buffer destination,
ulong destinationOffsetBytes,
ReadOnlySpan data,
string ownerName)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (data.IsEmpty)
return;
(Buffer source, ulong sourceOffset) = Stage(data, ownerName);
_bufferCopies.Add(new BufferCopy2(
source,
destination,
sourceOffset,
destinationOffsetBytes,
(ulong)data.Length));
}
/// Stages and queues a copy into one mip level of one array layer.
internal void StageImageWrite(
Image destination,
int mipLevel,
int layer,
int width,
int height,
ImageLayout entryLayout,
ReadOnlySpan data,
string ownerName)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (data.IsEmpty)
return;
// Vulkan requires a buffer-to-image copy's source offset to be a
// multiple of 4 and of the texel/block size; 16 covers every format
// acdream uses (BC3's 16-byte block is the largest).
(Buffer source, ulong sourceOffset) = Stage(data, ownerName, alignmentBytes: 16);
_imageCopies.Add(new ImageCopy2(
source,
sourceOffset,
destination,
(uint)mipLevel,
(uint)layer,
(uint)width,
(uint)height));
RecordEntryLayout(destination, entryLayout);
}
///
/// Queues a vkCmdBlitImage mip chain for an uncompressed image. BC
/// images cannot use this — a compressed image is not a legal blit
/// destination — and take a CPU-built chain instead (plan §4.3).
///
internal void EnqueueMipBlit(
Image image,
int width,
int height,
int mipLevelCount,
int layerCount,
ImageLayout entryLayout)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (mipLevelCount <= 1)
return;
_mipBlits.Add(new MipBlitRequest(image, width, height, mipLevelCount, layerCount));
RecordEntryLayout(image, entryLayout);
}
private void RecordEntryLayout(Image image, ImageLayout entryLayout)
{
// First writer of the batch wins: a later writer that found the image
// already in TRANSFER_DST is describing this batch's own effect, not the
// layout the batch started from.
_imageEntryLayouts.TryAdd(image, entryLayout);
}
/// Queues a device-side buffer copy — the mesh arena's grow-and-copy migration.
internal void EnqueueBufferCopy(
Buffer source,
Buffer destination,
ulong sourceOffsetBytes,
ulong destinationOffsetBytes,
ulong byteCount)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (byteCount == 0)
return;
_bufferCopies.Add(new BufferCopy2(
source,
destination,
sourceOffsetBytes,
destinationOffsetBytes,
byteCount));
}
///
/// Records every pending transfer into and
/// clears the queue. Must be called outside a dynamic-rendering block.
/// Returns false when there was nothing to do.
///
internal bool Record(CommandBuffer commands)
{
if (_bufferCopies.Count == 0 && _imageCopies.Count == 0 && _mipBlits.Count == 0)
return false;
if (_imageEntryLayouts.Count > 0)
TransitionImagesToTransfer(commands);
foreach (BufferCopy2 copy in _bufferCopies)
{
var region = new BufferCopy
{
SrcOffset = copy.SourceOffset,
DstOffset = copy.DestinationOffset,
Size = copy.SizeBytes,
};
_vk.CmdCopyBuffer(commands, copy.Source, copy.Destination, 1, ®ion);
}
foreach (ImageCopy2 copy in _imageCopies)
{
var region = new BufferImageCopy
{
BufferOffset = copy.SourceOffset,
BufferRowLength = 0,
BufferImageHeight = 0,
ImageSubresource = new ImageSubresourceLayers
{
AspectMask = ImageAspectFlags.ColorBit,
MipLevel = copy.MipLevel,
BaseArrayLayer = copy.Layer,
LayerCount = 1,
},
ImageOffset = new Offset3D(0, 0, 0),
ImageExtent = new Extent3D(copy.Width, copy.Height, 1),
};
_vk.CmdCopyBufferToImage(
commands,
copy.Source,
copy.Destination,
ImageLayout.TransferDstOptimal,
1,
®ion);
}
foreach (MipBlitRequest blit in _mipBlits)
RecordMipBlit(commands, blit);
if (_imageEntryLayouts.Count > 0)
TransitionImagesToShaderRead(commands);
// One buffer barrier for the whole batch: transfer writes become
// readable by every consumer stage a copied buffer can feed.
if (_bufferCopies.Count > 0)
{
var barrier = new MemoryBarrier2
{
SType = StructureType.MemoryBarrier2,
SrcStageMask = PipelineStageFlags2.AllTransferBit,
SrcAccessMask = AccessFlags2.TransferWriteBit,
DstStageMask = PipelineStageFlags2.VertexInputBit
| PipelineStageFlags2.VertexShaderBit
| PipelineStageFlags2.FragmentShaderBit
| PipelineStageFlags2.DrawIndirectBit,
DstAccessMask = AccessFlags2.VertexAttributeReadBit
| AccessFlags2.IndexReadBit
| AccessFlags2.ShaderReadBit
| AccessFlags2.UniformReadBit
| AccessFlags2.IndirectCommandReadBit,
};
var dependency = new DependencyInfo
{
SType = StructureType.DependencyInfo,
MemoryBarrierCount = 1,
PMemoryBarriers = &barrier,
};
_vk.CmdPipelineBarrier2(commands, &dependency);
}
_bufferCopies.Clear();
_imageCopies.Clear();
_mipBlits.Clear();
_imageEntryLayouts.Clear();
return true;
}
private static ImageSubresourceRange WholeColorImage => new()
{
AspectMask = ImageAspectFlags.ColorBit,
BaseMipLevel = 0,
LevelCount = Silk.NET.Vulkan.Vk.RemainingMipLevels,
BaseArrayLayer = 0,
LayerCount = Silk.NET.Vulkan.Vk.RemainingArrayLayers,
};
private void TransitionImagesToTransfer(CommandBuffer commands)
{
var barriers = new ImageMemoryBarrier2[_imageEntryLayouts.Count];
int index = 0;
foreach ((Image image, ImageLayout entryLayout) in _imageEntryLayouts)
{
barriers[index++] = new ImageMemoryBarrier2
{
SType = StructureType.ImageMemoryBarrier2,
SrcStageMask = PipelineStageFlags2.AllCommandsBit,
SrcAccessMask = AccessFlags2.None,
DstStageMask = PipelineStageFlags2.AllTransferBit,
DstAccessMask = AccessFlags2.TransferWriteBit | AccessFlags2.TransferReadBit,
OldLayout = entryLayout,
NewLayout = ImageLayout.TransferDstOptimal,
SrcQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
DstQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
Image = image,
SubresourceRange = WholeColorImage,
};
}
SubmitBarriers(commands, barriers);
}
private void TransitionImagesToShaderRead(CommandBuffer commands)
{
var barriers = new ImageMemoryBarrier2[_imageEntryLayouts.Count];
int index = 0;
foreach (Image image in _imageEntryLayouts.Keys)
{
barriers[index++] = new ImageMemoryBarrier2
{
SType = StructureType.ImageMemoryBarrier2,
SrcStageMask = PipelineStageFlags2.AllTransferBit,
SrcAccessMask = AccessFlags2.TransferWriteBit,
DstStageMask = PipelineStageFlags2.FragmentShaderBit | PipelineStageFlags2.VertexShaderBit,
DstAccessMask = AccessFlags2.ShaderReadBit,
OldLayout = ImageLayout.TransferDstOptimal,
NewLayout = ImageLayout.ShaderReadOnlyOptimal,
SrcQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
DstQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
Image = image,
SubresourceRange = WholeColorImage,
};
}
SubmitBarriers(commands, barriers);
}
private void SubmitBarriers(CommandBuffer commands, ImageMemoryBarrier2[] barriers)
{
if (barriers.Length == 0)
return;
fixed (ImageMemoryBarrier2* first = barriers)
{
var dependency = new DependencyInfo
{
SType = StructureType.DependencyInfo,
ImageMemoryBarrierCount = (uint)barriers.Length,
PImageMemoryBarriers = first,
};
_vk.CmdPipelineBarrier2(commands, &dependency);
}
}
///
/// Halves level N into level N+1 with a linear blit, all layers at once.
///
/// Each source level is moved to TRANSFER_SRC for its blit and then
/// moved BACK to TRANSFER_DST. Leaving the chain in mixed layouts would be
/// one barrier cheaper and would then need the batch's final
/// shader-read transition to name a different old layout per level; ending
/// every level in the same layout is what lets that final transition stay
/// one barrier per image.
///
private void RecordMipBlit(CommandBuffer commands, in MipBlitRequest request)
{
int width = request.Width;
int height = request.Height;
for (uint level = 1; level < request.MipLevelCount; level++)
{
int nextWidth = Math.Max(1, width / 2);
int nextHeight = Math.Max(1, height / 2);
TransitionMipLevel(
commands,
request.Image,
level - 1,
ImageLayout.TransferDstOptimal,
ImageLayout.TransferSrcOptimal,
AccessFlags2.TransferWriteBit,
AccessFlags2.TransferReadBit);
var blit = new ImageBlit2
{
SType = StructureType.ImageBlit2,
SrcSubresource = new ImageSubresourceLayers
{
AspectMask = ImageAspectFlags.ColorBit,
MipLevel = level - 1,
BaseArrayLayer = 0,
LayerCount = (uint)request.LayerCount,
},
DstSubresource = new ImageSubresourceLayers
{
AspectMask = ImageAspectFlags.ColorBit,
MipLevel = level,
BaseArrayLayer = 0,
LayerCount = (uint)request.LayerCount,
},
};
blit.SrcOffsets.Element0 = new Offset3D(0, 0, 0);
blit.SrcOffsets.Element1 = new Offset3D(width, height, 1);
blit.DstOffsets.Element0 = new Offset3D(0, 0, 0);
blit.DstOffsets.Element1 = new Offset3D(nextWidth, nextHeight, 1);
var info = new BlitImageInfo2
{
SType = StructureType.BlitImageInfo2,
SrcImage = request.Image,
SrcImageLayout = ImageLayout.TransferSrcOptimal,
DstImage = request.Image,
DstImageLayout = ImageLayout.TransferDstOptimal,
RegionCount = 1,
PRegions = &blit,
Filter = Filter.Linear,
};
_vk.CmdBlitImage2(commands, &info);
TransitionMipLevel(
commands,
request.Image,
level - 1,
ImageLayout.TransferSrcOptimal,
ImageLayout.TransferDstOptimal,
AccessFlags2.TransferReadBit,
AccessFlags2.TransferWriteBit);
width = nextWidth;
height = nextHeight;
}
}
private void TransitionMipLevel(
CommandBuffer commands,
Image image,
uint level,
ImageLayout oldLayout,
ImageLayout newLayout,
AccessFlags2 sourceAccess,
AccessFlags2 destinationAccess)
{
var barrier = new ImageMemoryBarrier2
{
SType = StructureType.ImageMemoryBarrier2,
SrcStageMask = PipelineStageFlags2.AllTransferBit,
SrcAccessMask = sourceAccess,
DstStageMask = PipelineStageFlags2.AllTransferBit,
DstAccessMask = destinationAccess,
OldLayout = oldLayout,
NewLayout = newLayout,
SrcQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
DstQueueFamilyIndex = Silk.NET.Vulkan.Vk.QueueFamilyIgnored,
Image = image,
SubresourceRange = new ImageSubresourceRange
{
AspectMask = ImageAspectFlags.ColorBit,
BaseMipLevel = level,
LevelCount = 1,
BaseArrayLayer = 0,
LayerCount = Silk.NET.Vulkan.Vk.RemainingArrayLayers,
},
};
var dependency = new DependencyInfo
{
SType = StructureType.DependencyInfo,
ImageMemoryBarrierCount = 1,
PImageMemoryBarriers = &barrier,
};
_vk.CmdPipelineBarrier2(commands, &dependency);
}
/// Reclaims staging bytes and temporary buffers belonging to completed frames.
internal void ReleaseCompleted(long completedSerial) => _ringState.Release(completedSerial);
private (Buffer Buffer, ulong Offset) Stage(
ReadOnlySpan data,
string ownerName,
ulong alignmentBytes = 4)
{
long serial = _flights.OpenSerial != 0 ? _flights.OpenSerial : _flights.SubmittedSerial + 1;
if (_ringState.TryAllocate(data.Length, alignmentBytes, serial, out ulong offset))
{
data.CopyTo(_stagingAllocation.AsSpan().Slice((int)offset, data.Length));
return (_stagingBuffer, offset);
}
(Buffer temporary, VulkanAllocation allocation) = CreateHostBuffer(
(ulong)data.Length,
BufferUsageFlags.TransferSrcBit,
$"vk-staging-temp-{ownerName}");
data.CopyTo(allocation.AsSpan());
_temporaries.Add(new TemporaryStaging(temporary, allocation));
Buffer captured = temporary;
VulkanAllocation capturedAllocation = allocation;
_flights.Retire(() =>
{
_vk.DestroyBuffer(_device, captured, null);
_allocator.Free(capturedAllocation);
_temporaries.RemoveAll(entry => entry.Buffer.Handle == captured.Handle);
});
return (temporary, 0);
}
private (Buffer Buffer, VulkanAllocation Allocation) CreateHostBuffer(
ulong sizeBytes,
BufferUsageFlags usage,
string name)
{
var create = new BufferCreateInfo
{
SType = StructureType.BufferCreateInfo,
Size = sizeBytes,
Usage = usage,
SharingMode = SharingMode.Exclusive,
};
VulkanInterop.Check(
_vk.CreateBuffer(_device, &create, null, out Buffer buffer),
$"vkCreateBuffer ({name})");
_vk.GetBufferMemoryRequirements(_device, buffer, out MemoryRequirements requirements);
VulkanAllocation allocation = _allocator.Allocate(
requirements,
GpuMemoryResidency.HostWritable,
name);
VulkanInterop.Check(
_vk.BindBufferMemory(_device, buffer, allocation.Memory, allocation.OffsetBytes),
$"vkBindBufferMemory ({name})");
_debugNames.NameBuffer(buffer, name);
return (buffer, allocation);
}
public void Dispose()
{
if (_disposed)
return;
_disposed = true;
foreach (TemporaryStaging temporary in _temporaries)
{
_vk.DestroyBuffer(_device, temporary.Buffer, null);
_allocator.Free(temporary.Allocation);
}
_temporaries.Clear();
_vk.DestroyBuffer(_device, _stagingBuffer, null);
_allocator.Free(_stagingAllocation);
_ringState.Reset();
_bufferCopies.Clear();
_imageCopies.Clear();
}
}