From 26090e2421a2ed2e67fa5d9ada97b3d871b81343 Mon Sep 17 00:00:00 2001 From: Rua Date: Fri, 29 Nov 2024 14:42:42 +0100 Subject: [PATCH 1/8] Add sparse binding flags for buffers and images --- vulkano/src/acceleration_structure.rs | 15 +- vulkano/src/buffer/mod.rs | 19 +- vulkano/src/buffer/sys.rs | 123 ++++++++----- vulkano/src/image/mod.rs | 15 +- vulkano/src/image/sys.rs | 239 ++++++++++++++++++++++++-- vulkano/src/image/view.rs | 11 ++ vulkano/src/memory/device_memory.rs | 20 +++ 7 files changed, 369 insertions(+), 73 deletions(-) diff --git a/vulkano/src/acceleration_structure.rs b/vulkano/src/acceleration_structure.rs index 422bfe1805..a98f4ebe30 100644 --- a/vulkano/src/acceleration_structure.rs +++ b/vulkano/src/acceleration_structure.rs @@ -85,7 +85,7 @@ //! [`WriteDescriptorSet::acceleration_structure`]: crate::descriptor_set::WriteDescriptorSet::acceleration_structure use crate::{ - buffer::{BufferUsage, IndexBuffer, Subbuffer}, + buffer::{BufferCreateFlags, BufferUsage, IndexBuffer, Subbuffer}, device::{Device, DeviceOwned}, format::{Format, FormatFeatures}, instance::InstanceOwnedDebugWrapper, @@ -379,6 +379,19 @@ impl AccelerationStructureCreateInfo { })); } + if buffer + .buffer() + .flags() + .intersects(BufferCreateFlags::SPARSE_RESIDENCY) + { + return Err(Box::new(ValidationError { + context: "buffer.buffer().flags()".into(), + problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(), + vuids: &["VUID-VkAccelerationStructureCreateInfoKHR-buffer-03615"], + ..Default::default() + })); + } + // VUID-VkAccelerationStructureCreateInfoKHR-offset-03616 // Ensured by the definition of `Subbuffer`. diff --git a/vulkano/src/buffer/mod.rs b/vulkano/src/buffer/mod.rs index 4631c6cfd5..a5428c466f 100644 --- a/vulkano/src/buffer/mod.rs +++ b/vulkano/src/buffer/mod.rs @@ -377,8 +377,10 @@ impl Buffer { allocation_info: AllocationCreateInfo, layout: DeviceLayout, ) -> Result, Validated> { - // TODO: Enable once sparse binding materializes - // assert!(!allocate_info.flags.contains(BufferCreateFlags::SPARSE_BINDING)); + assert!(layout.alignment().as_devicesize() <= 64); + assert!(!create_info + .flags + .contains(BufferCreateFlags::SPARSE_BINDING)); assert_eq!( create_info.size, 0, @@ -802,25 +804,24 @@ vulkan_bitflags! { /// Flags specifying additional properties of a buffer. BufferCreateFlags = BufferCreateFlags(u32); - /* TODO: enable - /// The buffer will be backed by sparse memory binding (through queue commands) instead of - /// regular binding (through [`bind_memory`]). + /// The buffer will be backed by sparse memory binding (through the [`bind_sparse`] queue + /// command) instead of regular binding (through [`bind_memory`]). /// /// The [`sparse_binding`] feature must be enabled on the device. /// + /// [`bind_sparse`]: crate::device::queue::QueueGuard::bind_sparse /// [`bind_memory`]: sys::RawBuffer::bind_memory /// [`sparse_binding`]: crate::device::DeviceFeatures::sparse_binding - SPARSE_BINDING = SPARSE_BINDING,*/ + SPARSE_BINDING = SPARSE_BINDING, - /* TODO: enable /// The buffer can be used without being fully resident in memory at the time of use. /// - /// This requires the `sparse_binding` flag as well. + /// This requires the [`BufferCreateFlags::SPARSE_BINDING`] flag as well. /// /// The [`sparse_residency_buffer`] feature must be enabled on the device. /// /// [`sparse_residency_buffer`]: crate::device::DeviceFeatures::sparse_residency_buffer - SPARSE_RESIDENCY = SPARSE_RESIDENCY,*/ + SPARSE_RESIDENCY = SPARSE_RESIDENCY, /* TODO: enable /// The buffer's memory can alias with another buffer or a different part of the same buffer. diff --git a/vulkano/src/buffer/sys.rs b/vulkano/src/buffer/sys.rs index 22a246009e..ed3d3ce836 100644 --- a/vulkano/src/buffer/sys.rs +++ b/vulkano/src/buffer/sys.rs @@ -69,6 +69,11 @@ impl RawBuffer { .validate(device) .map_err(|err| err.add_context("create_info"))?; + // TODO: sparse_address_space_size and extended_sparse_address_space_size limits + // VUID-vkCreateBuffer-flags-00911 + // VUID-vkCreateBuffer-flags-09383 + // VUID-vkCreateBuffer-flags-09384 + Ok(()) } @@ -245,6 +250,7 @@ impl RawBuffer { } /// Binds device memory to this buffer. + #[inline] pub fn bind_memory( self, allocation: ResourceMemory, @@ -261,6 +267,15 @@ impl RawBuffer { &self, allocation: &ResourceMemory, ) -> Result<(), Box> { + if self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "self.flags()".into(), + problem: "contains `BufferCreateFlags::SPARSE_BINDING`".into(), + vuids: &["VUID-VkBindBufferMemoryInfo-buffer-01030"], + ..Default::default() + })); + } + assert_ne!(allocation.allocation_type(), AllocationType::NonLinear); let physical_device = self.device().physical_device(); @@ -277,10 +292,6 @@ impl RawBuffer { // VUID-VkBindBufferMemoryInfo-buffer-07459 // Ensured by taking ownership of `RawBuffer`. - // VUID-VkBindBufferMemoryInfo-buffer-01030 - // Currently ensured by not having sparse binding flags, but this needs to be checked once - // those are enabled. - // VUID-VkBindBufferMemoryInfo-memoryOffset-01031 // Assume that `allocation` was created correctly. @@ -520,6 +531,34 @@ impl RawBuffer { Buffer::from_raw(self, BufferMemory::External) } + /// Converts a raw buffer, that was created with the [`BufferCreateInfo::SPARSE_BINDING`] flag, + /// into a full buffer without binding any memory. + /// + /// # Safety + /// + /// - If `self.flags()` does not contain [`BufferCreateFlags::SPARSE_RESIDENCY`], then the + /// buffer must be fully bound with memory before its memory is accessed by the device. + /// - If `self.flags()` contains [`BufferCreateFlags::SPARSE_RESIDENCY`], then you must ensure + /// that any reads from the buffer are prepared to handle unexpected or inconsistent values, + /// as determined by the [`Properties::residency_non_resident_strict`] device property. + /// + /// [`Properties::residency_non_resident_strict`]: crate::device::Properties::residency_non_resident_strict + pub unsafe fn into_buffer_sparse(self) -> Result, RawBuffer)> { + if !self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { + return Err(( + Box::new(ValidationError { + context: "self.flags()".into(), + problem: "does not contain `BufferCreateFlags::SPARSE_BINDING`".into(), + ..Default::default() + }) + .into(), + self, + )); + } + + Ok(Buffer::from_raw(self, BufferMemory::Sparse)) + } + /// Returns the memory requirements for this buffer. pub fn memory_requirements(&self) -> &MemoryRequirements { &self.memory_requirements @@ -678,45 +717,6 @@ impl BufferCreateInfo { })); } - /* Enable when sparse binding is properly handled - if let Some(sparse_level) = sparse { - if !device.enabled_features().sparse_binding { - return Err(Box::new(ValidationError { - context: "sparse".into(), - problem: "is `Some`".into(), - requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature( - "sparse_binding", - )])]), - vuids: &["VUID-VkBufferCreateInfo-flags-00915"], - })); - } - - if sparse_level.sparse_residency && !device.enabled_features().sparse_residency_buffer { - return Err(Box::new(ValidationError { - context: "sparse".into(), - problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(), - requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature( - "sparse_residency_buffer", - )])]), - vuids: &["VUID-VkBufferCreateInfo-flags-00916"], - })); - } - - if sparse_level.sparse_aliased && !device.enabled_features().sparse_residency_aliased { - return Err(Box::new(ValidationError { - context: "sparse".into(), - problem: "contains `BufferCreateFlags::SPARSE_ALIASED`".into(), - requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature( - "sparse_residency_aliased", - )])]), - vuids: &["VUID-VkBufferCreateInfo-flags-00917"], - })); - } - - // TODO: - // VUID-VkBufferCreateInfo-flags-00918 - }*/ - match sharing { Sharing::Exclusive => (), Sharing::Concurrent(queue_family_indices) => { @@ -761,6 +761,43 @@ impl BufferCreateInfo { } } + if flags.intersects(BufferCreateFlags::SPARSE_BINDING) { + if !device.enabled_features().sparse_binding { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `BufferCreateFlags::SPARSE_BINDING`".into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::DeviceFeature( + "sparse_binding", + )])]), + vuids: &["VUID-VkBufferCreateInfo-flags-00915"], + })); + } + } + + if flags.intersects(BufferCreateFlags::SPARSE_RESIDENCY) { + if !flags.intersects(BufferCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`, but does not also \ + contain `BufferCreateFlags::SPARSE_BINDING`" + .into(), + vuids: &["VUID-VkBufferCreateInfo-flags-00918"], + ..Default::default() + })); + } + + if !device.enabled_features().sparse_residency_buffer { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::DeviceFeature( + "sparse_residency_buffer", + )])]), + vuids: &["VUID-VkBufferCreateInfo-flags-00916"], + })); + } + } + if let Some(max_buffer_size) = device.physical_device().properties().max_buffer_size { if size > max_buffer_size { return Err(Box::new(ValidationError { diff --git a/vulkano/src/image/mod.rs b/vulkano/src/image/mod.rs index e76e88085a..4734f0b7d3 100644 --- a/vulkano/src/image/mod.rs +++ b/vulkano/src/image/mod.rs @@ -980,20 +980,21 @@ vulkan_bitflags! { /// Flags specifying additional properties of an image. ImageCreateFlags = ImageCreateFlags(u32); - /* TODO: enable - /// The image will be backed by sparse memory binding (through queue commands) instead of - /// regular binding (through [`bind_memory`]). + /// The image will be backed by sparse memory binding (through the [`bind_sparse`] queue + /// command) instead of regular binding (through [`bind_memory`]). /// /// The [`sparse_binding`] feature must be enabled on the device. /// + /// [`bind_sparse`]: crate::device::queue::QueueGuard::bind_sparse /// [`bind_memory`]: sys::RawImage::bind_memory /// [`sparse_binding`]: crate::device::DeviceFeatures::sparse_binding - SPARSE_BINDING = SPARSE_BINDING,*/ + SPARSE_BINDING = SPARSE_BINDING, - /* TODO: enable /// The image can be used without being fully resident in memory at the time of use. + /// It also allows non-opaque sparse binding operations, using the dimensions of the image, + /// to be performed. /// - /// This requires the `sparse_binding` flag as well. + /// This requires the [`ImageCreateFlags::SPARSE_BINDING`] flag as well. /// /// Depending on the image type, either the [`sparse_residency_image2_d`] or the /// [`sparse_residency_image3_d`] feature must be enabled on the device. @@ -1009,7 +1010,7 @@ vulkan_bitflags! { /// [`sparse_residency4_samples`]: crate::device::DeviceFeatures::sparse_residency4_samples /// [`sparse_residency8_samples`]: crate::device::DeviceFeatures::sparse_residency8_samples /// [`sparse_residency16_samples`]: crate::device::DeviceFeatures::sparse_residency16_samples - SPARSE_RESIDENCY = SPARSE_RESIDENCY,*/ + SPARSE_RESIDENCY = SPARSE_RESIDENCY, /* TODO: enable /// The buffer's memory can alias with another image or a different part of the same image. diff --git a/vulkano/src/image/sys.rs b/vulkano/src/image/sys.rs index e0e151a279..f436a006af 100644 --- a/vulkano/src/image/sys.rs +++ b/vulkano/src/image/sys.rs @@ -93,6 +93,11 @@ impl RawImage { .validate(device) .map_err(|err| err.add_context("create_info"))?; + // TODO: sparse_address_space_size and extended_sparse_address_space_size limits + // VUID-vkCreateImage-flags-00939 + // VUID-vkCreateImage-flags-09385 + // VUID-vkCreateImage-flags-09386 + Ok(()) } @@ -355,8 +360,11 @@ impl RawImage { ) } - #[allow(dead_code)] // Remove when sparse memory is implemented fn get_sparse_memory_requirements(&self) -> Vec { + if !self.flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) { + return Vec::new(); + } + let device = &self.device; let fns = self.device.fns(); @@ -512,6 +520,15 @@ impl RawImage { &self, allocations: &[ResourceMemory], ) -> Result<(), Box> { + if self.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "self.flags()".into(), + problem: "contains `ImageCreateFlags::SPARSE_BINDING`".into(), + vuids: &["VUID-VkBindImageMemoryInfo-image-01045"], + ..Default::default() + })); + } + let physical_device = self.device().physical_device(); if self.flags.intersects(ImageCreateFlags::DISJOINT) { @@ -601,10 +618,6 @@ impl RawImage { // VUID-VkBindImageMemoryInfo-image-07460 // Ensured by taking ownership of `RawImage`. - // VUID-VkBindImageMemoryInfo-image-01045 - // Currently ensured by not having sparse binding flags, but this needs to be checked - // once those are enabled. - // VUID-VkBindImageMemoryInfo-memoryOffset-01046 // Assume that `allocation` was created correctly. @@ -947,11 +960,59 @@ impl RawImage { return Err((VulkanError::from(err), self, allocations.into_iter())); } + let layout = self.default_layout(); + + Ok(Image::from_raw( + self, + ImageMemory::Normal(allocations), + layout, + )) + } + + /// Converts a raw image, that was created with the [`ImageCreateInfo::SPARSE_BINDING`] flag, + /// into a full image without binding any memory. + /// + /// # Safety + /// + /// - If `self.flags()` does not contain [`ImageCreateFlags::SPARSE_RESIDENCY`], then the image + /// must be fully bound with memory before any memory is accessed by the device. + /// - If `self.flags()` contains [`ImageCreateFlags::SPARSE_RESIDENCY`], then if the sparse + /// memory requirements include [`ImageAspects::METADATA`], then the metadata mip tail must + /// be fully bound with memory before any memory is accessed by the device. + /// - If `self.flags()` contains [`ImageCreateFlags::SPARSE_RESIDENCY`], then you must ensure + /// that any reads from the image are prepared to handle unexpected or inconsistent values, + /// as determined by the [`Properties::residency_non_resident_strict`] device property. + /// + /// [`Properties::residency_non_resident_strict`]: crate::device::Properties::residency_non_resident_strict + pub unsafe fn into_image_sparse(self) -> Result, RawImage)> { + if !self.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(( + Box::new(ValidationError { + context: "self.flags()".into(), + problem: "does not contain `ImageCreateFlags::SPARSE_BINDING`".into(), + ..Default::default() + }) + .into(), + self, + )); + } + + let sparse_image_memory_requirements = self.get_sparse_memory_requirements(); + let layout = self.default_layout(); + + Ok(Image::from_raw( + self, + ImageMemory::Sparse(sparse_image_memory_requirements), + layout, + )) + } + + fn default_layout(&self) -> ImageLayout { let usage = self .usage .difference(ImageUsage::TRANSFER_SRC | ImageUsage::TRANSFER_DST); - let layout = if usage.intersects(ImageUsage::SAMPLED | ImageUsage::INPUT_ATTACHMENT) + if usage.intersects(ImageUsage::SAMPLED | ImageUsage::INPUT_ATTACHMENT) && usage .difference(ImageUsage::SAMPLED | ImageUsage::INPUT_ATTACHMENT) .is_empty() @@ -969,13 +1030,7 @@ impl RawImage { ImageLayout::DepthStencilAttachmentOptimal } else { ImageLayout::General - }; - - Ok(Image::from_raw( - self, - ImageMemory::Normal(allocations), - layout, - )) + } } /// Assume that this image already has memory backing it. @@ -1826,6 +1881,16 @@ impl ImageCreateInfo { ..Default::default() })); } + + if flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) { + return Err(Box::new(ValidationError { + problem: "`image_type` is `ImageType::Dim1d`, but `flags` contains \ + `ImageCreateFlags::SPARSE_RESIDENCY`" + .into(), + vuids: &["VUID-VkImageCreateInfo-imageType-00970"], + ..Default::default() + })); + } } ImageType::Dim2d => { if extent[2] != 1 { @@ -1836,6 +1901,21 @@ impl ImageCreateInfo { ..Default::default() })); } + + if flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) + && !device.enabled_features().sparse_residency_image2_d + { + return Err(Box::new(ValidationError { + problem: "`image_type` is `ImageType::Dim2d`, and `flags` contains \ + `ImageCreateFlags::SPARSE_RESIDENCY`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency_image2_d"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00971"], + ..Default::default() + })); + } } ImageType::Dim3d => { if array_layers != 1 { @@ -1846,6 +1926,21 @@ impl ImageCreateInfo { ..Default::default() })); } + + if flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) + && !device.enabled_features().sparse_residency_image3_d + { + return Err(Box::new(ValidationError { + problem: "`image_type` is `ImageType::Dim3d`, and `flags` contains \ + `ImageCreateFlags::SPARSE_RESIDENCY`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency_image3_d"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00972"], + ..Default::default() + })); + } } } @@ -2287,6 +2382,113 @@ impl ImageCreateInfo { /* Check flags requirements */ + if flags.intersects(ImageCreateFlags::SPARSE_BINDING) { + if !device.enabled_features().sparse_binding { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `ImageCreateFlags::SPARSE_BINDING`".into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::DeviceFeature( + "sparse_binding", + )])]), + vuids: &["VUID-VkImageCreateInfo-flags-00969"], + })); + } + + if usage.intersects(ImageUsage::TRANSIENT_ATTACHMENT) { + return Err(Box::new(ValidationError { + problem: "`flags` contains `ImageCreateFlags::SPARSE_BINDING`, but \ + `usage` contains `ImageUsage::TRANSIENT_ATTACHMENT`" + .into(), + vuids: &["VUID-VkImageCreateInfo-None-01925"], + ..Default::default() + })); + } + } + + if flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) { + if !flags.intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `ImageCreateFlags::SPARSE_RESIDENCY`, but does not also \ + contain `ImageCreateFlags::SPARSE_BINDING`" + .into(), + vuids: &["VUID-VkImageCreateInfo-flags-00987"], + ..Default::default() + })); + } + + if tiling == ImageTiling::Linear { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `ImageCreateFlags::SPARSE_RESIDENCY`, but `tiling` is \ + `ImageTiling::Linear`" + .into(), + vuids: &["VUID-VkImageCreateInfo-tiling-04121"], + ..Default::default() + })); + } + + match samples { + SampleCount::Sample2 => { + if !device.enabled_features().sparse_residency2_samples { + return Err(Box::new(ValidationError { + problem: "`flags` contains `ImageCreateFlags::SPARSE_RESIDENCY`, and \ + `samples` is `SampleCount::Sample2`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency2_samples"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00973"], + ..Default::default() + })); + } + } + SampleCount::Sample4 => { + if !device.enabled_features().sparse_residency4_samples { + return Err(Box::new(ValidationError { + problem: "`flags` contains `ImageCreateFlags::SPARSE_RESIDENCY`, and \ + `samples` is `SampleCount::Sample4`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency4_samples"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00974"], + ..Default::default() + })); + } + } + SampleCount::Sample8 => { + if !device.enabled_features().sparse_residency8_samples { + return Err(Box::new(ValidationError { + problem: "`flags` contains `ImageCreateFlags::SPARSE_RESIDENCY`, and \ + `samples` is `SampleCount::Sample8`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency8_samples"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00975"], + ..Default::default() + })); + } + } + SampleCount::Sample16 => { + if !device.enabled_features().sparse_residency16_samples { + return Err(Box::new(ValidationError { + problem: "`flags` contains `ImageCreateFlags::SPARSE_RESIDENCY`, and \ + `samples` is `SampleCount::Sample16`" + .into(), + requires_one_of: RequiresOneOf(&[RequiresAllOf(&[ + Requires::DeviceFeature("sparse_residency16_samples"), + ])]), + vuids: &["VUID-VkImageCreateInfo-imageType-00976"], + ..Default::default() + })); + } + } + SampleCount::Sample1 | SampleCount::Sample32 | SampleCount::Sample64 => (), + } + } + if flags.intersects(ImageCreateFlags::CUBE_COMPATIBLE) { if image_type != ImageType::Dim2d { return Err(Box::new(ValidationError { @@ -2320,6 +2522,17 @@ impl ImageCreateInfo { } if flags.intersects(ImageCreateFlags::DIM2D_ARRAY_COMPATIBLE) { + if flags.intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "flags".into(), + problem: "contains `ImageCreateFlags::DIM2D_ARRAY_COMPATIBLE`, but \ + also contains `ImageCreateFlags::SPARSE_BINDING`" + .into(), + vuids: &["VUID-VkImageCreateInfo-flags-09403"], + ..Default::default() + })); + } + if image_type != ImageType::Dim3d { return Err(Box::new(ValidationError { problem: "`flags` contains `ImageCreateFlags::DIM2D_ARRAY_COMPATIBLE`, but \ diff --git a/vulkano/src/image/view.rs b/vulkano/src/image/view.rs index c079ba0ee4..617147f702 100644 --- a/vulkano/src/image/view.rs +++ b/vulkano/src/image/view.rs @@ -127,6 +127,17 @@ impl ImageView { if matches!(view_type, ImageViewType::Dim2d | ImageViewType::Dim2dArray) && image_type == ImageType::Dim3d { + if image.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + problem: "`create_info.view_type` is `ImageViewType::Dim2d` or \ + `ImageViewType::Dim2d`, and `image.image_type()` is `ImageType::Dim3d`, \ + but `image.flags()` contains `ImageCreateFlags::SPARSE_BINDING`" + .into(), + vuids: &["VUID-VkImageViewCreateInfo-image-04971"], + ..Default::default() + })); + } + match view_type { ImageViewType::Dim2d => { if !image diff --git a/vulkano/src/memory/device_memory.rs b/vulkano/src/memory/device_memory.rs index b8be8a900a..caf182ec53 100644 --- a/vulkano/src/memory/device_memory.rs +++ b/vulkano/src/memory/device_memory.rs @@ -1,6 +1,8 @@ use super::{DedicatedAllocation, DedicatedTo, DeviceAlignment}; use crate::{ + buffer::BufferCreateFlags, device::{Device, DeviceOwned}, + image::ImageCreateFlags, instance::InstanceOwnedDebugWrapper, macros::{impl_id_counter, vulkan_bitflags, vulkan_bitflags_enum}, memory::{is_aligned, MemoryPropertyFlags}, @@ -948,6 +950,15 @@ impl<'d> MemoryAllocateInfo<'d> { ..Default::default() })); } + + if buffer.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "dedicated_allocation.flags()".into(), + problem: "contains `BufferCreateFlags::SPARSE_BINDING`".into(), + vuids: &["VUID-VkMemoryDedicatedAllocateInfo-buffer-01436"], + ..Default::default() + })); + } } DedicatedAllocation::Image(image) => { // VUID-VkMemoryDedicatedAllocateInfo-commonparent @@ -964,6 +975,15 @@ impl<'d> MemoryAllocateInfo<'d> { ..Default::default() })); } + + if image.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "dedicated_allocation.flags()".into(), + problem: "contains `ImageCreateFlags::SPARSE_BINDING`".into(), + vuids: &["VUID-VkMemoryDedicatedAllocateInfo-image-01434"], + ..Default::default() + })); + } } } } From a630175a9badaf078f161f2e4f0b9a848bbb2d5b Mon Sep 17 00:00:00 2001 From: Rua Date: Fri, 29 Nov 2024 17:12:44 +0100 Subject: [PATCH 2/8] Add validated `bind_sparse` command --- vulkano/src/device/queue.rs | 85 +- vulkano/src/image/mod.rs | 4 +- vulkano/src/memory/mod.rs | 413 +----- vulkano/src/memory/sparse.rs | 1528 +++++++++++++++++++++++ vulkano/src/sync/future/fence_signal.rs | 6 +- vulkano/src/sync/future/mod.rs | 4 +- 6 files changed, 1621 insertions(+), 419 deletions(-) create mode 100644 vulkano/src/memory/sparse.rs diff --git a/vulkano/src/device/queue.rs b/vulkano/src/device/queue.rs index 37d17cb072..b748a314ad 100644 --- a/vulkano/src/device/queue.rs +++ b/vulkano/src/device/queue.rs @@ -3,7 +3,7 @@ use crate::{ command_buffer::{CommandBufferSubmitInfo, SemaphoreSubmitInfo, SubmitInfo}, instance::{debug::DebugUtilsLabel, InstanceOwnedDebugWrapper}, macros::vulkan_bitflags, - memory::BindSparseInfo, + memory::sparse::BindSparseInfo, swapchain::{PresentInfo, SwapchainPresentInfo}, sync::{fence::Fence, PipelineStages}, Requires, RequiresAllOf, RequiresOneOf, Validated, ValidationError, Version, VulkanError, @@ -112,6 +112,10 @@ impl Queue { _state: self.state.lock(), }) } + + fn queue_family_properties(&self) -> &QueueFamilyProperties { + &self.device().physical_device().queue_family_properties()[self.queue_family_index as usize] + } } impl Drop for Queue { @@ -216,6 +220,85 @@ impl QueueGuard<'_> { .map_err(VulkanError::from) } + /// Bind or unbind memory to resources with sparse memory. + /// + /// # Safety + /// + /// For every semaphore in the `wait_semaphores` elements of every `bind_infos` element: + /// - The semaphore must be kept alive while the command is being executed. + /// - The semaphore must be already in the signaled state, or there must be a previously + /// submitted operation that will signal it. + /// - When the wait operation is executed, no other queue must be waiting on the same + /// semaphore. + /// + /// For every element in the `buffer_binds`, `image_opaque_binds` and `image_binds` + /// elements of every `bind_infos` element: + /// - The buffers and images must be kept alive while the command is being executed. + /// - The memory allocations must be kept alive while they are bound to a buffer or image. + /// - Access to the affected regions of each buffer or image must be synchronized. + /// + /// For every semaphore in the `signal_semaphores` elements of every `bind_infos` element: + /// - The semaphore must be kept alive while the command is being executed. + /// - When the signal operation is executed, the semaphore must be in the unsignaled state. + /// + /// If `fence` is `Some`: + /// - The fence must be kept alive while the command is being executed. + /// - The fence must be unsignaled and must not be associated with any other command that is + /// still executing. + #[inline] + pub unsafe fn bind_sparse( + &mut self, + bind_infos: &[BindSparseInfo], + fence: Option<&Arc>, + ) -> Result<(), Validated> { + self.validate_bind_sparse(bind_infos, fence)?; + + Ok(self.bind_sparse_unchecked(bind_infos, fence)?) + } + + fn validate_bind_sparse( + &self, + bind_infos: &[BindSparseInfo], + fence: Option<&Arc>, + ) -> Result<(), Box> { + if !self + .queue + .queue_family_properties() + .queue_flags + .intersects(QueueFlags::SPARSE_BINDING) + { + return Err(Box::new(ValidationError { + problem: "the queue family of this queue does not support \ + sparse binding operations" + .into(), + vuids: &["VUID-vkQueueBindSparse-queuetype"], + ..Default::default() + })); + } + + let device = self.queue.device(); + + if let Some(fence) = fence { + // VUID-vkQueueBindSparse-commonparent + assert_eq!(device, fence.device()); + } + + for (index, bind_sparse_info) in bind_infos.iter().enumerate() { + bind_sparse_info + .validate(device) + .map_err(|err| err.add_context(format!("bind_infos[{}]", index)))?; + } + + // unsafe + // VUID-vkQueueBindSparse-fence-01113 + // VUID-vkQueueBindSparse-fence-01114 + // VUID-vkQueueBindSparse-pSignalSemaphores-01115 + // VUID-vkQueueBindSparse-pWaitSemaphores-01116 + // VUID-vkQueueBindSparse-pWaitSemaphores-03245 + + Ok(()) + } + #[cfg_attr(not(feature = "document_unchecked"), doc(hidden))] pub unsafe fn bind_sparse_unchecked( &mut self, diff --git a/vulkano/src/image/mod.rs b/vulkano/src/image/mod.rs index 4734f0b7d3..992fffcac6 100644 --- a/vulkano/src/image/mod.rs +++ b/vulkano/src/image/mod.rs @@ -2586,8 +2586,8 @@ pub struct SparseImageMemoryRequirements { /// The memory offset that must be used to bind the mip tail region. pub image_mip_tail_offset: DeviceSize, - /// If `format_properties.flags.single_miptail` is not set, specifies the stride between - /// the mip tail regions of each array layer. + /// If `format_properties.flags` does not contain `SparseImageFormatFlags::SINGLE_MIPTAIL`, + /// then this specifies the stride between the mip tail regions of each array layer. pub image_mip_tail_stride: Option, } diff --git a/vulkano/src/memory/mod.rs b/vulkano/src/memory/mod.rs index 00517a61ab..1c1c3759e1 100644 --- a/vulkano/src/memory/mod.rs +++ b/vulkano/src/memory/mod.rs @@ -94,14 +94,13 @@ use self::allocator::{ }; pub use self::{alignment::*, device_memory::*}; use crate::{ - buffer::{sys::RawBuffer, Subbuffer}, + buffer::sys::RawBuffer, device::{Device, DeviceOwned, DeviceOwnedDebugWrapper}, - image::{sys::RawImage, Image, ImageAspects}, + image::sys::RawImage, macros::vulkan_bitflags, - sync::{semaphore::Semaphore, HostAccessError}, + sync::HostAccessError, DeviceSize, Validated, ValidationError, Version, VulkanError, VulkanObject, }; -use smallvec::SmallVec; use std::{ cmp, mem::ManuallyDrop, @@ -114,6 +113,7 @@ use std::{ mod alignment; pub mod allocator; mod device_memory; +pub mod sparse; /// Memory that can be bound to resources. /// @@ -930,411 +930,6 @@ impl MemoryFdProperties { } } -/// Parameters to execute sparse bind operations on a queue. -#[derive(Clone, Debug)] -pub struct BindSparseInfo { - /// The semaphores to wait for before beginning the execution of this batch of - /// sparse bind operations. - /// - /// The default value is empty. - pub wait_semaphores: Vec>, - - /// The bind operations to perform for buffers. - /// - /// The default value is empty. - pub buffer_binds: Vec<(Subbuffer<[u8]>, Vec)>, - - /// The bind operations to perform for images with an opaque memory layout. - /// - /// This should be used for mip tail regions, the metadata aspect, and for the normal regions - /// of images that do not have the `sparse_residency` flag set. - /// - /// The default value is empty. - pub image_opaque_binds: Vec<(Arc, Vec)>, - - /// The bind operations to perform for images with a known memory layout. - /// - /// This type of sparse bind can only be used for images that have the `sparse_residency` - /// flag set. - /// Only the normal texel regions can be bound this way, not the mip tail regions or metadata - /// aspect. - /// - /// The default value is empty. - pub image_binds: Vec<(Arc, Vec)>, - - /// The semaphores to signal after the execution of this batch of sparse bind operations - /// has completed. - /// - /// The default value is empty. - pub signal_semaphores: Vec>, - - pub _ne: crate::NonExhaustive, -} - -impl Default for BindSparseInfo { - #[inline] - fn default() -> Self { - Self { - wait_semaphores: Vec::new(), - buffer_binds: Vec::new(), - image_opaque_binds: Vec::new(), - image_binds: Vec::new(), - signal_semaphores: Vec::new(), - _ne: crate::NonExhaustive(()), - } - } -} - -impl BindSparseInfo { - pub(crate) fn to_vk<'a>( - &self, - fields1_vk: &'a BindSparseInfoFields1Vk<'_>, - ) -> ash::vk::BindSparseInfo<'a> { - let BindSparseInfoFields1Vk { - wait_semaphores_vk, - buffer_bind_infos_vk, - image_opaque_bind_infos_vk, - image_bind_infos_vk, - signal_semaphores_vk, - } = fields1_vk; - - ash::vk::BindSparseInfo::default() - .wait_semaphores(wait_semaphores_vk) - .buffer_binds(buffer_bind_infos_vk) - .image_opaque_binds(image_opaque_bind_infos_vk) - .image_binds(image_bind_infos_vk) - .signal_semaphores(signal_semaphores_vk) - } - - pub(crate) fn to_vk_fields1<'a>( - &self, - fields2_vk: &'a BindSparseInfoFields2Vk, - ) -> BindSparseInfoFields1Vk<'a> { - let &BindSparseInfo { - ref wait_semaphores, - ref buffer_binds, - ref image_opaque_binds, - ref image_binds, - ref signal_semaphores, - _ne: _, - } = self; - let BindSparseInfoFields2Vk { - buffer_binds_vk, - image_opaque_binds_vk, - image_binds_vk, - } = fields2_vk; - - let wait_semaphores_vk = wait_semaphores - .iter() - .map(|semaphore| semaphore.handle()) - .collect(); - - let buffer_bind_infos_vk = buffer_binds - .iter() - .zip(buffer_binds_vk) - .map(|((buffer, _), buffer_binds_vk)| { - ash::vk::SparseBufferMemoryBindInfo::default() - .buffer(buffer.buffer().handle()) - .binds(buffer_binds_vk) - }) - .collect(); - - let image_opaque_bind_infos_vk = image_opaque_binds - .iter() - .zip(image_opaque_binds_vk) - .map(|((image, _), image_opaque_binds_vk)| { - ash::vk::SparseImageOpaqueMemoryBindInfo::default() - .image(image.handle()) - .binds(image_opaque_binds_vk) - }) - .collect(); - - let image_bind_infos_vk = image_binds - .iter() - .zip(image_binds_vk) - .map(|((image, _), image_binds_vk)| { - ash::vk::SparseImageMemoryBindInfo::default() - .image(image.handle()) - .binds(image_binds_vk) - }) - .collect(); - - let signal_semaphores_vk = signal_semaphores - .iter() - .map(|semaphore| semaphore.handle()) - .collect(); - - BindSparseInfoFields1Vk { - wait_semaphores_vk, - buffer_bind_infos_vk, - image_opaque_bind_infos_vk, - image_bind_infos_vk, - signal_semaphores_vk, - } - } - - pub(crate) fn to_vk_fields2(&self) -> BindSparseInfoFields2Vk { - let &Self { - wait_semaphores: _, - ref buffer_binds, - ref image_opaque_binds, - ref image_binds, - signal_semaphores: _, - _ne: _, - } = self; - - let buffer_binds_vk = buffer_binds - .iter() - .map(|(_, memory_binds)| { - memory_binds - .iter() - .map(SparseBufferMemoryBind::to_vk) - .collect() - }) - .collect(); - - let image_opaque_binds_vk = image_opaque_binds - .iter() - .map(|(_, memory_binds)| { - memory_binds - .iter() - .map(SparseImageOpaqueMemoryBind::to_vk) - .collect() - }) - .collect(); - - let image_binds_vk = image_binds - .iter() - .map(|(_, memory_binds)| { - memory_binds - .iter() - .map(SparseImageMemoryBind::to_vk) - .collect() - }) - .collect(); - - BindSparseInfoFields2Vk { - buffer_binds_vk, - image_opaque_binds_vk, - image_binds_vk, - } - } -} - -pub(crate) struct BindSparseInfoFields1Vk<'a> { - pub(crate) wait_semaphores_vk: SmallVec<[ash::vk::Semaphore; 4]>, - pub(crate) buffer_bind_infos_vk: SmallVec<[ash::vk::SparseBufferMemoryBindInfo<'a>; 4]>, - pub(crate) image_opaque_bind_infos_vk: - SmallVec<[ash::vk::SparseImageOpaqueMemoryBindInfo<'a>; 4]>, - pub(crate) image_bind_infos_vk: SmallVec<[ash::vk::SparseImageMemoryBindInfo<'a>; 4]>, - pub(crate) signal_semaphores_vk: SmallVec<[ash::vk::Semaphore; 4]>, -} - -pub(crate) struct BindSparseInfoFields2Vk { - pub(crate) buffer_binds_vk: SmallVec<[SmallVec<[ash::vk::SparseMemoryBind; 4]>; 4]>, - pub(crate) image_opaque_binds_vk: SmallVec<[SmallVec<[ash::vk::SparseMemoryBind; 4]>; 4]>, - pub(crate) image_binds_vk: SmallVec<[SmallVec<[ash::vk::SparseImageMemoryBind; 4]>; 4]>, -} - -/// Parameters for a single sparse bind operation on a buffer. -#[derive(Clone, Debug, Default)] -pub struct SparseBufferMemoryBind { - /// The offset in bytes from the start of the buffer's memory, where memory is to be (un)bound. - /// - /// The default value is `0`. - pub offset: DeviceSize, - - /// The size in bytes of the memory to be (un)bound. - /// - /// The default value is `0`, which must be overridden. - pub size: DeviceSize, - - /// If `Some`, specifies the memory and an offset into that memory that is to be bound. - /// The provided memory must match the buffer's memory requirements. - /// - /// If `None`, specifies that existing memory at the specified location is to be unbound. - /// - /// The default value is `None`. - pub memory: Option<(Arc, DeviceSize)>, -} - -impl SparseBufferMemoryBind { - pub(crate) fn to_vk(&self) -> ash::vk::SparseMemoryBind { - let &Self { - offset, - size, - ref memory, - } = self; - - let (memory, memory_offset) = memory - .as_ref() - .map_or_else(Default::default, |(memory, memory_offset)| { - (memory.handle(), *memory_offset) - }); - - ash::vk::SparseMemoryBind { - resource_offset: offset, - size, - memory, - memory_offset, - flags: ash::vk::SparseMemoryBindFlags::empty(), - } - } -} - -/// Parameters for a single sparse bind operation on parts of an image with an opaque memory -/// layout. -/// -/// This type of sparse bind should be used for mip tail regions, the metadata aspect, and for the -/// normal regions of images that do not have the `sparse_residency` flag set. -#[derive(Clone, Debug, Default)] -pub struct SparseImageOpaqueMemoryBind { - /// The offset in bytes from the start of the image's memory, where memory is to be (un)bound. - /// - /// The default value is `0`. - pub offset: DeviceSize, - - /// The size in bytes of the memory to be (un)bound. - /// - /// The default value is `0`, which must be overridden. - pub size: DeviceSize, - - /// If `Some`, specifies the memory and an offset into that memory that is to be bound. - /// The provided memory must match the image's memory requirements. - /// - /// If `None`, specifies that existing memory at the specified location is to be unbound. - /// - /// The default value is `None`. - pub memory: Option<(Arc, DeviceSize)>, - - /// Sets whether the binding should apply to the metadata aspect of the image, or to the - /// normal texel data. - /// - /// The default value is `false`. - pub metadata: bool, -} - -impl SparseImageOpaqueMemoryBind { - pub(crate) fn to_vk(&self) -> ash::vk::SparseMemoryBind { - let &Self { - offset, - size, - ref memory, - metadata, - } = self; - - let (memory, memory_offset) = memory - .as_ref() - .map_or_else(Default::default, |(memory, memory_offset)| { - (memory.handle(), *memory_offset) - }); - - ash::vk::SparseMemoryBind { - resource_offset: offset, - size, - memory, - memory_offset, - flags: if metadata { - ash::vk::SparseMemoryBindFlags::METADATA - } else { - ash::vk::SparseMemoryBindFlags::empty() - }, - } - } -} - -/// Parameters for a single sparse bind operation on parts of an image with a known memory layout. -/// -/// This type of sparse bind can only be used for images that have the `sparse_residency` flag set. -/// Only the normal texel regions can be bound this way, not the mip tail regions or metadata -/// aspect. -#[derive(Clone, Debug, Default)] -pub struct SparseImageMemoryBind { - /// The aspects of the image where memory is to be (un)bound. - /// - /// The default value is `ImageAspects::empty()`, which must be overridden. - pub aspects: ImageAspects, - - /// The mip level of the image where memory is to be (un)bound. - /// - /// The default value is `0`. - pub mip_level: u32, - - /// The array layer of the image where memory is to be (un)bound. - /// - /// The default value is `0`. - pub array_layer: u32, - - /// The offset in texels (or for compressed images, texel blocks) from the origin of the image, - /// where memory is to be (un)bound. - /// - /// This must be a multiple of the - /// [`SparseImageFormatProperties::image_granularity`](crate::image::SparseImageFormatProperties::image_granularity) - /// value of the image. - /// - /// The default value is `[0; 3]`. - pub offset: [u32; 3], - - /// The extent in texels (or for compressed images, texel blocks) of the image where - /// memory is to be (un)bound. - /// - /// This must be a multiple of the - /// [`SparseImageFormatProperties::image_granularity`](crate::image::SparseImageFormatProperties::image_granularity) - /// value of the image, or `offset + extent` for that dimension must equal the image's total - /// extent. - /// - /// The default value is `[0; 3]`, which must be overridden. - pub extent: [u32; 3], - - /// If `Some`, specifies the memory and an offset into that memory that is to be bound. - /// The provided memory must match the image's memory requirements. - /// - /// If `None`, specifies that existing memory at the specified location is to be unbound. - /// - /// The default value is `None`. - pub memory: Option<(Arc, DeviceSize)>, -} - -impl SparseImageMemoryBind { - pub(crate) fn to_vk(&self) -> ash::vk::SparseImageMemoryBind { - let &Self { - aspects, - mip_level, - array_layer, - offset, - extent, - ref memory, - } = self; - - let (memory, memory_offset) = memory - .as_ref() - .map_or_else(Default::default, |(memory, memory_offset)| { - (memory.handle(), *memory_offset) - }); - - ash::vk::SparseImageMemoryBind { - subresource: ash::vk::ImageSubresource { - aspect_mask: aspects.into(), - mip_level, - array_layer, - }, - offset: ash::vk::Offset3D { - x: offset[0] as i32, - y: offset[1] as i32, - z: offset[2] as i32, - }, - extent: ash::vk::Extent3D { - width: extent[0], - height: extent[1], - depth: extent[2], - }, - memory, - memory_offset, - flags: ash::vk::SparseMemoryBindFlags::empty(), - } - } -} - #[inline(always)] pub(crate) fn is_aligned(offset: DeviceSize, alignment: DeviceAlignment) -> bool { offset & (alignment.as_devicesize() - 1) == 0 diff --git a/vulkano/src/memory/sparse.rs b/vulkano/src/memory/sparse.rs new file mode 100644 index 0000000000..3746737a1f --- /dev/null +++ b/vulkano/src/memory/sparse.rs @@ -0,0 +1,1528 @@ +use super::{DeviceMemory, MemoryPropertyFlags}; +use crate::{ + buffer::{Buffer, BufferCreateFlags}, + device::{Device, DeviceOwned}, + image::{ + mip_level_extent, Image, ImageAspects, ImageCreateFlags, ImageMemory, + SparseImageFormatProperties, SparseImageMemoryRequirements, + }, + memory::{is_aligned, MemoryRequirements}, + sync::semaphore::Semaphore, + DeviceSize, ValidationError, VulkanObject as _, +}; +use smallvec::SmallVec; +use std::sync::Arc; + +/// Parameters to execute sparse bind operations on a queue. +#[derive(Clone, Debug)] +pub struct BindSparseInfo { + /// The semaphores to wait for before beginning the execution of this batch of + /// sparse bind operations. + /// + /// The default value is empty. + pub wait_semaphores: Vec>, + + /// The bind operations to perform for buffers. + /// + /// The default value is empty. + pub buffer_binds: Vec, + + /// The bind operations to perform for images with an opaque memory layout. + /// + /// This should be used for mip tail regions, the metadata aspect, and for the normal regions + /// of images that do not have the `sparse_residency` flag set. + /// + /// The default value is empty. + pub image_opaque_binds: Vec, + + /// The bind operations to perform for images with a known memory layout. + /// + /// This type of sparse bind can only be used for images that have the `sparse_residency` + /// flag set. + /// Only the normal texel regions can be bound this way, not the mip tail regions or metadata + /// aspect. + /// + /// The default value is empty. + pub image_binds: Vec, + + /// The semaphores to signal after the execution of this batch of sparse bind operations + /// has completed. + /// + /// The default value is empty. + pub signal_semaphores: Vec>, + + pub _ne: crate::NonExhaustive, +} + +impl Default for BindSparseInfo { + #[inline] + fn default() -> Self { + Self { + wait_semaphores: Vec::new(), + buffer_binds: Vec::new(), + image_opaque_binds: Vec::new(), + image_binds: Vec::new(), + signal_semaphores: Vec::new(), + _ne: crate::NonExhaustive(()), + } + } +} + +impl BindSparseInfo { + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + ref wait_semaphores, + ref buffer_binds, + ref image_opaque_binds, + ref image_binds, + ref signal_semaphores, + _ne: _, + } = self; + + for semaphore in wait_semaphores { + assert_eq!(device, semaphore.device().as_ref()); + } + + for (index, buffer_bind_info) in buffer_binds.iter().enumerate() { + buffer_bind_info + .validate(device) + .map_err(|err| err.add_context(format!("buffer_binds[{}]", index)))?; + } + + for (index, image_opaque_bind_info) in image_opaque_binds.iter().enumerate() { + image_opaque_bind_info + .validate(device) + .map_err(|err| err.add_context(format!("image_opaque_binds[{}]", index)))?; + } + + for (index, image_bind_info) in image_binds.iter().enumerate() { + image_bind_info + .validate(device) + .map_err(|err| err.add_context(format!("image_binds[{}]", index)))?; + } + + for semaphore in signal_semaphores { + assert_eq!(device, semaphore.device().as_ref()); + } + + Ok(()) + } + + pub(crate) fn to_vk<'a>( + &self, + fields1_vk: &'a BindSparseInfoFields1Vk<'_>, + ) -> ash::vk::BindSparseInfo<'a> { + let BindSparseInfoFields1Vk { + wait_semaphores_vk, + buffer_bind_infos_vk, + image_opaque_bind_infos_vk, + image_bind_infos_vk, + signal_semaphores_vk, + } = fields1_vk; + + ash::vk::BindSparseInfo::default() + .wait_semaphores(wait_semaphores_vk) + .buffer_binds(buffer_bind_infos_vk) + .image_opaque_binds(image_opaque_bind_infos_vk) + .image_binds(image_bind_infos_vk) + .signal_semaphores(signal_semaphores_vk) + } + + pub(crate) fn to_vk_fields1<'a>( + &self, + fields2_vk: &'a BindSparseInfoFields2Vk, + ) -> BindSparseInfoFields1Vk<'a> { + let &BindSparseInfo { + ref wait_semaphores, + ref buffer_binds, + ref image_opaque_binds, + ref image_binds, + ref signal_semaphores, + _ne: _, + } = self; + let BindSparseInfoFields2Vk { + buffer_bind_infos_fields1_vk, + image_opaque_bind_infos_fields1_vk, + image_bind_infos_fields1_vk, + } = fields2_vk; + + let wait_semaphores_vk = wait_semaphores + .iter() + .map(|semaphore| semaphore.handle()) + .collect(); + + let buffer_bind_infos_vk = buffer_binds + .iter() + .zip(buffer_bind_infos_fields1_vk) + .map(|(buffer_bind_info, fields1_vk)| buffer_bind_info.to_vk(fields1_vk)) + .collect(); + + let image_opaque_bind_infos_vk = image_opaque_binds + .iter() + .zip(image_opaque_bind_infos_fields1_vk) + .map(|(image_opaque_bind_info, fields1_vk)| image_opaque_bind_info.to_vk(fields1_vk)) + .collect(); + + let image_bind_infos_vk = image_binds + .iter() + .zip(image_bind_infos_fields1_vk) + .map(|(image_bind_info, fields1_vk)| image_bind_info.to_vk(fields1_vk)) + .collect(); + + let signal_semaphores_vk = signal_semaphores + .iter() + .map(|semaphore| semaphore.handle()) + .collect(); + + BindSparseInfoFields1Vk { + wait_semaphores_vk, + buffer_bind_infos_vk, + image_opaque_bind_infos_vk, + image_bind_infos_vk, + signal_semaphores_vk, + } + } + + pub(crate) fn to_vk_fields2(&self) -> BindSparseInfoFields2Vk { + let &Self { + wait_semaphores: _, + ref buffer_binds, + ref image_opaque_binds, + ref image_binds, + signal_semaphores: _, + _ne: _, + } = self; + + let buffer_bind_infos_fields1_vk = buffer_binds + .iter() + .map(SparseBufferMemoryBindInfo::to_vk_fields1) + .collect(); + + let image_opaque_bind_infos_fields1_vk = image_opaque_binds + .iter() + .map(SparseImageOpaqueMemoryBindInfo::to_vk_fields1) + .collect(); + + let image_bind_infos_fields1_vk = image_binds + .iter() + .map(SparseImageMemoryBindInfo::to_vk_fields1) + .collect(); + + BindSparseInfoFields2Vk { + buffer_bind_infos_fields1_vk, + image_opaque_bind_infos_fields1_vk, + image_bind_infos_fields1_vk, + } + } +} + +pub(crate) struct BindSparseInfoFields1Vk<'a> { + pub(crate) wait_semaphores_vk: SmallVec<[ash::vk::Semaphore; 4]>, + pub(crate) buffer_bind_infos_vk: SmallVec<[ash::vk::SparseBufferMemoryBindInfo<'a>; 4]>, + pub(crate) image_opaque_bind_infos_vk: + SmallVec<[ash::vk::SparseImageOpaqueMemoryBindInfo<'a>; 4]>, + pub(crate) image_bind_infos_vk: SmallVec<[ash::vk::SparseImageMemoryBindInfo<'a>; 4]>, + pub(crate) signal_semaphores_vk: SmallVec<[ash::vk::Semaphore; 4]>, +} + +pub(crate) struct BindSparseInfoFields2Vk { + pub(crate) buffer_bind_infos_fields1_vk: SmallVec<[SparseBufferMemoryBindInfoFields1Vk; 4]>, + pub(crate) image_opaque_bind_infos_fields1_vk: + SmallVec<[SparseImageOpaqueMemoryBindInfoFields1Vk; 4]>, + pub(crate) image_bind_infos_fields1_vk: SmallVec<[SparseImageMemoryBindInfoFields1Vk; 4]>, +} + +/// Parameters for sparse bind operations on a buffer. +#[derive(Clone, Debug)] +pub struct SparseBufferMemoryBindInfo { + /// The buffer to perform the binding operations on. + /// + /// There is no default value. + pub buffer: Arc, + + /// The bind operations to perform. + /// + /// The default value is empty. + pub binds: Vec, +} + +impl SparseBufferMemoryBindInfo { + /// Returns a `SparseBufferMemoryBindInfo` with the specified `buffer`. + #[inline] + pub fn new(buffer: Arc) -> Self { + Self { + buffer, + binds: Vec::new(), + } + } + + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + ref buffer, + ref binds, + } = self; + + assert_eq!(device, buffer.device().as_ref()); + assert!(!binds.is_empty()); + + if !buffer.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "buffer.flags()".into(), + problem: "does not contain `BufferCreateFlags::SPARSE_BINDING`".into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2253 + ..Default::default() + })); + } + + let &MemoryRequirements { + layout, + memory_type_bits, + prefers_dedicated_allocation: _, + requires_dedicated_allocation: _, + } = buffer.memory_requirements(); + let external_memory_handle_types = buffer.external_memory_handle_types(); + + for (index, bind) in binds.iter().enumerate() { + bind.validate(device) + .map_err(|err| err.add_context(format!("binds[{}]", index)))?; + + let &SparseBufferMemoryBind { + offset, + size, + ref memory, + } = bind; + + if offset >= layout.size() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset` is not less than \ + `buffer.memory_requirements().layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-resourceOffset-01099"], + ..Default::default() + })); + } + + if size > layout.size() - offset { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].offset + binds[{0}].size` is greater than \ + `buffer.memory_requirements().layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-size-01100"], + ..Default::default() + })); + } + + if !is_aligned(offset, layout.alignment()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset` is not aligned according to \ + `buffer.memory_requirements().layout.alignment()`", + index + ) + .into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2255, + ..Default::default() + })); + } + + if !(size == layout.size() - offset || is_aligned(size, layout.alignment())) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].offset + binds[{0}].size` is not equal to \ + `buffer.memory_requirements().layout.size()`, but is not aligned \ + according to `buffer.memory_requirements().layout.alignment()`", + index + ) + .into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2255, + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + if memory.allocation_size() < layout.size() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.allocation_size()` is less than \ + `buffer.memory_requirements().layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if !is_aligned(memory_offset, layout.alignment()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.1` is not aligned according to \ + `buffer.memory_requirements().layout.alignment()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if memory_type_bits & (1 << memory.memory_type_index()) == 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.memory_type_index()` is not a bit set in \ + `buffer.memory_requirements().memory_type_bits`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if !memory.export_handle_types().is_empty() { + if !external_memory_handle_types.intersects(memory.export_handle_types()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.export_handle_types()` is not empty, but \ + it does not share at least one memory type with \ + `buffer.external_memory_handle_types()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-02730"], + ..Default::default() + })); + } + } + + if let Some(handle_type) = memory.imported_handle_type() { + if !external_memory_handle_types.intersects(handle_type.into()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0` is imported, but \ + `buffer.external_memory_handle_types()` \ + does not contain the imported handle type", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-02731"], + ..Default::default() + })); + } + } + } + } + + Ok(()) + } + + pub(crate) fn to_vk<'a>( + &self, + fields1_vk: &'a SparseBufferMemoryBindInfoFields1Vk, + ) -> ash::vk::SparseBufferMemoryBindInfo<'a> { + let Self { buffer, binds: _ } = self; + let SparseBufferMemoryBindInfoFields1Vk { binds_vk } = fields1_vk; + + ash::vk::SparseBufferMemoryBindInfo::default() + .buffer(buffer.handle()) + .binds(binds_vk) + } + + pub(crate) fn to_vk_fields1(&self) -> SparseBufferMemoryBindInfoFields1Vk { + let Self { buffer: _, binds } = self; + + let binds_vk = binds.iter().map(SparseBufferMemoryBind::to_vk).collect(); + + SparseBufferMemoryBindInfoFields1Vk { binds_vk } + } +} + +pub(crate) struct SparseBufferMemoryBindInfoFields1Vk { + pub(crate) binds_vk: SmallVec<[ash::vk::SparseMemoryBind; 4]>, +} + +/// Parameters for a single sparse bind operation on a buffer. +#[derive(Clone, Debug, Default)] +pub struct SparseBufferMemoryBind { + /// The offset in bytes from the start of the buffer's memory, where memory is to be (un)bound. + /// + /// The default value is `0`. + pub offset: DeviceSize, + + /// The size in bytes of the memory to be (un)bound. + /// + /// The default value is `0`, which must be overridden. + pub size: DeviceSize, + + /// If `Some`, specifies the memory and an offset into that memory that is to be bound. + /// The provided memory must match the buffer's memory requirements. + /// + /// If `None`, specifies that existing memory at the specified location is to be unbound. + /// + /// The default value is `None`. + pub memory: Option<(Arc, DeviceSize)>, +} + +impl SparseBufferMemoryBind { + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + offset: _, + size, + ref memory, + } = self; + + if size == 0 { + return Err(Box::new(ValidationError { + context: "size".into(), + problem: "is zero".into(), + vuids: &["VUID-VkSparseMemoryBind-size-01098"], + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + let memory_type = &device.physical_device().memory_properties().memory_types + [memory.memory_type_index() as usize]; + + if memory_type + .property_flags + .intersects(MemoryPropertyFlags::LAZILY_ALLOCATED) + { + return Err(Box::new(ValidationError { + problem: "`memory.0.memory_type_index()` refers to a memory type whose \ + `property_flags` contains `MemoryPropertyFlags::LAZILY_ALLOCATED`" + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01097"], + ..Default::default() + })); + } + + if memory_offset >= memory.allocation_size() { + return Err(Box::new(ValidationError { + problem: "`memory.1` is not less than `memory.0.allocation_size()`".into(), + vuids: &["VUID-VkSparseMemoryBind-memoryOffset-01101"], + ..Default::default() + })); + } + + if size > memory.allocation_size() - memory_offset { + return Err(Box::new(ValidationError { + problem: "`size` is greater than `memory.0.allocation_size()` minus \ + `memory.1`" + .into(), + vuids: &["VUID-VkSparseMemoryBind-size-01102"], + ..Default::default() + })); + } + } + + Ok(()) + } + + pub(crate) fn to_vk(&self) -> ash::vk::SparseMemoryBind { + let &Self { + offset, + size, + ref memory, + } = self; + + let (memory, memory_offset) = memory + .as_ref() + .map_or_else(Default::default, |(memory, memory_offset)| { + (memory.handle(), *memory_offset) + }); + + ash::vk::SparseMemoryBind { + resource_offset: offset, + size, + memory, + memory_offset, + flags: ash::vk::SparseMemoryBindFlags::empty(), + } + } +} + +/// Parameters for sparse bind operations on parts of an image with an opaque memory layout. +/// +/// This type of sparse bind should be used for mip tail regions, the metadata aspect, and for the +/// normal regions of images that do not have the `sparse_residency` flag set. +#[derive(Clone, Debug)] +pub struct SparseImageOpaqueMemoryBindInfo { + /// The image to perform the binding operations on. + /// + /// There is no default value. + pub image: Arc, + + /// The bind operations to perform. + /// + /// The default value is empty. + pub binds: Vec, +} + +impl SparseImageOpaqueMemoryBindInfo { + /// Returns a `SparseImageOpaqueMemoryBindInfo` with the specified `image`. + #[inline] + pub fn new(image: Arc) -> Self { + Self { + image, + binds: Vec::new(), + } + } + + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + ref image, + ref binds, + } = self; + + assert_eq!(device, image.device().as_ref()); + assert!(!binds.is_empty()); + + if !image.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "image.flags()".into(), + problem: "does not contain `ImageCreateFlags::SPARSE_BINDING`".into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2253 + ..Default::default() + })); + } + + let &MemoryRequirements { + layout, + memory_type_bits, + prefers_dedicated_allocation: _, + requires_dedicated_allocation: _, + } = &image.memory_requirements()[0]; // TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2259 + let metadata_memory_requirements = match image.memory() { + ImageMemory::Sparse(sparse_memory_requirements) => { + sparse_memory_requirements.iter().find(|reqs| { + reqs.format_properties + .aspects + .intersects(ImageAspects::METADATA) + }) + } + _ => unreachable!(), + }; + let external_memory_handle_types = image.external_memory_handle_types(); + + for (index, bind) in binds.iter().enumerate() { + bind.validate(device) + .map_err(|err| err.add_context(format!("binds[{}]", index)))?; + + let &SparseImageOpaqueMemoryBind { + offset, + size, + ref memory, + metadata, + } = bind; + + if metadata { + // VkSparseMemoryBind spec: + // If flags contains VK_SPARSE_MEMORY_BIND_METADATA_BIT, + // the binding range must be within the mip tail region of the metadata aspect. + // This metadata region is defined by: + // metadataRegion = [base, base + imageMipTailSize) + // base = imageMipTailOffset + imageMipTailStride × n + + let &SparseImageMemoryRequirements { + format_properties: _, + image_mip_tail_first_lod: _, + image_mip_tail_size, + image_mip_tail_offset, + image_mip_tail_stride, + } = metadata_memory_requirements.ok_or_else(|| { + Box::new(ValidationError { + problem: format!( + "`binds[{}].metadata` is `true`, but there is no \ + `SparseImageMemoryRequirements` element in `image.memory()` where \ + `format_properties.aspects` contains `ImageAspects::METADATA`", + index + ) + .into(), + // vuids? + ..Default::default() + }) + })?; + + let offset_from_mip_tail = offset.checked_sub(image_mip_tail_offset); + + if let Some(image_mip_tail_stride) = image_mip_tail_stride { + let (array_layer, offset_from_array_layer) = offset_from_mip_tail + .map(|offset_from_mip_tail| { + ( + offset_from_mip_tail / image_mip_tail_stride, + offset_from_mip_tail % image_mip_tail_stride, + ) + }) + .filter(|&(array_layer, offset_from_array_layer)| { + array_layer < image.array_layers() as DeviceSize + && offset_from_array_layer < image_mip_tail_size + }) + .ok_or_else(|| { + Box::new(ValidationError { + problem: format!( + "`binds[{0}].metadata` is `true`, but `binds[{0}].offset` \ + does not fall within the metadata mip tail binding range \ + for any array layer of `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageOpaqueMemoryBindInfo-pBinds-01103"], + ..Default::default() + }) + })?; + + if size > image_mip_tail_size - offset_from_array_layer { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].metadata` is `true`, and `binds[{0}].offset` \ + falls within the metadata mip tail binding range for \ + array layer {1} of `image`, but \ + `binds[{0}].offset + binds[{0}].size` is greater than the \ + end of that binding range", + index, array_layer + ) + .into(), + vuids: &["VUID-VkSparseImageOpaqueMemoryBindInfo-pBinds-01103"], + ..Default::default() + })); + } + } else { + let offset_from_mip_tail = offset_from_mip_tail + .filter(|&offset_from_mip_tail| offset_from_mip_tail < image_mip_tail_size) + .ok_or_else(|| { + Box::new(ValidationError { + problem: format!( + "`binds[{0}].metadata` is `true`, but `binds[{0}].offset` \ + does not fall within the metadata mip tail binding range \ + of `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageOpaqueMemoryBindInfo-pBinds-01103"], + ..Default::default() + }) + })?; + + if size > image_mip_tail_size - offset_from_mip_tail { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].metadata` is `true`, but \ + `binds[{0}].offset + binds[{0}].size` is greater than \ + the end of the metadata mip tail binding range of `image`", + index, + ) + .into(), + vuids: &["VUID-VkSparseImageOpaqueMemoryBindInfo-pBinds-01103"], + ..Default::default() + })); + } + } + } else { + // VkSparseMemoryBind spec: + // If flags does not contain VK_SPARSE_MEMORY_BIND_METADATA_BIT, + // the binding range must be within the range [0,VkMemoryRequirements::size). + + if offset >= layout.size() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset` is not less than \ + `image.memory_requirements()[0].layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-resourceOffset-01099"], + ..Default::default() + })); + } + + if size > layout.size() - offset { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].offset + binds[{0}].size` is greater than \ + `image.memory_requirements()[0].layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-size-01100"], + ..Default::default() + })); + } + } + + if !is_aligned(offset, layout.alignment()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset` is not aligned according to \ + `image.memory_requirements()[0].layout.alignment()`", + index + ) + .into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2255, + ..Default::default() + })); + } + + if !(size == layout.size() - offset || is_aligned(size, layout.alignment())) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{0}].offset + binds[{}].size` is not equal to \ + `image.memory_requirements()[0].layout.size()`, but is not aligned \ + according to `image.memory_requirements()[0].layout.alignment()`", + index + ) + .into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2255, + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + if memory.allocation_size() < layout.size() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.allocation_size()` is less than \ + `image.memory_requirements()[0].layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if !is_aligned(memory_offset, layout.alignment()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.1` is not aligned according to \ + `image.memory_requirements()[0].layout.alignment()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if memory_type_bits & (1 << memory.memory_type_index()) == 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.memory_type_index()` is not a bit set in \ + `image.memory_requirements()[0].memory_type_bits`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01096"], + ..Default::default() + })); + } + + if !memory.export_handle_types().is_empty() { + if !external_memory_handle_types.intersects(memory.export_handle_types()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.export_handle_types()` is not empty, but \ + it does not share at least one memory type with \ + `image.external_memory_handle_types()`", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-02730"], + ..Default::default() + })); + } + } + + if let Some(handle_type) = memory.imported_handle_type() { + if !external_memory_handle_types.intersects(handle_type.into()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0` is imported, but \ + `image.external_memory_handle_types()` \ + does not contain the imported handle type", + index + ) + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-02731"], + ..Default::default() + })); + } + } + } + } + + Ok(()) + } + + pub(crate) fn to_vk<'a>( + &self, + fields1_vk: &'a SparseImageOpaqueMemoryBindInfoFields1Vk, + ) -> ash::vk::SparseImageOpaqueMemoryBindInfo<'a> { + let Self { image, binds: _ } = self; + let SparseImageOpaqueMemoryBindInfoFields1Vk { binds_vk } = fields1_vk; + + ash::vk::SparseImageOpaqueMemoryBindInfo::default() + .image(image.handle()) + .binds(binds_vk) + } + + pub(crate) fn to_vk_fields1(&self) -> SparseImageOpaqueMemoryBindInfoFields1Vk { + let Self { image: _, binds } = self; + + let binds_vk = binds + .iter() + .map(SparseImageOpaqueMemoryBind::to_vk) + .collect(); + + SparseImageOpaqueMemoryBindInfoFields1Vk { binds_vk } + } +} + +pub(crate) struct SparseImageOpaqueMemoryBindInfoFields1Vk { + pub(crate) binds_vk: SmallVec<[ash::vk::SparseMemoryBind; 4]>, +} + +/// Parameters for a single sparse bind operation on parts of an image with an opaque memory +/// layout. +/// +/// This type of sparse bind should be used for mip tail regions, the metadata aspect, and for the +/// normal regions of images that do not have the `sparse_residency` flag set. +#[derive(Clone, Debug, Default)] +pub struct SparseImageOpaqueMemoryBind { + /// The offset in bytes from the start of the image's memory, where memory is to be (un)bound. + /// + /// The default value is `0`. + pub offset: DeviceSize, + + /// The size in bytes of the memory to be (un)bound. + /// + /// The default value is `0`, which must be overridden. + pub size: DeviceSize, + + /// If `Some`, specifies the memory and an offset into that memory that is to be bound. + /// The provided memory must match the image's memory requirements. + /// + /// If `None`, specifies that existing memory at the specified location is to be unbound. + /// + /// The default value is `None`. + pub memory: Option<(Arc, DeviceSize)>, + + /// Sets whether the binding should apply to the metadata aspect of the image, or to the + /// normal texel data. + /// + /// The default value is `false`. + pub metadata: bool, +} + +impl SparseImageOpaqueMemoryBind { + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + offset: _, + size, + ref memory, + metadata: _, + } = self; + + if size == 0 { + return Err(Box::new(ValidationError { + context: "size".into(), + problem: "is zero".into(), + vuids: &["VUID-VkSparseMemoryBind-size-01098"], + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + let memory_type = &device.physical_device().memory_properties().memory_types + [memory.memory_type_index() as usize]; + + if memory_type + .property_flags + .intersects(MemoryPropertyFlags::LAZILY_ALLOCATED) + { + return Err(Box::new(ValidationError { + problem: "`memory.0.memory_type_index()` refers to a memory type whose \ + `property_flags` contains `MemoryPropertyFlags::LAZILY_ALLOCATED`" + .into(), + vuids: &["VUID-VkSparseMemoryBind-memory-01097"], + ..Default::default() + })); + } + + if memory_offset >= memory.allocation_size() { + return Err(Box::new(ValidationError { + problem: "`memory.1` is not less than `memory.0.allocation_size()`".into(), + vuids: &["VUID-VkSparseMemoryBind-memoryOffset-01101"], + ..Default::default() + })); + } + + if size > memory.allocation_size() - memory_offset { + return Err(Box::new(ValidationError { + problem: "`size` is greater than `memory.0.allocation_size()` minus \ + `memory.1`" + .into(), + vuids: &["VUID-VkSparseMemoryBind-size-01102"], + ..Default::default() + })); + } + } + + Ok(()) + } + + pub(crate) fn to_vk(&self) -> ash::vk::SparseMemoryBind { + let &Self { + offset, + size, + ref memory, + metadata, + } = self; + + let (memory, memory_offset) = memory + .as_ref() + .map_or_else(Default::default, |(memory, memory_offset)| { + (memory.handle(), *memory_offset) + }); + + ash::vk::SparseMemoryBind { + resource_offset: offset, + size, + memory, + memory_offset, + flags: if metadata { + ash::vk::SparseMemoryBindFlags::METADATA + } else { + ash::vk::SparseMemoryBindFlags::empty() + }, + } + } +} + +/// Parameters for sparse bind operations on parts of an image with a known memory layout. +/// +/// This type of sparse bind can only be used for images that have the `sparse_residency` flag set. +/// Only the normal texel regions can be bound this way, not the mip tail regions or metadata +/// aspect. +#[derive(Clone, Debug)] +pub struct SparseImageMemoryBindInfo { + /// The image to perform the binding operations on. + /// + /// There is no default value. + pub image: Arc, + + /// The bind operations to perform. + /// + /// The default value is empty. + pub binds: Vec, +} + +impl SparseImageMemoryBindInfo { + /// Returns a `SparseImageMemoryBindInfo` with the specified `image`. + #[inline] + pub fn new(image: Arc) -> Self { + Self { + image, + binds: Vec::new(), + } + } + + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + ref image, + ref binds, + } = self; + + assert_eq!(device, image.device().as_ref()); + assert!(!binds.is_empty()); + + if !image.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + return Err(Box::new(ValidationError { + context: "image.flags()".into(), + problem: "does not contain `ImageCreateFlags::SPARSE_BINDING`".into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2253 + ..Default::default() + })); + } + + if !image.flags().intersects(ImageCreateFlags::SPARSE_RESIDENCY) { + return Err(Box::new(ValidationError { + context: "image.flags()".into(), + problem: "does not contain `ImageCreateFlags::SPARSE_RESIDENCY`".into(), + vuids: &["VUID-VkSparseImageMemoryBindInfo-image-02901"], + ..Default::default() + })); + } + + let sparse_memory_requirements = match image.memory() { + ImageMemory::Sparse(sparse_memory_requirements) => sparse_memory_requirements, + _ => unreachable!(), + }; + let image_format_subsampled_extent = image + .format() + .ycbcr_chroma_sampling() + .map_or(image.extent(), |s| s.subsampled_extent(image.extent())); + let external_memory_handle_types = image.external_memory_handle_types(); + + for (index, bind) in binds.iter().enumerate() { + bind.validate(device) + .map_err(|err| err.add_context(format!("binds[{}]", index)))?; + + let &SparseImageMemoryBind { + aspects, + mip_level, + array_layer, + offset, + extent, + ref memory, + } = bind; + + if mip_level >= image.mip_levels() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].mip_level` is not less than `image.mip_levels()`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-subresource-01106"], + ..Default::default() + })); + } + + if array_layer >= image.array_layers() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].array_layer` is not less than `image.array_layers()`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-subresource-01106"], + ..Default::default() + })); + } + + if !image.format().aspects().contains(aspects) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].aspects` is not a subset of the aspects of the \ + format of `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-subresource-01106"], + ..Default::default() + })); + } + + let &SparseImageMemoryRequirements { + format_properties: + SparseImageFormatProperties { + aspects, + image_granularity, + flags: _, + }, + image_mip_tail_first_lod, + image_mip_tail_size: _, + image_mip_tail_offset: _, + image_mip_tail_stride: _, + } = sparse_memory_requirements + .iter() + .find(|reqs| reqs.format_properties.aspects == aspects) + .ok_or_else(|| { + Box::new(ValidationError { + problem: format!( + "there is no `SparseImageMemoryRequirements` element in \ + `image.memory()` where `format_properties.aspects` equals \ + binds[{}].aspects", + index + ) + .into(), + // vuids: https://github.com/KhronosGroup/Vulkan-Docs/issues/2254 + ..Default::default() + }) + })?; + + if mip_level >= image_mip_tail_first_lod { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].mip_level` is not less than \ + `SparseImageMemoryRequirements::image_mip_tail_first_lod` for `image`", + index + ) + .into(), + // vuids: TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2258 + ..Default::default() + })); + } + + if offset[0] % image_granularity[0] != 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset[0]` is not a multiple of ` + `SparseImageMemoryRequirements::format_properties.image_granularity[0]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-offset-01107"], + ..Default::default() + })); + } + + if offset[1] % image_granularity[1] != 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset[1]` is not a multiple of ` + `SparseImageMemoryRequirements::format_properties.image_granularity[1]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-offset-01109"], + ..Default::default() + })); + } + + if offset[2] % image_granularity[2] != 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].offset[2]` is not a multiple of ` + `SparseImageMemoryRequirements::format_properties.image_granularity[2]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-offset-01111"], + ..Default::default() + })); + } + + let mut subresource_extent = mip_level_extent(image.extent(), mip_level).unwrap(); + + // Only subsample if there are no other aspects. + if aspects.intersects(ImageAspects::PLANE_1 | ImageAspects::PLANE_2) + && (aspects - (ImageAspects::PLANE_1 | ImageAspects::PLANE_2)).is_empty() + { + subresource_extent = image_format_subsampled_extent; + } + + if !(extent[0] == subresource_extent[0] || extent[0] % image_granularity[0] == 0) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].extent[0]` is not equal to the width of the selected \ + subresource of `image`, but it is not a multiple of \ + `SparseImageMemoryRequirements::format_properties.image_granularity[0]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-01108"], + ..Default::default() + })); + } + + if !(extent[1] == subresource_extent[1] || extent[1] % image_granularity[1] == 0) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].extent[1]` is not equal to the height of the selected \ + subresource of `image`, but it is not a multiple of \ + `SparseImageMemoryRequirements::format_properties.image_granularity[1]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-01110"], + ..Default::default() + })); + } + + if !(extent[2] == subresource_extent[2] || extent[2] % image_granularity[2] == 0) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].extent[2]` is not equal to the depth of the selected \ + subresource of `image`, but it is not a multiple of \ + `SparseImageMemoryRequirements::format_properties.image_granularity[2]` \ + for `image`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-01112"], + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + let &MemoryRequirements { + layout, + memory_type_bits, + prefers_dedicated_allocation: _, + requires_dedicated_allocation: _, + } = &image.memory_requirements()[0]; // TODO: what to do about disjoint images? + + if memory.allocation_size() < layout.size() { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.allocation_size()` is less than \ + `image.memory_requirements()[0].layout.size()`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-memory-01105"], + ..Default::default() + })); + } + + if !is_aligned(memory_offset, layout.alignment()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.1` is not aligned according to \ + `image.memory_requirements()[0].layout.alignment()`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-memory-01105"], + ..Default::default() + })); + } + + if memory_type_bits & (1 << memory.memory_type_index()) == 0 { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.memory_type_index()` is not a bit set in \ + `image.memory_requirements()[0].memory_type_bits`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-memory-01105"], + ..Default::default() + })); + } + + if !memory.export_handle_types().is_empty() { + if !external_memory_handle_types.intersects(memory.export_handle_types()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0.export_handle_types()` is not empty, but \ + it does not share at least one memory type with \ + `image.external_memory_handle_types()`", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-memory-02732"], + ..Default::default() + })); + } + } + + if let Some(handle_type) = memory.imported_handle_type() { + if !external_memory_handle_types.intersects(handle_type.into()) { + return Err(Box::new(ValidationError { + problem: format!( + "`binds[{}].memory.0` is imported, but \ + `image.external_memory_handle_types()` \ + does not contain the imported handle type", + index + ) + .into(), + vuids: &["VUID-VkSparseImageMemoryBind-memory-02733"], + ..Default::default() + })); + } + } + } + } + + Ok(()) + } + + pub(crate) fn to_vk<'a>( + &self, + fields1_vk: &'a SparseImageMemoryBindInfoFields1Vk, + ) -> ash::vk::SparseImageMemoryBindInfo<'a> { + let Self { image, binds: _ } = self; + let SparseImageMemoryBindInfoFields1Vk { binds_vk } = fields1_vk; + + ash::vk::SparseImageMemoryBindInfo::default() + .image(image.handle()) + .binds(binds_vk) + } + + pub(crate) fn to_vk_fields1(&self) -> SparseImageMemoryBindInfoFields1Vk { + let Self { image: _, binds } = self; + + let binds_vk = binds.iter().map(SparseImageMemoryBind::to_vk).collect(); + + SparseImageMemoryBindInfoFields1Vk { binds_vk } + } +} + +pub(crate) struct SparseImageMemoryBindInfoFields1Vk { + pub(crate) binds_vk: SmallVec<[ash::vk::SparseImageMemoryBind; 4]>, +} + +/// Parameters for a single sparse bind operation on parts of an image with a known memory layout. +/// +/// This type of sparse bind can only be used for images that have the `sparse_residency` flag set. +/// Only the normal texel regions can be bound this way, not the mip tail regions or metadata +/// aspect. +#[derive(Clone, Debug, Default)] +pub struct SparseImageMemoryBind { + /// The aspects of the image where memory is to be (un)bound. + /// + /// The default value is `ImageAspects::empty()`, which must be overridden. + pub aspects: ImageAspects, + + /// The mip level of the image where memory is to be (un)bound. + /// + /// The default value is `0`. + pub mip_level: u32, + + /// The array layer of the image where memory is to be (un)bound. + /// + /// The default value is `0`. + pub array_layer: u32, + + /// The offset in texels (or for compressed images, texel blocks) from the origin of the image, + /// where memory is to be (un)bound. + /// + /// This must be a multiple of the + /// [`SparseImageFormatProperties::image_granularity`](crate::image::SparseImageFormatProperties::image_granularity) + /// value of the image. + /// + /// The default value is `[0; 3]`. + pub offset: [u32; 3], + + /// The extent in texels (or for compressed images, texel blocks) of the image where + /// memory is to be (un)bound. + /// + /// This must be a multiple of the + /// [`SparseImageFormatProperties::image_granularity`](crate::image::SparseImageFormatProperties::image_granularity) + /// value of the image, or `offset + extent` for that dimension must equal the image's total + /// extent. + /// + /// The default value is `[0; 3]`, which must be overridden. + pub extent: [u32; 3], + + /// If `Some`, specifies the memory and an offset into that memory that is to be bound. + /// The provided memory must match the image's memory requirements. + /// + /// If `None`, specifies that existing memory at the specified location is to be unbound. + /// + /// The default value is `None`. + pub memory: Option<(Arc, DeviceSize)>, +} + +impl SparseImageMemoryBind { + pub(crate) fn validate(&self, device: &Device) -> Result<(), Box> { + let &Self { + aspects, + mip_level: _, + array_layer: _, + offset: _, + extent, + ref memory, + } = self; + + aspects.validate_device(device).map_err(|err| { + err.add_context("aspects") + .set_vuids(&["VUID-VkImageSubresource-aspectMask-parameter"]) + })?; + + if aspects.is_empty() { + return Err(Box::new(ValidationError { + context: "aspects".into(), + problem: "is empty".into(), + vuids: &["VUID-VkImageSubresource-aspectMask-requiredbitmask"], + ..Default::default() + })); + } + + if let &Some((ref memory, memory_offset)) = memory { + if memory_offset >= memory.allocation_size() { + return Err(Box::new(ValidationError { + problem: "`memory.1` is not less than `memory.0.allocation_size()`".into(), + // vuids? + ..Default::default() + })); + } + } + + if extent[0] == 0 { + return Err(Box::new(ValidationError { + context: "extent[0]".into(), + problem: "is zero".into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-09388"], + ..Default::default() + })); + } + + if extent[1] == 0 { + return Err(Box::new(ValidationError { + context: "extent[1]".into(), + problem: "is zero".into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-09389"], + ..Default::default() + })); + } + + if extent[2] == 0 { + return Err(Box::new(ValidationError { + context: "extent[2]".into(), + problem: "is zero".into(), + vuids: &["VUID-VkSparseImageMemoryBind-extent-09390"], + ..Default::default() + })); + } + + // VUID-VkSparseImageMemoryBind-memory-01104 + // If the sparseResidencyAliased feature is not enabled, and if any other resources are + // bound to ranges of memory, the range of memory being bound must not overlap with + // those bound ranges + + Ok(()) + } + + pub(crate) fn to_vk(&self) -> ash::vk::SparseImageMemoryBind { + let &Self { + aspects, + mip_level, + array_layer, + offset, + extent, + ref memory, + } = self; + + let (memory, memory_offset) = memory + .as_ref() + .map_or_else(Default::default, |(memory, memory_offset)| { + (memory.handle(), *memory_offset) + }); + + ash::vk::SparseImageMemoryBind { + subresource: ash::vk::ImageSubresource { + aspect_mask: aspects.into(), + mip_level, + array_layer, + }, + offset: ash::vk::Offset3D { + x: offset[0] as i32, + y: offset[1] as i32, + z: offset[2] as i32, + }, + extent: ash::vk::Extent3D { + width: extent[0], + height: extent[1], + depth: extent[2], + }, + memory, + memory_offset, + flags: ash::vk::SparseMemoryBindFlags::empty(), + } + } +} diff --git a/vulkano/src/sync/future/fence_signal.rs b/vulkano/src/sync/future/fence_signal.rs index bb114bd07a..c255df4c51 100644 --- a/vulkano/src/sync/future/fence_signal.rs +++ b/vulkano/src/sync/future/fence_signal.rs @@ -2,7 +2,7 @@ use super::{AccessCheckError, GpuFuture}; use crate::{ buffer::Buffer, command_buffer::{SemaphoreSubmitInfo, SubmitInfo}, - device::{Device, DeviceOwned, Queue, QueueFlags}, + device::{Device, DeviceOwned, Queue}, image::{Image, ImageLayout}, swapchain::Swapchain, sync::{ @@ -282,10 +282,6 @@ where debug_assert!(!partially_flushed); // Same remark as `CommandBuffer`. assert!(fence.is_none()); - debug_assert!(queue.device().physical_device().queue_family_properties() - [queue.queue_family_index() as usize] - .queue_flags - .intersects(QueueFlags::SPARSE_BINDING)); unsafe { queue_bind_sparse(&queue, bind_infos, Some(new_fence.clone())) } .map_err(OutcomeErr::Full) diff --git a/vulkano/src/sync/future/mod.rs b/vulkano/src/sync/future/mod.rs index 504ef1d2d7..423e6dbd33 100644 --- a/vulkano/src/sync/future/mod.rs +++ b/vulkano/src/sync/future/mod.rs @@ -97,7 +97,7 @@ use crate::{ }, device::{DeviceOwned, Queue}, image::{Image, ImageLayout, ImageState}, - memory::BindSparseInfo, + memory::sparse::BindSparseInfo, swapchain::{self, PresentFuture, PresentInfo, Swapchain, SwapchainPresentInfo}, DeviceSize, Validated, ValidationError, VulkanError, VulkanObject, }; @@ -556,7 +556,7 @@ pub(crate) unsafe fn queue_bind_sparse( fence: Option>, ) -> Result<(), Validated> { let bind_infos: SmallVec<[_; 4]> = bind_infos.into_iter().collect(); - queue.with(|mut queue_guard| queue_guard.bind_sparse_unchecked(&bind_infos, fence.as_ref()))?; + queue.with(|mut queue_guard| queue_guard.bind_sparse(&bind_infos, fence.as_ref()))?; Ok(()) } From 3c185a3b59a59620176e1dca2ed0f13fceada509 Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 10:52:46 +0100 Subject: [PATCH 3/8] Merge `assume_bound`, make sparse image memory requirements available for raw images --- vulkano/src/buffer/sys.rs | 34 ++++------ vulkano/src/image/mod.rs | 22 +++++-- vulkano/src/image/sys.rs | 117 ++++++++++++++--------------------- vulkano/src/memory/sparse.rs | 26 +++----- 4 files changed, 84 insertions(+), 115 deletions(-) diff --git a/vulkano/src/buffer/sys.rs b/vulkano/src/buffer/sys.rs index ed3d3ce836..c870931895 100644 --- a/vulkano/src/buffer/sys.rs +++ b/vulkano/src/buffer/sys.rs @@ -522,19 +522,15 @@ impl RawBuffer { Ok(Buffer::from_raw(self, BufferMemory::Normal(allocation))) } - /// Assume this buffer has memory bound to it. + /// Converts a raw buffer into a full buffer without binding any memory. /// /// # Safety /// - /// - The buffer must have memory bound to it. - pub unsafe fn assume_bound(self) -> Buffer { - Buffer::from_raw(self, BufferMemory::External) - } - - /// Converts a raw buffer, that was created with the [`BufferCreateInfo::SPARSE_BINDING`] flag, - /// into a full buffer without binding any memory. + /// If `self.flags()` does not contain [`BufferCreateFlags::SPARSE_BINDING`]: /// - /// # Safety + /// - The buffer must already have a suitable memory allocation bound to it. + /// + /// If `self.flags()` does contain [`BufferCreateFlags::SPARSE_BINDING`]: /// /// - If `self.flags()` does not contain [`BufferCreateFlags::SPARSE_RESIDENCY`], then the /// buffer must be fully bound with memory before its memory is accessed by the device. @@ -543,20 +539,14 @@ impl RawBuffer { /// as determined by the [`Properties::residency_non_resident_strict`] device property. /// /// [`Properties::residency_non_resident_strict`]: crate::device::Properties::residency_non_resident_strict - pub unsafe fn into_buffer_sparse(self) -> Result, RawBuffer)> { - if !self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { - return Err(( - Box::new(ValidationError { - context: "self.flags()".into(), - problem: "does not contain `BufferCreateFlags::SPARSE_BINDING`".into(), - ..Default::default() - }) - .into(), - self, - )); - } + pub unsafe fn assume_bound(self) -> Buffer { + let memory = if self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) { + BufferMemory::Sparse + } else { + BufferMemory::External + }; - Ok(Buffer::from_raw(self, BufferMemory::Sparse)) + Buffer::from_raw(self, memory) } /// Returns the memory requirements for this buffer. diff --git a/vulkano/src/image/mod.rs b/vulkano/src/image/mod.rs index 992fffcac6..e11a9b9d4f 100644 --- a/vulkano/src/image/mod.rs +++ b/vulkano/src/image/mod.rs @@ -125,7 +125,7 @@ pub enum ImageMemory { /// The image is backed by sparse memory, bound with [`bind_sparse`]. /// /// [`bind_sparse`]: crate::device::QueueGuard::bind_sparse - Sparse(Vec), + Sparse, /// The image is backed by memory owned by a [`Swapchain`]. Swapchain { @@ -247,14 +247,24 @@ impl Image { /// Returns the memory requirements for this image. /// /// - If the image is a swapchain image, this returns a slice with a length of 0. - /// - If `self.flags().disjoint` is not set, this returns a slice with a length of 1. - /// - If `self.flags().disjoint` is set, this returns a slice with a length equal to - /// `self.format().planes().len()`. + /// - If `self.flags()` does not contain `ImageCreateFlags::DISJOINT`, this returns a slice + /// with a length of 1. + /// - If `self.flags()` does contain `ImageCreateFlags::DISJOINT`, this returns a slice with a + /// length equal to `self.format().unwrap().planes().len()`. #[inline] pub fn memory_requirements(&self) -> &[MemoryRequirements] { self.inner.memory_requirements() } + /// Returns the sparse memory requirements for this image. + /// + /// If `self.flags()` does not contain both `ImageCreateFlags::SPARSE_BINDING` and + /// `ImageCreateFlags::SPARSE_RESIDENCY`, this returns an empty slice. + #[inline] + pub fn sparse_memory_requirements(&self) -> &[SparseImageMemoryRequirements] { + self.inner.sparse_memory_requirements() + } + /// Returns the flags the image was created with. #[inline] pub fn flags(&self) -> ImageCreateFlags { @@ -510,7 +520,7 @@ impl Image { pub(crate) unsafe fn layout_initialized(&self) { match &self.memory { - ImageMemory::Normal(..) | ImageMemory::Sparse(..) | ImageMemory::External => { + ImageMemory::Normal(..) | ImageMemory::Sparse | ImageMemory::External => { self.is_layout_initialized.store(true, Ordering::Release); } ImageMemory::Swapchain { @@ -524,7 +534,7 @@ impl Image { pub(crate) fn is_layout_initialized(&self) -> bool { match &self.memory { - ImageMemory::Normal(..) | ImageMemory::Sparse(..) | ImageMemory::External => { + ImageMemory::Normal(..) | ImageMemory::Sparse | ImageMemory::External => { self.is_layout_initialized.load(Ordering::Acquire) } ImageMemory::Swapchain { diff --git a/vulkano/src/image/sys.rs b/vulkano/src/image/sys.rs index f436a006af..f9bdd120ab 100644 --- a/vulkano/src/image/sys.rs +++ b/vulkano/src/image/sys.rs @@ -69,6 +69,7 @@ pub struct RawImage { external_memory_handle_types: ExternalMemoryHandleTypes, memory_requirements: SmallVec<[MemoryRequirements; 4]>, + sparse_memory_requirements: Vec, needs_destruction: bool, // `vkDestroyImage` is called only if true. subresource_layout: OnceCache<(ImageAspect, u32, u32), SubresourceLayout>, } @@ -235,6 +236,14 @@ impl RawImage { smallvec![] }; + let sparse_memory_requirements = if flags + .contains(ImageCreateFlags::SPARSE_BINDING | ImageCreateFlags::SPARSE_RESIDENCY) + { + Self::get_sparse_memory_requirements(&device, handle) + } else { + Vec::new() + }; + Ok(RawImage { handle, device: InstanceOwnedDebugWrapper(device), @@ -258,6 +267,7 @@ impl RawImage { external_memory_handle_types, memory_requirements, + sparse_memory_requirements, needs_destruction, subresource_layout: OnceCache::new(), }) @@ -360,19 +370,16 @@ impl RawImage { ) } - fn get_sparse_memory_requirements(&self) -> Vec { - if !self.flags.intersects(ImageCreateFlags::SPARSE_RESIDENCY) { - return Vec::new(); - } - - let device = &self.device; - let fns = self.device.fns(); + unsafe fn get_sparse_memory_requirements( + device: &Device, + handle: ash::vk::Image, + ) -> Vec { + let fns = device.fns(); if device.api_version() >= Version::V1_1 || device.enabled_extensions().khr_get_memory_requirements2 { - let info2_vk = - ash::vk::ImageSparseMemoryRequirementsInfo2::default().image(self.handle); + let info2_vk = ash::vk::ImageSparseMemoryRequirementsInfo2::default().image(handle); let mut count = 0; @@ -403,7 +410,7 @@ impl RawImage { if device.api_version() >= Version::V1_1 { unsafe { (fns.v1_1.get_image_sparse_memory_requirements2)( - self.device.handle(), + device.handle(), &info2_vk, &mut count, requirements2_vk.as_mut_ptr(), @@ -413,7 +420,7 @@ impl RawImage { unsafe { (fns.khr_get_memory_requirements2 .get_image_sparse_memory_requirements2_khr)( - self.device.handle(), + device.handle(), &info2_vk, &mut count, requirements2_vk.as_mut_ptr(), @@ -432,7 +439,7 @@ impl RawImage { unsafe { (fns.v1_0.get_image_sparse_memory_requirements)( device.handle(), - self.handle, + handle, &mut count, ptr::null_mut(), ) @@ -444,7 +451,7 @@ impl RawImage { unsafe { (fns.v1_0.get_image_sparse_memory_requirements)( device.handle(), - self.handle, + handle, &mut count, requirements_vk.as_mut_ptr(), ) @@ -969,11 +976,16 @@ impl RawImage { )) } - /// Converts a raw image, that was created with the [`ImageCreateInfo::SPARSE_BINDING`] flag, - /// into a full image without binding any memory. + /// Converts a raw image into a full image without binding any memory. /// /// # Safety /// + /// - The image must already have a suitable memory allocation bound to it. + /// + /// If `self.flags()` does not contain [`ImageCreateFlags::SPARSE_BINDING`]: + /// + /// If `self.flags()` does contain [`ImageCreateFlags::SPARSE_BINDING`]: + /// /// - If `self.flags()` does not contain [`ImageCreateFlags::SPARSE_RESIDENCY`], then the image /// must be fully bound with memory before any memory is accessed by the device. /// - If `self.flags()` contains [`ImageCreateFlags::SPARSE_RESIDENCY`], then if the sparse @@ -984,27 +996,15 @@ impl RawImage { /// as determined by the [`Properties::residency_non_resident_strict`] device property. /// /// [`Properties::residency_non_resident_strict`]: crate::device::Properties::residency_non_resident_strict - pub unsafe fn into_image_sparse(self) -> Result, RawImage)> { - if !self.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { - return Err(( - Box::new(ValidationError { - context: "self.flags()".into(), - problem: "does not contain `ImageCreateFlags::SPARSE_BINDING`".into(), - ..Default::default() - }) - .into(), - self, - )); - } - - let sparse_image_memory_requirements = self.get_sparse_memory_requirements(); + pub unsafe fn assume_bound(self) -> Image { + let memory = if self.flags().intersects(ImageCreateFlags::SPARSE_BINDING) { + ImageMemory::Sparse + } else { + ImageMemory::External + }; let layout = self.default_layout(); - Ok(Image::from_raw( - self, - ImageMemory::Sparse(sparse_image_memory_requirements), - layout, - )) + Image::from_raw(self, memory, layout) } fn default_layout(&self) -> ImageLayout { @@ -1033,50 +1033,27 @@ impl RawImage { } } - /// Assume that this image already has memory backing it. - /// - /// # Safety - /// - /// - The image must be backed by suitable memory allocations. - pub unsafe fn assume_bound(self) -> Image { - let usage = self - .usage - .difference(ImageUsage::TRANSFER_SRC | ImageUsage::TRANSFER_DST); - - let layout = if usage.intersects(ImageUsage::SAMPLED | ImageUsage::INPUT_ATTACHMENT) - && usage - .difference(ImageUsage::SAMPLED | ImageUsage::INPUT_ATTACHMENT) - .is_empty() - { - ImageLayout::ShaderReadOnlyOptimal - } else if usage.intersects(ImageUsage::COLOR_ATTACHMENT) - && usage.difference(ImageUsage::COLOR_ATTACHMENT).is_empty() - { - ImageLayout::ColorAttachmentOptimal - } else if usage.intersects(ImageUsage::DEPTH_STENCIL_ATTACHMENT) - && usage - .difference(ImageUsage::DEPTH_STENCIL_ATTACHMENT) - .is_empty() - { - ImageLayout::DepthStencilAttachmentOptimal - } else { - ImageLayout::General - }; - - Image::from_raw(self, ImageMemory::External, layout) - } - /// Returns the memory requirements for this image. /// /// - If the image is a swapchain image, this returns a slice with a length of 0. - /// - If `self.flags().disjoint` is not set, this returns a slice with a length of 1. - /// - If `self.flags().disjoint` is set, this returns a slice with a length equal to - /// `self.format().unwrap().planes().len()`. + /// - If `self.flags()` does not contain `ImageCreateFlags::DISJOINT`, this returns a slice + /// with a length of 1. + /// - If `self.flags()` does contain `ImageCreateFlags::DISJOINT`, this returns a slice with a + /// length equal to `self.format().unwrap().planes().len()`. #[inline] pub fn memory_requirements(&self) -> &[MemoryRequirements] { &self.memory_requirements } + /// Returns the sparse memory requirements for this image. + /// + /// If `self.flags()` does not contain both `ImageCreateFlags::SPARSE_BINDING` and + /// `ImageCreateFlags::SPARSE_RESIDENCY`, this returns an empty slice. + #[inline] + pub fn sparse_memory_requirements(&self) -> &[SparseImageMemoryRequirements] { + &self.sparse_memory_requirements + } + /// Returns the flags the image was created with. #[inline] pub fn flags(&self) -> ImageCreateFlags { diff --git a/vulkano/src/memory/sparse.rs b/vulkano/src/memory/sparse.rs index 3746737a1f..782e0566a8 100644 --- a/vulkano/src/memory/sparse.rs +++ b/vulkano/src/memory/sparse.rs @@ -3,8 +3,8 @@ use crate::{ buffer::{Buffer, BufferCreateFlags}, device::{Device, DeviceOwned}, image::{ - mip_level_extent, Image, ImageAspects, ImageCreateFlags, ImageMemory, - SparseImageFormatProperties, SparseImageMemoryRequirements, + mip_level_extent, Image, ImageAspects, ImageCreateFlags, SparseImageFormatProperties, + SparseImageMemoryRequirements, }, memory::{is_aligned, MemoryRequirements}, sync::semaphore::Semaphore, @@ -599,16 +599,11 @@ impl SparseImageOpaqueMemoryBindInfo { prefers_dedicated_allocation: _, requires_dedicated_allocation: _, } = &image.memory_requirements()[0]; // TODO: https://github.com/KhronosGroup/Vulkan-Docs/issues/2259 - let metadata_memory_requirements = match image.memory() { - ImageMemory::Sparse(sparse_memory_requirements) => { - sparse_memory_requirements.iter().find(|reqs| { - reqs.format_properties - .aspects - .intersects(ImageAspects::METADATA) - }) - } - _ => unreachable!(), - }; + let metadata_memory_requirements = image.sparse_memory_requirements().iter().find(|reqs| { + reqs.format_properties + .aspects + .intersects(ImageAspects::METADATA) + }); let external_memory_handle_types = image.external_memory_handle_types(); for (index, bind) in binds.iter().enumerate() { @@ -1060,10 +1055,6 @@ impl SparseImageMemoryBindInfo { })); } - let sparse_memory_requirements = match image.memory() { - ImageMemory::Sparse(sparse_memory_requirements) => sparse_memory_requirements, - _ => unreachable!(), - }; let image_format_subsampled_extent = image .format() .ycbcr_chroma_sampling() @@ -1131,7 +1122,8 @@ impl SparseImageMemoryBindInfo { image_mip_tail_size: _, image_mip_tail_offset: _, image_mip_tail_stride: _, - } = sparse_memory_requirements + } = image + .sparse_memory_requirements() .iter() .find(|reqs| reqs.format_properties.aspects == aspects) .ok_or_else(|| { From 9c5d55dfb3aaa7c08fdda68d689da1a7470839f0 Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 10:56:54 +0100 Subject: [PATCH 4/8] Remove assert that sneaked back in --- vulkano/src/buffer/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vulkano/src/buffer/mod.rs b/vulkano/src/buffer/mod.rs index a5428c466f..6485ec0595 100644 --- a/vulkano/src/buffer/mod.rs +++ b/vulkano/src/buffer/mod.rs @@ -377,7 +377,6 @@ impl Buffer { allocation_info: AllocationCreateInfo, layout: DeviceLayout, ) -> Result, Validated> { - assert!(layout.alignment().as_devicesize() <= 64); assert!(!create_info .flags .contains(BufferCreateFlags::SPARSE_BINDING)); From 247d212bdd42412038bb01889dd9189a3acc66c2 Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 11:01:19 +0100 Subject: [PATCH 5/8] Missing doc --- vulkano/src/image/sys.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulkano/src/image/sys.rs b/vulkano/src/image/sys.rs index f9bdd120ab..22883d0ca5 100644 --- a/vulkano/src/image/sys.rs +++ b/vulkano/src/image/sys.rs @@ -984,6 +984,8 @@ impl RawImage { /// /// If `self.flags()` does not contain [`ImageCreateFlags::SPARSE_BINDING`]: /// + /// - The image must already have a suitable memory allocation bound to it. + /// /// If `self.flags()` does contain [`ImageCreateFlags::SPARSE_BINDING`]: /// /// - If `self.flags()` does not contain [`ImageCreateFlags::SPARSE_RESIDENCY`], then the image From b5f45e5907cdd30ece4a9a9c03b662a3d6a3d447 Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 11:10:35 +0100 Subject: [PATCH 6/8] Update vulkano/src/memory/sparse.rs Co-authored-by: Lachlan Deakin --- vulkano/src/memory/sparse.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vulkano/src/memory/sparse.rs b/vulkano/src/memory/sparse.rs index 782e0566a8..8a1c69f835 100644 --- a/vulkano/src/memory/sparse.rs +++ b/vulkano/src/memory/sparse.rs @@ -1204,11 +1204,13 @@ impl SparseImageMemoryBindInfo { subresource_extent = image_format_subsampled_extent; } - if !(extent[0] == subresource_extent[0] || extent[0] % image_granularity[0] == 0) { + if !(offset[0] + extent[0] == subresource_extent[0] + || extent[0] % image_granularity[0] == 0) + { return Err(Box::new(ValidationError { problem: format!( - "`binds[{}].extent[0]` is not equal to the width of the selected \ - subresource of `image`, but it is not a multiple of \ + "`binds[{0}].offset[0]` + `binds[{0}].extent[0]` is not equal to the \ + width of the selected subresource of `image`, but it is not a multiple of \ `SparseImageMemoryRequirements::format_properties.image_granularity[0]` \ for `image`", index From 6771a75fad42d34f8bd29623426bed5bb438b26a Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 11:10:41 +0100 Subject: [PATCH 7/8] Update vulkano/src/memory/sparse.rs Co-authored-by: Lachlan Deakin --- vulkano/src/memory/sparse.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vulkano/src/memory/sparse.rs b/vulkano/src/memory/sparse.rs index 8a1c69f835..aa20fba52f 100644 --- a/vulkano/src/memory/sparse.rs +++ b/vulkano/src/memory/sparse.rs @@ -1221,11 +1221,13 @@ impl SparseImageMemoryBindInfo { })); } - if !(extent[1] == subresource_extent[1] || extent[1] % image_granularity[1] == 0) { + if !(offset[1] + extent[1] == subresource_extent[1] + || extent[1] % image_granularity[1] == 0) + { return Err(Box::new(ValidationError { problem: format!( - "`binds[{}].extent[1]` is not equal to the height of the selected \ - subresource of `image`, but it is not a multiple of \ + "`binds[{0}].offset[1]` + `binds[{0}].extent[1]` is not equal to the \ + height of the selected subresource of `image`, but it is not a multiple of \ `SparseImageMemoryRequirements::format_properties.image_granularity[1]` \ for `image`", index From c134079390e6c889833aa359f1ce826d00c310b3 Mon Sep 17 00:00:00 2001 From: Rua Date: Sat, 30 Nov 2024 11:10:46 +0100 Subject: [PATCH 8/8] Update vulkano/src/memory/sparse.rs Co-authored-by: Lachlan Deakin --- vulkano/src/memory/sparse.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vulkano/src/memory/sparse.rs b/vulkano/src/memory/sparse.rs index aa20fba52f..c753e98c1a 100644 --- a/vulkano/src/memory/sparse.rs +++ b/vulkano/src/memory/sparse.rs @@ -1238,11 +1238,13 @@ impl SparseImageMemoryBindInfo { })); } - if !(extent[2] == subresource_extent[2] || extent[2] % image_granularity[2] == 0) { + if !(offset[2] + extent[2] == subresource_extent[2] + || extent[2] % image_granularity[2] == 0) + { return Err(Box::new(ValidationError { problem: format!( - "`binds[{}].extent[2]` is not equal to the depth of the selected \ - subresource of `image`, but it is not a multiple of \ + "`binds[{0}].offset[2]` + `binds[{0}].extent[2]` is not equal to the \ + depth of the selected subresource of `image`, but it is not a multiple of \ `SparseImageMemoryRequirements::format_properties.image_granularity[2]` \ for `image`", index