Cuda interop vk13

include/nbl/video/CCUDADevice.h

@@ -37,6 +37,13 @@ class CCUDADevice : public core::IReferenceCounted
 		static constexpr IDeviceMemoryBacked::E_EXTERNAL_HANDLE_TYPE EXTERNAL_MEMORY_HANDLE_TYPE = IDeviceMemoryBacked::EHT_OPAQUE_FD;
 		static constexpr CUmemAllocationHandleType ALLOCATION_TYPE = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR;
 #endif
+		struct SCUDACleaner : video::ICleanup
+		{
+			core::smart_refctd_ptr<const core::IReferenceCounted> resource;
+			SCUDACleaner(core::smart_refctd_ptr<const core::IReferenceCounted> resource)
+				: resource(std::move(resource))
+			{ }
+		};
 
 		enum E_VIRTUAL_ARCHITECTURE
 		{
@@ -95,18 +102,10 @@ class CCUDADevice : public core::IReferenceCounted
 	protected:
 		CUresult reserveAdrressAndMapMemory(CUdeviceptr* outPtr, size_t size, size_t alignment, CUmemLocationType location, CUmemGenericAllocationHandle memory);
 
+
+		// CUDAHandler creates CUDADevice, it needs to access ctor
 		friend class CCUDAHandler;
-		friend class CCUDASharedMemory;
-		friend class CCUDASharedSemaphore;
 
-		struct SCUDACleaner : video::ICleanup
-		{
-			core::smart_refctd_ptr<const core::IReferenceCounted> resource;
-			SCUDACleaner(core::smart_refctd_ptr<const core::IReferenceCounted> resource)
-				: resource(std::move(resource))
-			{ }
-		};
-
 		CCUDADevice(core::smart_refctd_ptr<CVulkanConnection>&& _vulkanConnection, IPhysicalDevice* const _vulkanDevice, const E_VIRTUAL_ARCHITECTURE _virtualArchitecture, CUdevice _handle, core::smart_refctd_ptr<CCUDAHandler>&& _handler);
 		~CCUDADevice();
 

include/nbl/video/CCUDAHandler.h

@@ -136,6 +136,7 @@ class CCUDAHandler : public core::IReferenceCounted
 			,cuDestroyExternalSemaphore
 			,cuImportExternalSemaphore
 			,cuSignalExternalSemaphoresAsync
+			,cuWaitExternalSemaphoresAsync
 		);
 		const CUDA& getCUDAFunctionTable() const {return m_cuda;}
 

include/nbl/video/CCUDASharedMemory.h

@@ -23,6 +23,7 @@ namespace nbl::video
 class CCUDASharedMemory : public core::IReferenceCounted
 {
 public:
+    // required for us to see the move ctor
     friend class CCUDADevice;
 
     CUdeviceptr getDeviceptr() const { return m_params.ptr;  }
@@ -49,11 +50,11 @@ class CCUDASharedMemory : public core::IReferenceCounted
 
     core::smart_refctd_ptr<IDeviceMemoryAllocation> exportAsMemory(ILogicalDevice* device, IDeviceMemoryBacked* dedication = nullptr) const;
 
-    core::smart_refctd_ptr<IGPUImage>  createAndBindImage(ILogicalDevice* device, IGPUImage::SCreationParams&& params) const;
+    core::smart_refctd_ptr<IGPUImage>  createAndBindImage(ILogicalDevice* device, asset::IImage::SCreationParams&& params) const;
 
 protected:
 
-    CCUDASharedMemory(core::smart_refctd_ptr<CCUDADevice> device, SCachedCreationParams&& params)
+    CCUDASharedMemory(core::smart_refctd_ptr<CCUDADevice>&& device, SCachedCreationParams&& params)
         : m_device(std::move(device))
         , m_params(std::move(params))
     {}

include/nbl/video/IDeviceMemoryAllocation.h

@@ -164,14 +164,21 @@ class IDeviceMemoryAllocation : public virtual core::IReferenceCounted
         //! Constant variant of getMappedPointer
         inline const void* getMappedPointer() const { return m_mappedPtr; }
 
-        struct SCreationParams
+        struct SInfo
+        {
+            uint64_t allocationSize = 0;
+            core::bitflag<IDeviceMemoryAllocation::E_MEMORY_ALLOCATE_FLAGS> allocateFlags = IDeviceMemoryAllocation::EMAF_NONE;
+			// Handle Type for external resources
+			IDeviceMemoryAllocation::E_EXTERNAL_HANDLE_TYPE externalHandleType = IDeviceMemoryAllocation::EHT_NONE;
+			//! Imports the given handle  if externalHandle != nullptr && externalHandleType != EHT_NONE
+			//! Creates exportable memory if externalHandle == nullptr && externalHandleType != EHT_NONE
+			void* externalHandle = nullptr;
+        };
+
+        struct SCreationParams: SInfo
         {
-            core::bitflag<E_MEMORY_ALLOCATE_FLAGS> allocateFlags = E_MEMORY_ALLOCATE_FLAGS::EMAF_NONE;
             core::bitflag<E_MEMORY_PROPERTY_FLAGS> memoryPropertyFlags = E_MEMORY_PROPERTY_FLAGS::EMPF_NONE;
-            E_EXTERNAL_HANDLE_TYPE externalHandleType = E_EXTERNAL_HANDLE_TYPE::EHT_NONE;
-            void* externalHandle = nullptr;
             const bool dedicated = false;
-            const size_t allocationSize;
         };
 
     protected:
@@ -183,21 +190,21 @@ class IDeviceMemoryAllocation : public virtual core::IReferenceCounted
         IDeviceMemoryAllocation(
             const ILogicalDevice* originDevice, SCreationParams&& params = {})
             : m_originDevice(originDevice)
+            , m_params(std::move(params))
             , m_mappedPtr(nullptr)
             , m_mappedRange{ 0, 0 }
             , m_currentMappingAccess(EMCAF_NO_MAPPING_ACCESS)
-            , m_params(std::move(params))
         {}
 
         virtual void* map_impl(const MemoryRange& range, const core::bitflag<E_MAPPING_CPU_ACCESS_FLAGS> accessHint) = 0;
         virtual bool unmap_impl() = 0;
 
 
         const ILogicalDevice* m_originDevice = nullptr;
-        uint8_t* m_mappedPtr;
-        MemoryRange m_mappedRange;
-        core::bitflag<E_MAPPING_CPU_ACCESS_FLAGS> m_currentMappingAccess;
-        SCreationParams m_params;
+        SCreationParams m_params = {};
+        uint8_t* m_mappedPtr = nullptr;
+        MemoryRange m_mappedRange = {};
+        core::bitflag<E_MAPPING_CPU_ACCESS_FLAGS> m_currentMappingAccess = EMCAF_NO_MAPPING_ACCESS;
         std::unique_ptr<struct ICleanup> m_postDestroyCleanup = nullptr;
 };
 

include/nbl/video/IDeviceMemoryAllocator.h

@@ -12,19 +12,11 @@ namespace nbl::video
 class IDeviceMemoryAllocator
 {
 	public:
-		struct SAllocateInfo
+		struct SAllocateInfo: IDeviceMemoryAllocation::SInfo
 		{
-			size_t size : 54 = 0ull;
-			size_t flags : 5 = 0u; // IDeviceMemoryAllocation::E_MEMORY_ALLOCATE_FLAGS
-			size_t memoryTypeIndex : 5 = 0u;
+			uint32_t memoryTypeIndex = 0u;
 			IDeviceMemoryBacked* dedication = nullptr; // if you make the info have a `dedication` the memory will be bound right away, also it will use VK_KHR_dedicated_allocation on vulkan
 			// size_t opaqueCaptureAddress = 0u; Note that this mechanism is intended only to support capture/replay tools, and is not recommended for use in other applications.
-
-			// Handle Type for external resources
-			IDeviceMemoryAllocation::E_EXTERNAL_HANDLE_TYPE externalHandleType = IDeviceMemoryAllocation::EHT_NONE;
-			//! Imports the given handle  if externalHandle != nullptr && externalHandleType != EHT_NONE
-			//! Creates exportable memory if externalHandle == nullptr && externalHandleType != EHT_NONE
-			void* externalHandle = nullptr;
 		};
 
 		//! IMemoryTypeIterator extracts memoryType indices from memoryTypeBits in arbitrary order
@@ -54,8 +46,8 @@ class IDeviceMemoryAllocator
 				inline SAllocateInfo operator()(IDeviceMemoryBacked* dedication)
 				{
 					SAllocateInfo ret = {};
-					ret.size = m_reqs.size;
-					ret.flags = m_allocateFlags;
+					ret.allocationSize = m_reqs.size;
+					ret.allocateFlags = core::bitflag<IDeviceMemoryAllocation::E_MEMORY_ALLOCATE_FLAGS>(m_allocateFlags);
 					ret.memoryTypeIndex = dereference();
 					ret.dedication = dedication;
 					ret.externalHandleType = m_handleType;

include/nbl/video/IDeviceMemoryBacked.h

@@ -126,7 +126,7 @@ class IDeviceMemoryBacked : public IBackendObject
 
         //! members
         SCachedCreationParams m_cachedCreationParams;
-        SDeviceMemoryRequirements m_cachedMemoryReqs;
+        const SDeviceMemoryRequirements m_cachedMemoryReqs;
         void* m_cachedExternalHandle = nullptr;
 };
 

include/nbl/video/ILogicalDevice.h

@@ -147,7 +147,7 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe
         virtual IQueue::RESULT waitIdle() const = 0;
 
         //! Semaphore Stuff
-        virtual core::smart_refctd_ptr<ISemaphore> createSemaphore(ISemaphore::SCreationParams&&) = 0;
+        virtual core::smart_refctd_ptr<ISemaphore> createSemaphore(uint64_t initialValue = 0, ISemaphore::SCreationParams&& = {}) = 0;
         virtual ISemaphore::WAIT_RESULT waitForSemaphores(const std::span<const ISemaphore::SWaitInfo> infos, const bool waitAll, const uint64_t timeout) = 0;
         // Forever waiting variant if you're confident that the fence will eventually be signalled
         inline ISemaphore::WAIT_RESULT blockForSemaphores(const std::span<const ISemaphore::SWaitInfo> infos, const bool waitAll=true)
@@ -285,29 +285,14 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe
 
         //! Descriptor Creation
         // Buffer (@see ICPUBuffer)
-        inline core::smart_refctd_ptr<IGPUBuffer> createBuffer(IGPUBuffer::SCreationParams&& creationParams)
-        {
-            const auto maxSize = getPhysicalDeviceLimits().maxBufferSize;
-            if (creationParams.size>maxSize)
-            {
-                m_logger.log("Failed to create Buffer, size %d larger than Device %p's limit!",system::ILogger::ELL_ERROR,creationParams.size,this,maxSize);
-                return nullptr;
-            }
-            return createBuffer_impl(std::move(creationParams));
-        }
+        core::smart_refctd_ptr<IGPUBuffer> createBuffer(IGPUBuffer::SCreationParams&& creationParams);
+
         // Create a BufferView, to a shader; a fake 1D-like texture with no interpolation (@see ICPUBufferView)
         core::smart_refctd_ptr<IGPUBufferView> createBufferView(const asset::SBufferRange<const IGPUBuffer>& underlying, const asset::E_FORMAT _fmt);
+
         // Creates an Image (@see ICPUImage)
-        inline core::smart_refctd_ptr<IGPUImage> createImage(IGPUImage::SCreationParams&& creationParams)
-        {
-            if (!IGPUImage::validateCreationParameters(creationParams))
-            {
-                m_logger.log("Failed to create Image, invalid creation parameters!",system::ILogger::ELL_ERROR);
-                return nullptr;
-            }
-            // TODO: @Cyprian validation of creationParams against the device's limits (sample counts, etc.) see vkCreateImage
-            return createImage_impl(std::move(creationParams));
-        }
+        core::smart_refctd_ptr<IGPUImage> createImage(IGPUImage::SCreationParams&& params);
+
         // Create an ImageView that can actually be used by shaders (@see ICPUImageView)
         inline core::smart_refctd_ptr<IGPUImageView> createImageView(IGPUImageView::SCreationParams&& params)
         {
@@ -765,9 +750,9 @@ class NBL_API2 ILogicalDevice : public core::IReferenceCounted, public IDeviceMe
         virtual bool bindBufferMemory_impl(const uint32_t count, const SBindBufferMemoryInfo* pInfos) = 0;
         virtual bool bindImageMemory_impl(const uint32_t count, const SBindImageMemoryInfo* pInfos) = 0;
 
-        virtual core::smart_refctd_ptr<IGPUBuffer> createBuffer_impl(IGPUBuffer::SCreationParams&& creationParams) = 0;
+        virtual core::smart_refctd_ptr<IGPUBuffer> createBuffer_impl(IGPUBuffer::SCreationParams&& creationParams, bool dedicatedOnly = false) = 0;
         virtual core::smart_refctd_ptr<IGPUBufferView> createBufferView_impl(const asset::SBufferRange<const IGPUBuffer>& underlying, const asset::E_FORMAT _fmt) = 0;
-        virtual core::smart_refctd_ptr<IGPUImage> createImage_impl(IGPUImage::SCreationParams&& params) = 0;
+        virtual core::smart_refctd_ptr<IGPUImage> createImage_impl(IGPUImage::SCreationParams&& params, bool dedicatedOnly = false) = 0;
         virtual core::smart_refctd_ptr<IGPUImageView> createImageView_impl(IGPUImageView::SCreationParams&& params) = 0;
         virtual core::smart_refctd_ptr<IGPUBottomLevelAccelerationStructure> createBottomLevelAccelerationStructure_impl(IGPUAccelerationStructure::SCreationParams&& params) = 0;
         virtual core::smart_refctd_ptr<IGPUTopLevelAccelerationStructure> createTopLevelAccelerationStructure_impl(IGPUTopLevelAccelerationStructure::SCreationParams&& params) = 0;

include/nbl/video/ISemaphore.h

@@ -69,8 +69,6 @@ class ISemaphore : public IBackendObject
             //! Imports the given handle  if externalHandle != nullptr && externalMemoryHandleType != EHT_NONE
             //! Creates exportable memory if externalHandle == nullptr && externalMemoryHandleType != EHT_NONE
             void* externalHandle = nullptr;
-
-            uint64_t initialValue = 0;
         };
 
         auto const& getCreationParams() const
@@ -85,7 +83,7 @@ class ISemaphore : public IBackendObject
         {}
         virtual ~ISemaphore() = default;
 
-        SCreationParams m_creationParams;
+        const SCreationParams m_creationParams;
 };
 
 }

include/nbl/video/SPhysicalDeviceLimits.h

@@ -552,10 +552,6 @@ struct SPhysicalDeviceLimits
     /* CooperativeMatrixPropertiesKHR  *//* VK_KHR_cooperative_matrix */
     core::bitflag<asset::IShader::E_SHADER_STAGE> cooperativeMatrixSupportedStages = asset::IShader::ESS_UNKNOWN;
 
-    bool externalFenceWin32 = false; /* VK_KHR_external_fence_win32 */ // [TODO] requires instance extensions, add them
-    bool externalMemoryWin32 = false; /* VK_KHR_external_memory_win32 */ // [TODO] requires instance extensions, add them
-    bool externalSemaphoreWin32 = false; /* VK_KHR_external_semaphore_win32 */ // [TODO] requires instance extensions, add them
-
     /*  Always enabled if available, reported as limits */
 
     // Core 1.0 Features

include/nbl/video/utilities/IUtilities.h

@@ -234,7 +234,7 @@ class NBL_API2 IUtilities : public core::IReferenceCounted
         //! WARNING: This function blocks CPU and stalls the GPU!
         inline bool autoSubmitAndBlock(const SIntendedSubmitInfo::SFrontHalf& submit, const std::function<bool(SIntendedSubmitInfo&)>& what)
         {            
-            auto semaphore = m_device->createSemaphore(ISemaphore::SCreationParams{.initialValue=0});
+            auto semaphore = m_device->createSemaphore();
             // so we begin latching everything on the value of 1, but if we overflow it increases
             IQueue::SSubmitInfo::SSemaphoreInfo info = {semaphore.get(),1};
 

src/nbl/video/CCUDADevice.cpp

@@ -139,7 +139,7 @@ CUresult CCUDADevice::importGPUSemaphore(core::smart_refctd_ptr<CCUDASharedSemap
 
 	if (!handleType.hasFlags(ISemaphore::EHT_OPAQUE_WIN32) || !handle)
 		return CUDA_ERROR_INVALID_VALUE;
-
+    
 	CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC desc = {
 		.type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32,
 		.handle = {.win32 = {.handle = handle }},

src/nbl/video/CCUDASharedMemory.cpp

@@ -11,9 +11,11 @@ namespace nbl::video
 core::smart_refctd_ptr<IDeviceMemoryAllocation> CCUDASharedMemory::exportAsMemory(ILogicalDevice* device, IDeviceMemoryBacked* dedication) const
 {
 	IDeviceMemoryAllocator::SAllocateInfo info = {
-		.size = m_params.granularSize,
-		.externalHandleType = CCUDADevice::EXTERNAL_MEMORY_HANDLE_TYPE,
-		.externalHandle = m_params.osHandle,
+		{
+			.allocationSize = m_params.granularSize,
+			.externalHandleType = CCUDADevice::EXTERNAL_MEMORY_HANDLE_TYPE,
+			.externalHandle = m_params.osHandle,
+		}
 	};
 
 	auto pd = device->getPhysicalDevice();
@@ -43,46 +45,12 @@ core::smart_refctd_ptr<IDeviceMemoryAllocation> CCUDASharedMemory::exportAsMemor
 		std::make_unique<CCUDADevice::SCUDACleaner>(core::smart_refctd_ptr<const CCUDASharedMemory>(this))).memory;
 }
 
-#if 0
-core::smart_refctd_ptr<IGPUBuffer> CCUDASharedMemory::exportAsBuffer(ILogicalDevice* device, core::bitflag<asset::IBuffer::E_USAGE_FLAGS> usage) const
+core::smart_refctd_ptr<IGPUImage>  CCUDASharedMemory::createAndBindImage(ILogicalDevice* device, asset::IImage::SCreationParams&& params) const
 {
 	if (!device || !m_device->isMatchingDevice(device->getPhysicalDevice()))
 		return nullptr;
 
-	auto buf = device->createBuffer({{
-			.size = m_params.granularSize,
-			.usage = usage }, {{
-			.postDestroyCleanup = std::make_unique<CCUDADevice::SCUDACleaner>(core::smart_refctd_ptr<const CCUDASharedMemory>(this)),
-			.externalHandleTypes = CCUDADevice::EXTERNAL_MEMORY_HANDLE_TYPE,
-			.externalHandle = m_params.osHandle
-		}}});
-
-	auto req = buf->getMemoryReqs();
-	auto pd = device->getPhysicalDevice();
-	switch (m_params.location)
-	{
-	case CU_MEM_LOCATION_TYPE_DEVICE: req.memoryTypeBits &= pd->getDeviceLocalMemoryTypeBits(); break;
-	case CU_MEM_LOCATION_TYPE_HOST: req.memoryTypeBits &= pd->getHostVisibleMemoryTypeBits(); break;
-	// TODO(Atil): Figure out how to handle these
-	case CU_MEM_LOCATION_TYPE_HOST_NUMA: 
-	case CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT: 
-	default: break;
-	}
-
-	if (!device->allocate(req, buf.get()).isValid())
-		return nullptr;
-
-	return buf;
-}
-
-#endif
-
-core::smart_refctd_ptr<IGPUImage>  CCUDASharedMemory::createAndBindImage(ILogicalDevice* device, IGPUImage::SCreationParams&& params) const
-{
-	if (!device || !m_device->isMatchingDevice(device->getPhysicalDevice()))
-		return nullptr;
-
-	auto img = device->createImage(std::move(params));
+	auto img = device->createImage({ std::move(params), { {.externalHandleTypes = CCUDADevice::EXTERNAL_MEMORY_HANDLE_TYPE } }, IGPUImage::TILING::LINEAR });
 
 	if (exportAsMemory(device, img.get()))
 		return img;

src/nbl/video/CVulkanCommandBuffer.cpp

@@ -48,25 +48,41 @@ void fill(vk_barrier_t& out, const ResourceBarrier& in, uint32_t selfQueueFamily
     // https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#VUID-VkBufferMemoryBarrier2-buffer-04088
     if (concurrentSharing)
         selfQueueFamilyIndex = IQueue::FamilyIgnored;
+
+    auto mapQFIdx = [](uint32_t idx)
+    {
+        switch (idx)
+        {
+        case IQueue::FamilyExternal:
+        case IQueue::FamilyIgnored:
+        case IQueue::FamilyForeign:
+            idx |= 1u << 31;
+            break;
+        }
+        return idx;
+    };
+
     if constexpr (!std::is_same_v<vk_barrier_t,VkMemoryBarrier2>)
     {
-        out.srcQueueFamilyIndex = selfQueueFamilyIndex;
-        out.dstQueueFamilyIndex = selfQueueFamilyIndex;
+        out.srcQueueFamilyIndex = mapQFIdx(selfQueueFamilyIndex);
+        out.dstQueueFamilyIndex = mapQFIdx(selfQueueFamilyIndex);
     }
     const asset::SMemoryBarrier* memoryBarrier;
     if constexpr (std::is_same_v<IGPUCommandBuffer::SOwnershipTransferBarrier,ResourceBarrier>)
     {
         memoryBarrier = &in.dep;
         // in.otherQueueFamilyIndex==selfQueueFamilyIndex not resulting in ownership transfer is implicit
-        if (!concurrentSharing && in.otherQueueFamilyIndex!=IQueue::FamilyIgnored)
-        switch (in.ownershipOp)
+        if (!concurrentSharing && in.otherQueueFamilyIndex != IQueue::FamilyIgnored)
         {
+            switch (in.ownershipOp)
+            {
             case IGPUCommandBuffer::SOwnershipTransferBarrier::OWNERSHIP_OP::RELEASE:
-                out.dstQueueFamilyIndex = in.otherQueueFamilyIndex;
+                out.dstQueueFamilyIndex = mapQFIdx(in.otherQueueFamilyIndex);
                 break;
             case IGPUCommandBuffer::SOwnershipTransferBarrier::OWNERSHIP_OP::ACQUIRE:
-                out.srcQueueFamilyIndex = in.otherQueueFamilyIndex;
+                out.srcQueueFamilyIndex = mapQFIdx(in.otherQueueFamilyIndex);
                 break;
+            }
         }
     }
     else

src/nbl/video/CVulkanImage.cpp

@@ -9,7 +9,6 @@ namespace nbl::video
 CVulkanImage::~CVulkanImage()
 {
     preDestroyStep();
-    // don't destroy imported handles
     if (!m_cachedCreationParams.skipHandleDestroy)
     {
         const CVulkanLogicalDevice* vulkanDevice = static_cast<const CVulkanLogicalDevice*>(getOriginDevice());