Hello,
I am working on a research project that uses Vulkan for GPU accelerated computation. I encountered a very weird issue where no validation layer output was being printed, but trying to use `vk::raii::Device::createComputePipeline` would crash in the Nvidia driver nvoglv64.dll. Interestingly enough, the same application would not crash under Intel or AMD systems. This seems to be the exact same problem as briefly discussed in Crash deep in nvoglv64.dll when calling vkCreateComputePipelines .
I have now had time to sit down and isolate the problem. It seems that if the first binding is not set in the SPIR-V module, even if unused, results in this crash. Admittedly, such a situation seems rather odd, but as we can see, it does occur. It also took quite some effort to figure this out, seeing as the crash is in a driver where we have no symbols and there are no validation errors.
What follows is my minimal example to recreate this:
CMakeLists.txt
cmake_minimum_required(VERSION 3.29)
project(broken-spirv)
set(CMAKE_CXX_STANDARD 20)
find_package(Vulkan REQUIRED COMPONENTS SPIRV-Tools)
add_executable(broken-spirv main.cpp)
target_link_libraries(broken-spirv PRIVATE Vulkan::Vulkan)
set(shader_source ${CMAKE_SOURCE_DIR}/shader.txt)
set(shader_out ${CMAKE_BINARY_DIR}/shader.spv)
set(Vulkan_SPIRV_AS_EXECUTABLE ${Vulkan_GLSLC_EXECUTABLE}/../spirv-as)
set(Vulkan_SPIRV_VAL_EXECUTABLE ${Vulkan_GLSLC_EXECUTABLE}/../spirv-val)
add_custom_command(OUTPUT ${shader_out}
COMMAND "${Vulkan_SPIRV_AS_EXECUTABLE}"
-o "${shader_out}"
--target-env vulkan1.1
${shader_source}
COMMENT "Compiling shader"
DEPENDS ${shader_source}
)
add_custom_target(compile-shader
DEPENDS ${shader_out}
)
add_custom_command(TARGET compile-shader
POST_BUILD
COMMAND "${Vulkan_SPIRV_VAL_EXECUTABLE}"
${shader_out}
--target-env vulkan1.1
DEPENDS ${shader_out}
)
add_dependencies(broken-spirv compile-shader)
main.cpp
#include <iostream>
#include <fstream>
#define VULKAN_HPP_NO_STRUCT_CONSTRUCTORS
#include <vulkan/vulkan_raii.hpp>
int main() {
const std::vector<char const*> enabledLayers = {
"VK_LAYER_KHRONOS_validation"
};
constexpr vk::ApplicationInfo appInfo{.pApplicationName = "Broken SPIRV",
.applicationVersion = VK_MAKE_VERSION( 1, 0, 0 ),
.pEngineName = "No Engine",
.engineVersion = VK_MAKE_VERSION( 1, 0, 0 ),
.apiVersion = vk::ApiVersion11};
vk::InstanceCreateInfo instanceCreateInfo{
.pApplicationInfo = &appInfo,
.enabledLayerCount = static_cast<uint32_t>(enabledLayers.size()),
.ppEnabledLayerNames = enabledLayers.data(),
};
vk::raii::Context context;
auto instance = vk::raii::Instance(context, instanceCreateInfo);
vk::raii::PhysicalDevice physicalDevice = instance.enumeratePhysicalDevices()[0];
vk::PhysicalDeviceProperties props = physicalDevice.getProperties2().properties;
std::cout << "Vendor id: 0x" << std::hex << props.vendorID << std::dec << std::endl;
std::cout << "Device name: " << props.deviceName << std::endl;
std::vector<vk::QueueFamilyProperties> queueFamilyProperties = physicalDevice.getQueueFamilyProperties();
auto computeQueueFamilyProperty = std::ranges::find_if(queueFamilyProperties, [](auto const &qfp) { return (qfp.queueFlags & vk::QueueFlagBits::eCompute) != static_cast<vk::QueueFlags>(0); });
auto computeIndex = static_cast<uint32_t>(std::distance(queueFamilyProperties.begin(), computeQueueFamilyProperty));
float queuePriority = 0.5f;
vk::DeviceQueueCreateInfo deviceQueueCreateInfo { .queueFamilyIndex = computeIndex, .queueCount = 1, .pQueuePriorities = &queuePriority };
vk::StructureChain<vk::PhysicalDeviceFeatures2, vk::PhysicalDeviceVulkan11Features> featureChain = {
{.features = {
.shaderInt64 = true
}},
{.variablePointersStorageBuffer = true}
};
std::vector<const char*> requiredDeviceExtension;
vk::DeviceCreateInfo deviceCreateInfo{
.pNext = &featureChain.get<vk::PhysicalDeviceFeatures2>(),
.queueCreateInfoCount = 1,
.pQueueCreateInfos = &deviceQueueCreateInfo,
.enabledExtensionCount = static_cast<uint32_t>(requiredDeviceExtension.size()),
.ppEnabledExtensionNames = requiredDeviceExtension.data()
};
vk::raii::Device device = vk::raii::Device( physicalDevice, deviceCreateInfo );
vk::raii::Queue computeQueue = vk::raii::Queue( device, computeIndex, 0 );
std::ifstream file("shader.spv", std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("failed to open file!");
}
std::vector<char> shaderCode(file.tellg());
file.seekg(0, std::ios::beg);
file.read(shaderCode.data(), static_cast<std::streamsize>(shaderCode.size()));
file.close();
vk::ShaderModuleCreateInfo shaderCreateInfo{ .codeSize = shaderCode.size() * sizeof(char), .pCode = reinterpret_cast<const uint32_t*>(shaderCode.data())};
vk::raii::ShaderModule shaderModule{ device, shaderCreateInfo };
uint32_t numInvocs = 4;
vk::SpecializationMapEntry specializationEntry{ .constantID = 100, .offset = 0, .size = 4 };
vk::SpecializationInfo specializationInfo { .mapEntryCount = 1, .pMapEntries = &specializationEntry, .dataSize = 4, .pData = &numInvocs};
vk::PipelineShaderStageCreateInfo computeShaderStageInfo{
.stage = vk::ShaderStageFlagBits::eCompute,
.module = shaderModule,
.pName = "main",
.pSpecializationInfo = &specializationInfo
};
std::vector<vk::DescriptorSetLayoutBinding> layoutBindings{
{ .binding = 1, .descriptorType = vk::DescriptorType::eStorageBuffer, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eCompute }, // data
// ---------------- Uncomment this to fix crash --------------
// { .binding = 0, .descriptorType = vk::DescriptorType::eStorageBuffer, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eCompute } // accessor
};
vk::DescriptorSetLayoutCreateInfo layoutInfo{ .bindingCount = static_cast<uint32_t>(layoutBindings.size()), .pBindings = layoutBindings.data() };
vk::raii::DescriptorSetLayout descriptorSetLayout{ device, layoutInfo };
uint64_t bufferSize = sizeof(uint32_t) * 4;
vk::BufferCreateInfo bufferCreateInfo{ .size = bufferSize, .usage = vk::BufferUsageFlagBits::eStorageBuffer, .sharingMode = vk::SharingMode::eExclusive };
vk::raii::Buffer buffer { device, bufferCreateInfo };
bufferCreateInfo.size = 8;
vk::raii::Buffer accessorBuffer { device, bufferCreateInfo };
vk::MemoryRequirements memRequirements = buffer.getMemoryRequirements();
vk::PhysicalDeviceMemoryProperties memProperties = physicalDevice.getMemoryProperties();
vk::MemoryPropertyFlags wantedMemoryProperties = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent;
uint32_t foundMemType = -1;
for (uint32_t i = 0; i < memProperties.memoryTypeCount; ++i) {
if ((memRequirements.memoryTypeBits & (1 << i)) && (memProperties.memoryTypes[i].propertyFlags & wantedMemoryProperties) == wantedMemoryProperties) {
foundMemType = i;
}
}
vk::MemoryAllocateInfo memoryAllocateInfo { .allocationSize = 128, .memoryTypeIndex = foundMemType };
vk::raii::DeviceMemory deviceMemory{ device, memoryAllocateInfo };
buffer.bindMemory( *deviceMemory, 0 );
accessorBuffer.bindMemory(*deviceMemory, 64);
vk::DescriptorPoolSize descriptorPoolSize{ .type = vk::DescriptorType::eStorageBuffer, .descriptorCount = 2 };
vk::DescriptorPoolCreateInfo descriptPoolInfo{ .flags = vk::DescriptorPoolCreateFlagBits::eFreeDescriptorSet, .maxSets = 1, .poolSizeCount = 1, .pPoolSizes = &descriptorPoolSize };
vk::raii::DescriptorPool descriptorPool{ device, descriptPoolInfo };
vk::DescriptorSetAllocateInfo descriptorSetAllocInfo{ .descriptorPool = descriptorPool, .descriptorSetCount = 1, .pSetLayouts = &*descriptorSetLayout };
std::vector<vk::raii::DescriptorSet> descriptorSets = device.allocateDescriptorSets(descriptorSetAllocInfo);
std::vector<vk::DescriptorBufferInfo> descriptorBufferInfos{
{ .buffer = buffer, .offset = 0, .range = bufferSize },
{ .buffer = accessorBuffer, .offset = 0, .range = 8 }
};
std::vector<vk::WriteDescriptorSet> descriptorWrites {
{ .dstSet = descriptorSets[0], .dstBinding = 1, .dstArrayElement = 0, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eStorageBuffer, .pBufferInfo = &descriptorBufferInfos[0] },
// ---------------- Uncomment this to fix crash --------------
// { .dstSet = descriptorSets[0], .dstBinding = 0, .dstArrayElement = 0, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eStorageBuffer, .pBufferInfo = &descriptorBufferInfos[1] }
};
device.updateDescriptorSets(descriptorWrites, {});
vk::PipelineLayoutCreateInfo pipelineLayoutInfo{ .setLayoutCount = 1, .pSetLayouts = &*descriptorSetLayout };
vk::raii::PipelineLayout pipelineLayout{device, pipelineLayoutInfo};
vk::ComputePipelineCreateInfo pipelineInfo{.stage = computeShaderStageInfo, .layout = pipelineLayout};
vk::raii::Pipeline pipeline = device.createComputePipeline(nullptr, pipelineInfo);
vk::CommandPoolCreateInfo poolInfo{.queueFamilyIndex = computeIndex};
vk::raii::CommandPool commandPool{device, poolInfo};
vk::CommandBufferAllocateInfo allocInfo{ .commandPool = commandPool, .level = vk::CommandBufferLevel::ePrimary, .commandBufferCount = 1 };
vk::raii::CommandBuffer commandBuffer = std::move(vk::raii::CommandBuffers(device, allocInfo).front());
commandBuffer.begin(vk::CommandBufferBeginInfo{});
commandBuffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline);
commandBuffer.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipelineLayout, 0, *descriptorSets[0], nullptr);
commandBuffer.dispatch(4, 1, 1);
commandBuffer.end();
vk::raii::Fence fence{device, vk::FenceCreateInfo{}};
const vk::SubmitInfo submitInfo{ .commandBufferCount = 1, .pCommandBuffers = &*commandBuffer};
computeQueue.submit(submitInfo, fence);
device.waitForFences(*fence, vk::True, UINT64_MAX);
auto* data = reinterpret_cast<uint32_t*>(deviceMemory.mapMemory(0, bufferSize));
std::cout << "Data:" << std::endl;
for (int i = 0; i < 4; ++i) {
std::cout << data[i] << " ";
}
deviceMemory.unmapMemory();
return 0;
}
shader.txt
OpCapability Shader
OpCapability VariablePointersStorageBuffer
OpExtension "SPV_KHR_variable_pointers"
OpMemoryModel Logical GLSL450
OpEntryPoint GLCompute %main "main" %BuiltInGlobalInvocationId
OpDecorate %wg_x SpecId 100
OpDecorate %wg_y SpecId 101
OpDecorate %wg_z SpecId 102
OpDecorate %struct_runtimearr_uint Block
OpDecorate %_runtimearr_uint ArrayStride 4
OpDecorate %gl_WorkGroupSize BuiltIn WorkgroupSize
OpDecorate %BuiltInGlobalInvocationId BuiltIn GlobalInvocationId
OpDecorate %_arg_0 Binding 0
OpDecorate %_arg_1 Binding 1
OpDecorate %_arg_0 DescriptorSet 0
OpDecorate %_arg_1 DescriptorSet 0
OpMemberDecorate %struct_runtimearr_uint 0 Offset 0
%void = OpTypeVoid
%void_fn = OpTypeFunction %void
%uint = OpTypeInt 32 0
%uint_0 = OpConstant %uint 0
%_runtimearr_uint = OpTypeRuntimeArray %uint
%struct_runtimearr_uint = OpTypeStruct %_runtimearr_uint
%_ptr_StorageBuffer_struct_runtimearr_uint = OpTypePointer StorageBuffer %struct_runtimearr_uint
%_ptr_StorageBuffer_uint = OpTypePointer StorageBuffer %uint
%v3uint = OpTypeVector %uint 3
%_ptr_Input_v3uint = OpTypePointer Input %v3uint
%wg_x = OpSpecConstant %uint 1
%wg_y = OpSpecConstant %uint 1
%wg_z = OpSpecConstant %uint 1
%BuiltInGlobalInvocationId = OpVariable %_ptr_Input_v3uint Input
%_arg_0 = OpVariable %_ptr_StorageBuffer_struct_runtimearr_uint StorageBuffer
%_arg_1 = OpVariable %_ptr_StorageBuffer_struct_runtimearr_uint StorageBuffer
%gl_WorkGroupSize = OpSpecConstantComposite %v3uint %wg_x %wg_y %wg_z
%main = OpFunction %void None %void_fn
%entry = OpLabel
%InvocationId = OpLoad %v3uint %BuiltInGlobalInvocationId None
%InvocationX = OpCompositeExtract %uint %InvocationId 0
%ptr = OpInBoundsAccessChain %_ptr_StorageBuffer_uint %_arg_1 %uint_0 %InvocationX
OpStore %ptr %InvocationX Aligned 4
OpReturn
OpFunctionEnd
I have tested this with the Vulkan SDK 1.4.357.0 on a Windows 11 system.
It would be great to hear an official confirmation of this and potentially a fix (be it allowing this or throwing some kind of more informative diagnostic in such situations).