Hi! I’ve a real problem, I’m trying to use 3 threads, the one to register secondary command buffers for my offscreen rendering, another to submit primary command buffers for my offscreen rendering, and the main thread to render my offscreen rendering to the window. The goal is to submit/register commands for next frames while processing the current frame to optimize my game engine. The problem is it seems the driver is updating my descriptors sets while my secondary command buffer is executed despite my synchronisation primitive.
void PerPixelLinkedListRenderComponent::executeThreadPools() {
executeThreadPoolRunning = true;
while (executeThreadPoolRunning) {
if (isVisible()) {
if (datasReady.load()) {
datasReady = false;
std::lock_guard<std::recursive_mutex> lock(rec_mutex);
m_instances = batcher.getInstances();
m_normals = normalBatcher.getInstances();
m_instancesIndexed = batcherIndexed.getInstances();
m_normalsIndexed = normalBatcherIndexed.getInstances();
m_selected = selectedBatcher.getInstances();
m_selectedScale = selectedScaleBatcher.getInstances();
m_selectedIndexed = selectedIndexBatcher.getInstances();
m_selectedScaleIndexed = selectedIndexScaleBatcher.getInstances();
m_selectedInstance = selectedInstanceBatcher.getInstances();
m_selectedScaleInstance = selectedInstanceScaleBatcher.getInstances();
m_selectedInstanceIndexed = selectedInstanceIndexBatcher.getInstances();
m_selectedScaleInstanceIndexed = selectedInstanceIndexScaleBatcher.getInstances();
m_skyboxInstance = skyboxBatcher.getInstances();
}
std::unique_lock<std::mutex> lock(mtx[0]);
//std::cout<<"mutex locked register thread pool,"<<threadpoolCurrentFrame<<","<<std::this_thread::get_id()<<std::endl;
cv[0].wait(lock, [this]{ return offscreenCommandBufferReady[threadpoolCurrentFrame].load() || stop.load();});
offscreenCommandBufferReady[threadpoolCurrentFrame] = false;
VkSemaphoreWaitInfo waitInfo{};
uint64_t waitValue = values2[threadpoolCurrentFrame];
waitInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO;
waitInfo.semaphoreCount = 1;
waitInfo.pSemaphores = &offscreenFinishedSemaphore[threadpoolCurrentFrame];
waitInfo.pValues = &waitValue;
vkWaitSemaphores(vkDevice.getDevice(), &waitInfo, UINT64_MAX);
//std::cout<<"wait register pools of frame : "<<threadpoolCurrentFrame<<","<<getLayer()<<std::endl;
std::cout<<"register pools of frame : "<<threadpoolCurrentFrame<<","<<getLayer()<<std::endl;
//std::cout<<"unlock mutex register thread pool : "<<threadpoolCurrentFrame<<","<<std::this_thread::get_id()<<std::endl;
if (!stop.load()) {
RenderStates currentStates;
math::Matrix4f projMatrix = view.getProjMatrix().getMatrix()/*.transpose()*/;
math::Matrix4f viewMatrix = view.getViewMatrix().getMatrix()/*.transpose()*/;
indirectDrawPushConsts.projMatrix = toVulkanMatrix(projMatrix);
indirectDrawPushConsts.viewMatrix = toVulkanMatrix(viewMatrix);
//projMatrix.identity();
skyboxPushConsts.projMatrix = toVulkanMatrix(projMatrix);
skyboxPushConsts.viewMatrix = toVulkanMatrix(viewMatrix);
resetBuffers();
fillBuffersMT();
fillIndexedBuffersMT();
fillSelectedBuffersMT();
fillSelectedIndexedBuffersMT();
fillOutlineBuffersMT();
fillOutlineIndexedBuffersMT();
////std::cout<<"buffer filled"<<std::endl;
skyboxVB.clear();
for (unsigned int i = 0; i < m_skyboxInstance.size(); i++) {
if (m_skyboxInstance[i].getAllVertices().getVertexCount() > 0) {
skyboxVB.setPrimitiveType(m_skyboxInstance[i].getAllVertices().getPrimitiveType());
for (unsigned int j = 0; j < m_skyboxInstance[i].getAllVertices().getVertexCount(); j++) {
skyboxVB.append(m_skyboxInstance[i].getAllVertices()[j]);
}
}
}
unsigned int currentFrame = threadpoolCurrentFrame;
skyboxVB.updateBuffers(currentFrame);
vb.clear();
vb.setPrimitiveType(Triangles);
Vertex v1 (math::Vec3f(0, 0, quad.getSize().z()));
Vertex v2 (math::Vec3f(quad.getSize().x(),0, quad.getSize().z()));
Vertex v3 (math::Vec3f(quad.getSize().x(), quad.getSize().y(), quad.getSize().z()));
Vertex v4 (math::Vec3f(0, quad.getSize().y(), quad.getSize().z()));
vb.append(v1);
vb.append(v2);
vb.append(v3);
vb.append(v1);
vb.append(v3);
vb.append(v4);
vb.updateBuffers(currentFrame);
math::Matrix4f matrix = quad.getTransform().getMatrix();
//////std::cout<<"world mat : "<<matrix<<std::endl;
ppll2PushConsts.worldMat = toVulkanMatrix(matrix);
for (unsigned int p = 0; p < Batcher::nbPrimitiveTypes; p++) {
if (vbBindlessTex[p].getVertexCount() > 0)
vbBindlessTex[p].updateBuffers(currentFrame);
if (vbBindlessTexIndexed[p].getVertexCount() > 0)
vbBindlessTexIndexed[p].updateBuffers(currentFrame);
}
jobFence[currentFrame].reset(numThreads);
threadPool.enqueue([this, currentFrame, currentStates]{
VkCommandBufferInheritanceInfo inheritanceInfo{};
inheritanceInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
inheritanceInfo.renderPass = VK_NULL_HANDLE; // pas de render pass
inheritanceInfo.subpass = 0;
inheritanceInfo.framebuffer = VK_NULL_HANDLE;
inheritanceInfo.occlusionQueryEnable = VK_FALSE;
inheritanceInfo.queryFlags = 0;
inheritanceInfo.pipelineStatistics = 0;
VkCommandBufferBeginInfo beginInfo{};
beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
beginInfo.pInheritanceInfo = &inheritanceInfo; // obligatoire pour secondaire
vkResetCommandBuffer(copyModelDataBufferCommandBuffer[currentFrame], 0);
vkResetCommandBuffer(copyMaterialDataBufferCommandBuffer[currentFrame], 0);
vkResetCommandBuffer(copyDrawBufferCommandBuffer[currentFrame], 0);
vkResetCommandBuffer(copyVbBufferCommandBuffer[currentFrame], 0);
vkResetCommandBuffer(copyDrawIndexedBufferCommandBuffer[currentFrame], 0);
vkResetCommandBuffer(copyVbIndexedBufferCommandBuffer[currentFrame], 0);
if (vkBeginCommandBuffer(copyModelDataBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
if (vkBeginCommandBuffer(copyMaterialDataBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
if (vkBeginCommandBuffer(copyDrawBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
if (vkBeginCommandBuffer(copyVbBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
if (vkBeginCommandBuffer(copyDrawIndexedBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
if (vkBeginCommandBuffer(copyVbIndexedBufferCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
for (unsigned int p = 0; p < Batcher::nbPrimitiveTypes; p++) {
VkDeviceSize bufferSize = sizeof(ModelData) * modelDatas[p].size();
if (bufferSize > 0)
copyBuffer(modelDataStagingBufferMT[p][currentFrame], modelDataBufferMT[p][currentFrame], bufferSize, copyModelDataBufferCommandBuffer[currentFrame]);
bufferSize = sizeof(MaterialData) * materialDatas[p].size();
if (bufferSize > 0)
copyBuffer(materialDataStagingBufferMT[p][currentFrame],materialDataBufferMT[p][currentFrame], bufferSize, copyMaterialDataBufferCommandBuffer[currentFrame]);
bufferSize = sizeof(DrawArraysIndirectCommand) * drawArraysIndirectCommands[p].size();
if (bufferSize > 0)
copyBuffer(vboIndirectStagingBufferMT[p][currentFrame], drawCommandBufferMT[p][currentFrame], totalBufferSizeDrawCommand[p], copyDrawBufferCommandBuffer[currentFrame]);
bufferSize = sizeof(DrawElementsIndirectCommand) * drawElementsIndirectCommands[p].size();
if (bufferSize > 0)
copyBuffer(vboIndexedIndirectStagingBufferMT[p][currentFrame], drawCommandBufferIndexedMT[p][currentFrame], bufferSize, copyDrawIndexedBufferCommandBuffer[currentFrame]);
if (vbBindlessTex[p].getVertexCount() > 0)
vbBindlessTex[p].registerCopyCmdBuffers(currentFrame, copyVbBufferCommandBuffer[currentFrame]);
if (vbBindlessTexIndexed[p].getVertexCount() > 0)
vbBindlessTexIndexed[p].registerCopyCmdBuffers(currentFrame, copyVbIndexedBufferCommandBuffer[currentFrame]);
}
if (vkEndCommandBuffer(copyModelDataBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (vkEndCommandBuffer(copyMaterialDataBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (vkEndCommandBuffer(copyDrawBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (vkEndCommandBuffer(copyVbBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (vkEndCommandBuffer(copyDrawIndexedBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (vkEndCommandBuffer(copyVbIndexedBufferCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
if (skybox != nullptr) {
inheritanceInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
inheritanceInfo.renderPass = VK_NULL_HANDLE; // pas de render pass
inheritanceInfo.subpass = 0;
inheritanceInfo.framebuffer = VK_NULL_HANDLE;
inheritanceInfo.occlusionQueryEnable = VK_FALSE;
inheritanceInfo.queryFlags = 0;
inheritanceInfo.pipelineStatistics = 0;
beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
beginInfo.pInheritanceInfo = &inheritanceInfo; // obligatoire pour secondaire
vkResetCommandBuffer(copySkyboxCommandBuffer[currentFrame], 0);
if (vkBeginCommandBuffer(copySkyboxCommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
//std::cout<<"copy skybox vb"<<std::endl;
skyboxVB.registerCopyCmdBuffers(currentFrame, copySkyboxCommandBuffer[currentFrame]);
if (vkEndCommandBuffer(copySkyboxCommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
}
inheritanceInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
inheritanceInfo.renderPass = VK_NULL_HANDLE; // pas de render pass
inheritanceInfo.subpass = 0;
inheritanceInfo.framebuffer = VK_NULL_HANDLE;
inheritanceInfo.occlusionQueryEnable = VK_FALSE;
inheritanceInfo.queryFlags = 0;
inheritanceInfo.pipelineStatistics = 0;
beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
beginInfo.pInheritanceInfo = &inheritanceInfo; // obligatoire pour secondaire
vkResetCommandBuffer(copyVbPpllPass2CommandBuffer[currentFrame], 0);
if (vkBeginCommandBuffer(copyVbPpllPass2CommandBuffer[currentFrame], &beginInfo) != VK_SUCCESS) {
throw core::Erreur(0, "failed to begin recording command buffer!", 1);
}
vb.registerCopyCmdBuffers(currentFrame, copyVbPpllPass2CommandBuffer[currentFrame]);
if (vkEndCommandBuffer(copyVbPpllPass2CommandBuffer[currentFrame]) != VK_SUCCESS) {
throw core::Erreur(0, "failed to record command buffer!", 1);
}
jobFence[currentFrame].jobDone();
});
drawBuffers();
//std::cout<<"wait : "<<threadpoolCurrentFrame<<","<<getLayer()<<std::endl;
threadpoolCurrentFrame = (threadpoolCurrentFrame + 1) % MAX_FRAMES_IN_FLIGHT;
//std::cout<<"unlock mutex register thread pool,"<<threadpoolCurrentFrame<<","<<getLayer()<<std::endl;
}
}
}
}
I’m trying to wait on my timeline semaphore before, updating everything for the same frame in flight. I submit my secondary command buffers here :
void PerPixelLinkedListRenderComponent::drawNextFrame() {
////std::cout<<"next frame datasReady"<<datasReady<<std::endl;
/*math::Matrix4f viewMatrix = view.getViewMatrix().getMatrix().transpose();
math::Matrix4f projMatrix = view.getProjMatrix().getMatrix().transpose();
viewMatrix = math::Matrix4f(math::Matrix3f(viewMatrix));
vb.clear();
//vb.name = "SKYBOXVB";
for (unsigned int i = 0; i < m_skyboxInstance.size(); i++) {
if (m_skyboxInstance[i].getAllVertices().getVertexCount() > 0) {
vb.setPrimitiveType(m_skyboxInstance[i].getAllVertices().getPrimitiveType());
for (unsigned int j = 0; j < m_skyboxInstance[i].getAllVertices().getVertexCount(); j++) {
//if (m_skyboxInstance[i].getAllVertices()[j].position.x != 0 && m_skyboxInstance[i].getAllVertices()[j].position.y != 0 && m_skyboxInstance[i].getAllVertices()[j].position.z != 0);
vb.append(m_skyboxInstance[i].getAllVertices()[j]);
}
}
}
currentStates.blendMode = sf::BlendAlpha;
currentStates.shader = &skyboxShader;
currentStates.texture = (skybox == nullptr ) ? nullptr : &static_cast<g3d::Skybox*>(skybox)->getTexture();
vb.update();
frameBuffer.drawVertexBuffer(vb, currentStates);
vb.clear();*/
//indirectDrawPushConsts.projMatrix.m22 *= -1;
if (isVisible()) {
if (useThread) {
jobFence[frameBuffer.getCurrentFrame()].wait();
std::cout<<"draw offscreen frame : "<<frameBuffer.getCurrentFrame()<<","<<getLayer()<<std::endl;
//std::cout<<"mutex locked thread next frame,"<<currentFrame<<","<<std::this_thread::get_id()<<std::endl;
std::unique_lock<std::mutex> lock(mtx[0]);
unsigned int currentFrame = frameBuffer.getCurrentFrame();
cv[0].wait(lock, [this, currentFrame](){return registerFrameJob[currentFrame].load() || stop.load();});
registerFrameJob[currentFrame] = false;
//std::cout<<"unlock mutex thread next frame,"<<currentFrame<<","<<std::this_thread::get_id()<<std::endl;
//std::unique_lock<std::mutex> lock2(mtx2);
if (!stop.load()) {
//std::cout<<"register frame : "<<frameBuffer.getCurrentFrame()<<std::endl;
frameBuffer.clear(Color::Transparent);
VkClearColorValue clearColor;
clearColor.uint32[0] = 0xffffffff;
VkImageSubresourceRange subresRange = {};
subresRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
subresRange.levelCount = 1;
subresRange.layerCount = 1;
vkCmdClearColorImage(frameBuffer.getCommandBuffers()[currentFrame], headPtrTextureImage[currentFrame], VK_IMAGE_LAYOUT_GENERAL, &clearColor, 1, &subresRange);
vkCmdFillBuffer(frameBuffer.getCommandBuffers()[currentFrame], counterShaderStorageBuffers[currentFrame], 0, sizeof(uint32_t), 0);
VkMemoryBarrier memoryBarrier0;
memoryBarrier0.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
memoryBarrier0.pNext = VK_NULL_HANDLE;
memoryBarrier0.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
memoryBarrier0.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
vkCmdPipelineBarrier(frameBuffer.getCommandBuffers()[currentFrame], VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 1, &memoryBarrier0, 0, nullptr, 0, nullptr);
frameBuffer.beginRecordCommandBuffers();
std::vector<VkCommandBuffer> commandBuffers = frameBuffer.getCommandBuffers();
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©ModelDataBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©MaterialDataBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©DrawBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©VbBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©DrawIndexedBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©VbIndexedBufferCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©VbPpllPass2CommandBuffer[currentFrame]);
if (skybox != nullptr)
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, ©SkyboxCommandBuffer[currentFrame]);
std::vector<VkSemaphore> signalSemaphores;
signalSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
std::vector<VkSemaphore> waitSemaphores;
waitSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
std::vector<VkPipelineStageFlags> waitStages;
waitStages.push_back(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
std::vector<uint64_t> signalValues;
std::vector<uint64_t> waitValues;
waitValues.push_back(values[currentFrame]);
values[currentFrame]++;
signalValues.push_back(values[currentFrame]);
std::vector<VkFence> windowInFlightFence = {windowFences[frameBuffer.getCurrentFrame()]};
//std::cout<<"wait on fence : "<<windowFences[frameBuffer.getCurrentFrame()]<<std::endl;
//std::cout<<"submit fence : "<<fences[frameBuffer.getCurrentFrame()]<<std::endl;
frameBuffer.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, windowInFlightFence, getLayer()+2);
frameBuffer.beginRecordCommandBuffers();
//std::cout<<"reseted : "<<frameBuffer.getCurrentFrame()<<","<<stop.load()<<std::endl;
//std::cout<<"skybox"<<std::endl;
VkBufferMemoryBarrier bufferMemoryBarrier{};
bufferMemoryBarrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
bufferMemoryBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
bufferMemoryBarrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
bufferMemoryBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
bufferMemoryBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
bufferMemoryBarrier.offset = 0;
bufferMemoryBarrier.size = VK_WHOLE_SIZE;
bufferMemoryBarrier.buffer = vb.getVertexBuffer(currentFrame);
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &bufferMemoryBarrier,
0, nullptr
);
if (skybox != nullptr) {
bufferMemoryBarrier.buffer = skyboxVB.getVertexBuffer(currentFrame);
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &bufferMemoryBarrier,
0, nullptr
);
}
for (unsigned int p = 0; p < Batcher::nbPrimitiveTypes; p++) {
VkBufferMemoryBarrier buffersMemoryBarrier{};
buffersMemoryBarrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
buffersMemoryBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
buffersMemoryBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
buffersMemoryBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
buffersMemoryBarrier.offset = 0;
buffersMemoryBarrier.size = VK_WHOLE_SIZE;
if (vbBindlessTex[p].getVertexBuffer(currentFrame) != nullptr) {
buffersMemoryBarrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
buffersMemoryBarrier.buffer = vbBindlessTex[p].getVertexBuffer(currentFrame);
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
if (vbBindlessTexIndexed[p].getVertexBuffer(currentFrame) != nullptr && vbBindlessTexIndexed[p].getIndexBuffer(currentFrame) != nullptr) {
buffersMemoryBarrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
buffersMemoryBarrier.buffer = vbBindlessTexIndexed[p].getVertexBuffer(currentFrame);
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
buffersMemoryBarrier.dstAccessMask = VK_ACCESS_INDEX_READ_BIT;
buffersMemoryBarrier.buffer = vbBindlessTexIndexed[p].getIndexBuffer(currentFrame);
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
buffersMemoryBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
if (modelDataBufferMT[p][currentFrame] != nullptr) {
buffersMemoryBarrier.buffer = modelDataBufferMT[p][currentFrame];
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
if (materialDataBufferMT[p][currentFrame] != nullptr) {
buffersMemoryBarrier.buffer = materialDataBufferMT[p][currentFrame];
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
buffersMemoryBarrier.dstAccessMask = VK_ACCESS_INDIRECT_COMMAND_READ_BIT;
if (drawCommandBufferMT[p][currentFrame] != nullptr) {
buffersMemoryBarrier.buffer = drawCommandBufferMT[p][currentFrame];
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
if (drawCommandBufferIndexedMT[p][currentFrame] != nullptr) {
buffersMemoryBarrier.buffer = drawCommandBufferIndexedMT[p][currentFrame];
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
0,
0, nullptr,
1, &buffersMemoryBarrier,
0, nullptr
);
}
}
if (skybox != nullptr) {
frameBuffer.beginRecordCommandBuffers();
frameBuffer.beginRenderPass();
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, &skyboxCommandBuffer[currentFrame]);
frameBuffer.endRenderPass();
signalSemaphores.clear();
signalSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
signalValues.clear();
waitSemaphores.clear();
waitSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
waitStages.clear();
waitStages.push_back(VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
waitValues.clear();
waitValues.push_back(values[currentFrame]);
values[currentFrame]++;
signalValues.push_back(values[currentFrame]);
frameBuffer.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, std::vector<VkFence>(), getLayer()+2);
}
//std::cout<<"copies ok"<<std::endl;
frameBuffer.beginRecordCommandBuffers();
frameBuffer.beginRenderPass();
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, &ppllCommandBuffer[currentFrame]);
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, &ppllSelectedCommandBuffer[currentFrame]);
frameBuffer.endRenderPass();
signalSemaphores.clear();
signalSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
signalValues.clear();
waitSemaphores.clear();
waitSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
waitStages.clear();
waitStages.push_back(VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
waitValues.clear();
waitValues.push_back(values[currentFrame]);
values[currentFrame]++;
signalValues.push_back(values[currentFrame]);
//std::cout<<"compute"<<std::endl;
if (visibleEntities.size() > computeSemaphores.size()) {
computeSemaphores.resize(visibleEntities.size());
computeFences.resize(visibleEntities.size());
for (unsigned int i = 0; i < visibleEntities.size(); i++) {
for (unsigned int j = 0; j < MAX_FRAMES_IN_FLIGHT; j++) {
VkSemaphoreCreateInfo semaphoreInfo{};
semaphoreInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
VkFenceCreateInfo fenceInfo{};
fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
if (computeFences[i][j] == nullptr) {
if (vkCreateFence(vkDevice.getDevice(), &fenceInfo, nullptr, &computeFences[i][j]) != VK_SUCCESS) {
throw core::Erreur(0, "échec de la création des objets de synchronisation pour une entité!", 1);
}
}
if(computeSemaphores[i][j] == nullptr) {
if(vkCreateSemaphore(vkDevice.getDevice(), &semaphoreInfo, nullptr, &computeSemaphores[i][j]) != VK_SUCCESS) {
throw core::Erreur(0, "échec de la création des objets de synchronisation pour une entité!", 1);
}
}
}
}
}
std::vector<VkFence> fencesToWait;
for (unsigned int i = 0; i < visibleEntities.size(); i++) {
if (visibleEntities[i] != nullptr && visibleEntities[i]->isLeaf() && (visibleEntities[i]->getRootType() == "E_PARTICLES"
|| visibleEntities[i]->getRootType() == "E_BONE_ANIMATION")) {
std::unique_lock<std::mutex> lock2(mtx2[0]);
std::array<std::mutex*, MAX_FRAMES_IN_FLIGHT> pmtx2;
std::array<std::condition_variable*, MAX_FRAMES_IN_FLIGHT> pcv2;
for (unsigned int i = 0; i < MAX_FRAMES_IN_FLIGHT; i++) {
pmtx2[i] = &mtx2[i];
pcv2[i] = &cv2[i];
}
visibleEntities[i]->getRootEntity()->computeParticles(pmtx2, pcv2, vbBindlessTex[Triangles], frameBuffer.getCurrentFrame(),visibleEntities[i]->getTransform(), (visibleEntities[i]->getDrawMode() == Entity::INSTANCED) ? true : false, computeSemaphores[i], computeFences[i], getLayer());
auto entity = visibleEntities[i]->getRootEntity();
//std::cout<<"wait : "<<visibleEntities[i]<<" current frame : "<<frameBuffer.getCurrentFrame()<<std::endl;
cv2[0].wait(lock2, [&, this, entity, currentFrame](){return entity->isComputeFinished(currentFrame, getLayer()) || (entity->isComputeFinished(currentFrame, getLayer()) && stop.load());});
//std::cout<<"wait finished"<<std::endl;
waitSemaphores.push_back(computeSemaphores[i][frameBuffer.getCurrentFrame()]);
waitValues.push_back(0);
waitStages.push_back(VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
//if (!firstFrame[frameBuffer.getCurrentFrame()]) {
if (vbBindlessTex[Triangles].getVertexBuffer(currentFrame) != nullptr) {
bufferMemoryBarrier.buffer = vbBindlessTex[Triangles].getVertexBuffer(currentFrame);
bufferMemoryBarrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
bufferMemoryBarrier.dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
vkCmdPipelineBarrier(
commandBuffers[currentFrame],
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
0,
0, nullptr,
1, &bufferMemoryBarrier,
0, nullptr
);
}
firstFrame[frameBuffer.getCurrentFrame()] = false;
fencesToWait.push_back(computeFences[i][currentFrame]);
//}
}
//std::cout<<"wait for fence"<<std::endl;
//std::cout<<"fence reseted"<<std::endl;
}
//std::cout<<"submit fence : "<<fences[frameBuffer.getCurrentFrame()]<<std::endl;
frameBuffer.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, fencesToWait, getLayer()+2);
//std::cout<<"ppll ok"<<std::endl;
//std::cout<<"draw"<<std::endl;
frameBuffer.beginRecordCommandBuffers();
frameBuffer.beginRenderPass();
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, &ppllOutlineCommandBuffer[currentFrame]);
frameBuffer.endRenderPass();
signalSemaphores.clear();
signalSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
signalValues.clear();
waitSemaphores.clear();
waitSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
waitStages.clear();
waitStages.push_back(VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
waitValues.clear();
waitValues.push_back(values[currentFrame]);
values[currentFrame]++;
signalValues.push_back(values[currentFrame]);
frameBuffer.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, std::vector<VkFence>(), getLayer()+2);
//std::cout<<"outline ok"<<std::endl;
frameBuffer.beginRecordCommandBuffers();
//vkCmdPipelineBarrier(commandBuffers[currentFrame], VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 0, nullptr);
VkMemoryBarrier memoryBarrier{};
memoryBarrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
memoryBarrier.pNext = VK_NULL_HANDLE;
memoryBarrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
memoryBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
vkCmdPipelineBarrier(commandBuffers[currentFrame], VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 1, &memoryBarrier, 0, nullptr, 0, nullptr);
frameBuffer.beginRenderPass();
vkCmdExecuteCommands(commandBuffers[currentFrame], 1, &ppllPass2CommandBuffer[currentFrame]);
frameBuffer.endRenderPass();
signalSemaphores.clear();
signalSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
signalValues.clear();
waitSemaphores.clear();
waitSemaphores.push_back(offscreenFinishedSemaphore[currentFrame]);
waitStages.clear();
waitStages.push_back(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
waitValues.clear();
waitValues.push_back(values[currentFrame]);
values[currentFrame]++;
signalValues.push_back(values[currentFrame]);
frameBuffer.submit(true, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, std::vector<VkFence>(), getLayer()+2, false);
values2[frameBuffer.getCurrentFrame()] = values[frameBuffer.getCurrentFrame()];
}
std::cout<<"offscreen drawn : "<<frameBuffer.getCurrentFrame()<<","<<getLayer()<<std::endl;
offscreenCommandBufferReady[windowCurrentFrame] = true;
commandBufferReady[windowCurrentFrame] = true;
frameBuffer.display();
cv[0].notify_all();
} else {
RenderStates currentStates;
math::Matrix4f projMatrix = view.getProjMatrix().getMatrix()/*.transpose()*/;
math::Matrix4f viewMatrix = view.getViewMatrix().getMatrix()/*.transpose()*/;
indirectDrawPushConsts.projMatrix = toVulkanMatrix(projMatrix);
indirectDrawPushConsts.viewMatrix = toVulkanMatrix(viewMatrix);
//projMatrix.identity();
skyboxPushConsts.projMatrix = toVulkanMatrix(projMatrix);
skyboxPushConsts.viewMatrix = toVulkanMatrix(viewMatrix);
drawInstances();
drawInstancesIndexed();
drawSelectedInstances();
drawSelectedInstancesIndexed();
vb.clear();
vb.setPrimitiveType(Triangles);
Vertex v1 (math::Vec3f(0, 0, quad.getSize().z()));
Vertex v2 (math::Vec3f(quad.getSize().x(),0, quad.getSize().z()));
Vertex v3 (math::Vec3f(quad.getSize().x(), quad.getSize().y(), quad.getSize().z()));
Vertex v4 (math::Vec3f(0, quad.getSize().y(), quad.getSize().z()));
vb.append(v1);
vb.append(v2);
vb.append(v3);
vb.append(v1);
vb.append(v3);
vb.append(v4);
vb.update();
math::Matrix4f matrix = quad.getTransform().getMatrix()/*.transpose()*/;
//////std::cout<<"world mat : "<<matrix<<std::endl;
ppll2PushConsts.worldMat = toVulkanMatrix(matrix);
//system("PAUSE");
currentStates.shader = &perPixelLinkedListP2;
currentStates.blendMode = BlendNone;
//frameBuffer.enableStencilTest(false);
//createDescriptorSets2(currentStates);
createCommandBufferVertexBuffer(currentStates);
}
}
}
I put the value to wait for the timeline semaphore to values2, at the end of commands submitions. Finally I draw my offscreen rendering to the main frame like this (by wating on the fence of my offscreen rendering and signaling window fence to tell that we can do render to that offscreen texture of that frame again :
void PerPixelLinkedListRenderComponent::draw(RenderTarget& target, RenderStates states) {
std::unique_lock<std::mutex> lock(mtx[0]);
if (useThread) {
//std::cout<<"lock mutex draw frame thread : "<<windowCurrentFrame<<","<<std::this_thread::get_id()<<std::endl;
cv[0].wait(lock, [this] {
return commandBufferReady[windowCurrentFrame].load() || stop.load();
});
//std::cout<<"unlock mutex draw frame thread : "<<windowCurrentFrame<<","<<std::this_thread::get_id()<<std::endl;
}
std::cout<<"draw frame : "<<windowCurrentFrame<<","<<getLayer()<<std::endl;
commandBufferReady[windowCurrentFrame] = false;
registerFrameJob[frameBuffer.getCurrentFrame()] = true;
cv[0].notify_all();
//std::cout<<"draw : "<<frameBuffer.getCurrentFrame()<<std::endl;
target.beginRecordCommandBuffers();
const_cast<Texture&>(frameBuffer.getTexture(windowCurrentImage)).toShaderReadOnlyOptimal(target.getCommandBuffers()[target.getCurrentFrame()]);
frameBufferSprite.setCenter(target.getView().getPosition());
frameBufferSprite.setTexture(frameBuffer.getTexture(windowCurrentImage));
////////std::cout<<"view position : "<<view.getPosition()<<std::endl;
////////std::cout<<"sprite position : "<<frameBufferSprite.getCenter()<<std::endl;
states.blendMode = BlendAlpha;
states.blendMode.updateIds();
//////std::cout<<"blend mode : "<<states.blendMode.colorSrcFactor<<std::endl;
target.draw(frameBufferSprite, states);
std::vector<VkSemaphore> waitSemaphores, signalSemaphores;
std::vector<VkPipelineStageFlags> waitStages;
std::vector<uint64_t> waitValues, signalValues;
waitSemaphores.push_back(offscreenFinishedSemaphore[windowCurrentFrame]);
waitStages.push_back(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
signalSemaphores.push_back(offscreenFinishedSemaphore[windowCurrentFrame]);
waitValues.push_back(values[windowCurrentFrame]);
values[windowCurrentFrame]++;
signalValues.push_back(values[windowCurrentFrame]);
std::vector<VkFence> inFligthFence = {fences[windowCurrentFrame]};
target.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, (stop.load()) ? std::vector<VkFence>() : inFligthFence);
if (!stop.load()) {
target.beginRecordCommandBuffers();
const_cast<Texture&>(frameBuffer.getTexture(windowCurrentImage)).toColorAttachmentOptimal(target.getCommandBuffers()[target.getCurrentFrame()]);
waitValues.clear();
signalValues.clear();
waitValues.push_back(values[windowCurrentFrame]);
values[windowCurrentFrame]++;
signalValues.push_back(values[windowCurrentFrame]);
target.submit(false, signalSemaphores, waitSemaphores, waitStages, signalValues, waitValues, std::vector<VkFence> (), 0, false, true, windowFences[windowCurrentFrame]);
}
std::cout<<"frame drawn : "<<windowCurrentFrame<<","<<getLayer()<<std::endl;
windowCurrentFrame = (windowCurrentFrame + 1) % MAX_FRAMES_IN_FLIGHT;
windowCurrentImage = (windowCurrentImage + 1) % frameBuffer.getSwapchainImagesSize();
//std::cout<<"frame drawn : "<<frameBuffer.getCurrentFrame()<<std::endl;
//std::cout<<"signaled : "<<frameBuffer.getCurrentFrame()<<std::endl;
//std::cout<<"next frame"<<std::endl;
}
But, despite every synchronization mecanism I’ve those annoying validation error messages. (Not at every executions of my demo program) :
validation layer: vkCmdExecuteCommands(): pCommandBuffers[0] was called in VkCommandBuffer 0x1d1c032bb90 which is invalid because bound VkDescriptorSet 0x7c046a0000000c24 was destroyed or updated.
The Vulkan spec states: Each element of pCommandBuffers must be in the pending or executable state (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/cmdbuffers.html#VUID-vkCmdExecuteCommands-pCommandBuffers-00089``)
validation layer: vkQueueSubmit(): pSubmits[0].pCommandBuffers[0] Secondary command buffer VkCommandBuffer 0x1d1c032bb90 is not in a valid (pending or executable) state.
The Vulkan spec states: Any secondary command buffers recorded into any element of the pCommandBuffers member of any element of pSubmits must be in the pending or executable state (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/cmdbuffers.html#VUID-vkQueueSubmit-pCommandBuffers-00072``)
wait for fence result : -4
Fence status before reset: -4
validation layer: vkResetFences(): pFences[0] (VkFence 0x651e090000000232) is in use.
The Vulkan spec states: Each element of pFences must not be currently associated with any queue command that has not yet completed execution on that queue (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/synchronization.html#VUID-vkResetFences-pFences-01123``)
validation layer: vkResetCommandBuffer(): (VkCommandBuffer 0x1d1c025fb10) is in use.
The Vulkan spec states: commandBuffer must not be in the pending state (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/cmdbuffers.html#VUID-vkResetCommandBuffer-commandBuffer-00045``)
validation layer: vkBeginCommandBuffer(): on active VkCommandBuffer 0x1d1c025fb10 before it has completed. You must check command buffer fence before this call.
The Vulkan spec states: commandBuffer must not be in the recording or pending state (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/cmdbuffers.html#VUID-vkBeginCommandBuffer-commandBuffer-00049``)
validation layer: vkQueueSubmit(): pSubmits[0].pCommandBuffers[0] VkCommandBuffer 0x1d1c025fb10 is already in use and is not marked for simultaneous use.
The Vulkan spec states: If any element of the pCommandBuffers member of any element of pSubmits was not recorded with the VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT, it must not be in the pending state (``https://vulkan.lunarg.com/doc/view/1.4.309.0/windows/antora/spec/latest/chapters/cmdbuffers.html#VUID-vkQueueSubmit-pCommandBuffers-00071``)
terminate called after throwing an instance of ‘odfaeg::core::Erreur’
what(): Úchec de l’envoi d’un command buffer!
I don’t understand why the driver, try to update descriptor sets/buffers which are still in use, even with the vkWaitSemaphores call…