I am getting a VK_ERROR_DEVICE_LOST return code in my queue submit command when using vkCmdDrawIndexedIndirect(), with Nvidia hardware only. It works perfectly fine with Intel graphics.
It also works fine if I use vkCmdDrawIndexed for each batch, one by one, on both Nvidia and Intel.
The device->SetBufferData method uses a staging buffer that is associated with the command buffer to transfer data to the indirect buffer.
This has a lot of my own wrapper code in it, but you can see some of what is going on. Can you see anything wrong with this?:
bool VkuContext::RecordFrame(const int currentFrame)
{
commandbuffers[currentFrame]->Reset();
refreshcommandbuffer[currentFrame] = false;
//---------------------------------------------------------------
// Draw commands
//---------------------------------------------------------------
// Build draw structures
uint32_t firstInstance = 0;
VkDrawIndexedIndirectCommand command;
for (auto mesh : queuedmeshes)
{
if (mesh->indiceCount != 0 and mesh->instanceCount != 0)
{
command.indexCount = mesh->indiceCount;
command.vertexOffset = mesh->vertexoffset;
command.firstInstance = firstInstance;
command.firstIndex = mesh->indiceoffset;
command.instanceCount = mesh->instanceCount;
commandbuffers[currentFrame]->drawcommands.push_back(command);
}
else
{
Assert(0);
}
firstInstance += mesh->instanceCount;
}
// Create buffer and send data
if (!commandbuffers[currentFrame]->drawcommands.empty())
{
auto buffersz = commandbuffers[currentFrame]->drawcommands.size() * sizeof(VkDrawIndexedIndirectCommand);
if (commandbuffers[currentFrame]->indirectcommandbuffer != nullptr)
{
if (buffersz > commandbuffers[currentFrame]->indirectcommandbuffer->GetSize())
{
commandbuffers[currentFrame]->indirectcommandbuffer = nullptr;
}
}
if (commandbuffers[currentFrame]->indirectcommandbuffer == nullptr)
{
commandbuffers[currentFrame]->indirectcommandbuffer = make_shared<GPUBuffer>(device);
commandbuffers[currentFrame]->indirectcommandbuffer->Initialize(buffersz, VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VMA_MEMORY_USAGE_GPU_ONLY);
}
device->SetBufferData(commandbuffers[currentFrame]->indirectcommandbuffer, &commandbuffers[currentFrame]->drawcommands[0], buffersz, false);
commandbuffers[currentFrame]->BindResource(commandbuffers[currentFrame]->indirectcommandbuffer);
}
//---------------------------------------------------------------
// Send instance data
//---------------------------------------------------------------
// Keep storage buffers in scope until command buffer is reset
commandbuffers[currentFrame]->boundresources.insert(commandbuffers[currentFrame]->boundresources.end(), device->storagebuffer.begin(), device->storagebuffer.end());
//---------------------------------------------------------------
// Begin recording
//---------------------------------------------------------------
commandbuffers[currentFrame]->Begin();
//---------------------------------------------------------------
// Begin render pass
//---------------------------------------------------------------
VkRenderPassBeginInfo renderPassBeginInfo = {};
renderPassBeginInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
renderPassBeginInfo.renderPass = renderPass;
renderPassBeginInfo.framebuffer = framebuffers[currentFrame];
renderPassBeginInfo.renderArea.offset = { 0, 0 };
renderPassBeginInfo.renderArea.extent = chaininfo.imageExtent;
VkClearValue clear = { clearColor[0], clearColor[1], clearColor[2], clearColor[3] };
renderPassBeginInfo.clearValueCount = 1;
renderPassBeginInfo.pClearValues = &clear;
vkCmdBeginRenderPass(commandbuffers[currentFrame]->commandbuffer, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
//---------------------------------------------------------------
// Record commands
//---------------------------------------------------------------
// Bind pipeline
vkCmdBindPipeline(commandbuffers[currentFrame]->commandbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, graphicsPipeline);
// Bind storage buffers
vkCmdBindDescriptorSets(commandbuffers[currentFrame]->commandbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, environment->pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
// Push constants
vkCmdPushConstants(commandbuffers[currentFrame]->commandbuffer, device->instance->pipelineLayout, VK_SHADER_STAGE_ALL, 0, sizeof(device->instance->shaderglobals), &device->instance->shaderglobals);
// Bind vertex and index buffers
vkCmdBindVertexBuffers(commandbuffers[currentFrame]->commandbuffer, 0, 1, &device->vertexbuffer->buffer, offsets);
vkCmdBindIndexBuffer(commandbuffers[currentFrame]->commandbuffer, device->indicebuffer->buffer, 0, VK_INDEX_TYPE_UINT32);
commandbuffers[currentFrame]->BindResource(device->vertexbuffer);
commandbuffers[currentFrame]->BindResource(device->indicebuffer);
// Draw
if (!commandbuffers[currentFrame]->drawcommands.empty())
{
if (commandbuffers[currentFrame]->drawcommands.size() > device->limits.maxDrawIndirectCount)
{
// One batch at a time
for (int n = 0; n < commandbuffers[currentFrame]->drawcommands.size(); ++n)
{
//vkCmdDrawIndexed(commandbuffers[currentFrame]->commandbuffer, commandbuffers[currentFrame]->drawcommands[n].indexCount, commandbuffers[currentFrame]->drawcommands[n].instanceCount, commandbuffers[currentFrame]->drawcommands[n].firstIndex, commandbuffers[currentFrame]->drawcommands[n].vertexOffset, commandbuffers[currentFrame]->drawcommands[n].firstInstance);
vkCmdDrawIndexedIndirect(commandbuffers[currentFrame]->commandbuffer, commandbuffers[currentFrame]->indirectcommandbuffer->buffer, n * sizeof(VkDrawIndexedIndirectCommand), 1, sizeof(VkDrawIndexedIndirectCommand));
}
}
else
{
//All at once
vkCmdDrawIndexedIndirect(commandbuffers[currentFrame]->commandbuffer, commandbuffers[currentFrame]->indirectcommandbuffer->buffer, 0, commandbuffers[currentFrame]->drawcommands.size(), sizeof(VkDrawIndexedIndirectCommand));
}
}
// Finish up
commandbuffers[currentFrame]->EndRenderPass();
commandbuffers[currentFrame]->End();
return true;
}