Top VkAccelerationStructureKHR update doesnt reduce Instances

[Rlocksley]

I try to integrate dynamic spawn and despawn of Instances in my RayTracing Engine.

When I try to update my Top VkAccelerationStructureKHR to reduce the number of spawned VkAccelerationStructureInstanceKHR from maxInstanceCount to maxInstanceCount-1 the last Instance stays visible.

I am using a Nvidia RTX 2060 Super.
There are no Validation Errors.
Here is my update Function:

void updateVkTopAccelerationStructure(RobX_VkTopAccelerationStructure& acceleration, RobX_VkInstanceBuffer& instanceBuffer, uint32_t numberInstances)
    {
        VkDeviceOrHostAddressConstKHR instanceAddress{};
        instanceAddress.deviceAddress = instanceBuffer.deviceAddress;

        VkAccelerationStructureGeometryKHR geometry{};
        geometry.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR;
        geometry.flags = VK_GEOMETRY_OPAQUE_BIT_KHR;
        geometry.geometryType = VK_GEOMETRY_TYPE_INSTANCES_KHR;
        geometry.geometry.instances.sType = 
        VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_INSTANCES_DATA_KHR;
        geometry.geometry.instances.data = instanceAddress;
        geometry.geometry.instances.arrayOfPointers = VK_FALSE;

        VkAccelerationStructureBuildGeometryInfoKHR buildInfo{};
        buildInfo.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR;
        buildInfo.type = VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR;
        buildInfo.flags = VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR |
                        VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_BUILD_BIT_KHR;
        buildInfo.mode = VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR;
        buildInfo.srcAccelerationStructure = acceleration.vkAccelerationStructureKHR;
        buildInfo.dstAccelerationStructure = acceleration.vkAccelerationStructureKHR;
        buildInfo.geometryCount = 1;
        buildInfo.pGeometries = &geometry;
        buildInfo.scratchData.deviceAddress = acceleration.vkScratchBuffer.deviceAddress;

        VkAccelerationStructureBuildRangeInfoKHR rangeInfo{};
        rangeInfo.primitiveCount = numberInstances;
        rangeInfo.primitiveOffset = 0;
        rangeInfo.firstVertex = 0;
        rangeInfo.transformOffset = 0;

        const VkAccelerationStructureBuildRangeInfoKHR* rangeInfoArr[1] = {&rangeInfo};    

        beginVkCommand(vkTopAccelerationStructureUpdateCommandBuffer);

        vkCmdBuildAccelerationStructuresKHR
        (vkTopAccelerationStructureUpdateCommandBuffer,
        1,
        &buildInfo,
        rangeInfoArr);

        endVkCommand(vkTopAccelerationStructureUpdateCommandBuffer);

        submitVkCommand
        (VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR,
        vkImageIndexSemaphore,
        vkTopAccelerationStructureUpdateCommandBuffer,
        vkTopAccelerationStructureUpdateSemaphore,
        nullptr);
    }