Attempting to do pbr raytracing, but keep getting a device lost error

Been playing around with PBR ray tracing again, and having some crashing problems.

When do the irradiance reflection alone, it works fine.
When I’m doing the specular reflection alone, it works fine.
When I try both at the same time it crashes.

It’s a device lost error. I’ve confirmed that it’s running all the way through the shader, printing out the red value. I’m running a 3080 ti, and MaxReflectCount is set to 5. rnd is from vk_raytracing_tutorial_KHR/random.glsl at master · nvpro-samples/vk_raytracing_tutorial_KHR · GitHub.

Is there something I’m missing, or something that you guys can that’s making it crash?

#version 460
#extension GL_EXT_ray_tracing : require
#extension GL_EXT_nonuniform_qualifier : enable
#extension GL_EXT_scalar_block_layout : enable
#extension GL_EXT_debug_printf : enable
#include "Lights.glsl"

struct Vertex
{
	vec3 Position;
	float PositionPadding;
	vec3 Normal;
	float NormalPadding;
	vec2 UV;
	vec2 UVPadding;
	vec3 Tangant;
	float TangantPadding;
	vec3 BiTangant;
	float BiTangentPadding;
	vec3 Color;
	float ColorPadding;
};
struct MeshProperties
{
	uint MaterialBufferIndex;
	uint SkyBoxIndex;
	mat4 MeshTransform;
	vec2 UVOffset;
	vec2 UVScale;
	vec2 UVFlip;
	int SelectedMesh;
	float heightScale;
	float minLayers;
	float maxLayers;
};

struct MaterialProperties
{
	vec3 Ambient;
	vec3 Diffuse;
	vec3 Specular;
	float Shininess;
	float Reflectivness;

	vec3 Albedo;
	float Matallic;
	float Roughness;
	float AmbientOcclusion;
	vec3 Emission;
	float Alpha;

	uint DiffuseMapID;
	uint SpecularMapID;
	uint AlbedoMapID;
	uint MetallicMapID;
	uint RoughnessMapID;
	uint AmbientOcclusionMapID;
	uint NormalMapID;
	uint DepthMapID;
	uint AlphaMapID;
	uint EmissionMapID;
};


layout(push_constant) uniform SceneData
{
    uint MeshIndex;
    mat4 proj;
    mat4 view;
    vec3 CameraPos;
    vec3 MeshColorID;
    vec3 AmbientLight;
    uint DirectionalLightCount;
    uint PointLightCount;
    uint SpotLightCount;
    float Timer;
    float PBRMaxMipLevel;
    uint frame;
    int MaxRefeflectCount;
} sceneData;

struct RayPayload {
	vec3 color;
	uint seed;
	int reflectCount;
};

layout(location = 0) rayPayloadInEXT RayPayload rayHitInfo;
layout(location = 1) rayPayloadEXT bool shadowed;
hitAttributeEXT vec2 attribs;

layout(binding = 0, set = 0) uniform accelerationStructureEXT topLevelAS;
layout(binding = 1, set = 0, rgba8) uniform image2D RayTracedTexture;
layout(binding = 2, scalar) buffer Vertices { Vertex v[]; } vertices[];
layout(binding = 3) buffer Indices { uint i[]; } indices[];
layout(binding = 4) buffer MeshPropertiesBuffer { MeshProperties meshProperties; } meshBuffer[];
layout(binding = 5) buffer MaterialPropertiesBuffer { MaterialProperties materialProperties; } materialBuffer[];
layout(binding = 6) buffer DirectionalLightBuffer { DirectionalLight directionalLight; } DLight[];
layout(binding = 7) buffer PointLightBuffer { PointLight pointLight; } PLight[];
layout(binding = 8) buffer SpotLightBuffer { SpotLight spotLight; } SLight[];
layout(binding = 9) uniform sampler2D TextureMap[];
layout(binding = 10) uniform samplerCube CubeMap;

#include "RTXRandom.glsl"
Vertex BuildVertexInfo()
{
	Vertex vertex;
	const ivec3 index = ivec3(indices[gl_InstanceCustomIndexEXT].i[3 * gl_PrimitiveID],
						      indices[gl_InstanceCustomIndexEXT].i[3 * gl_PrimitiveID + 1],
							  indices[gl_InstanceCustomIndexEXT].i[3 * gl_PrimitiveID + 2]);

	const Vertex v0 = vertices[gl_InstanceCustomIndexEXT].v[index.x];
	const Vertex v1 = vertices[gl_InstanceCustomIndexEXT].v[index.y];
	const Vertex v2 = vertices[gl_InstanceCustomIndexEXT].v[index.z];

	const vec3 barycentricCoords = vec3(1.0f - attribs.x - attribs.y, attribs.x, attribs.y);

	vertex.Position = v0.Position * barycentricCoords.x + v1.Position * barycentricCoords.y + v2.Position * barycentricCoords.z;
	vertex.Position = vec3(meshBuffer[gl_InstanceCustomIndexEXT].meshProperties.MeshTransform * vec4(vertex.Position, 1.0));

	vertex.UV = v0.UV * barycentricCoords.x + v1.UV * barycentricCoords.y + v2.UV * barycentricCoords.z;
	vertex.Normal = v0.Normal * barycentricCoords.x + v1.Normal * barycentricCoords.y + v2.Normal * barycentricCoords.z;
    vertex.Tangant = v0.Tangant * barycentricCoords.x + v1.Tangant * barycentricCoords.y + v2.Tangant * barycentricCoords.z;
	vertex.BiTangant = v0.BiTangant * barycentricCoords.x + v1.BiTangant * barycentricCoords.y + v2.BiTangant * barycentricCoords.z;
	vertex.Color = v0.Color * barycentricCoords.x + v1.Color * barycentricCoords.y + v2.Color * barycentricCoords.z;

	return vertex;
}

const float PI = 3.14159265359;

void main()
{		
   Vertex vertex = BuildVertexInfo();
    const uint materialID = meshBuffer[gl_InstanceCustomIndexEXT].meshProperties.MaterialBufferIndex;
    MaterialProperties material = materialBuffer[materialID].materialProperties;

    vec2 FinalUV = vertex.UV + meshBuffer[sceneData.MeshIndex].meshProperties.UVOffset;
         FinalUV *= meshBuffer[sceneData.MeshIndex].meshProperties.UVScale;

   if(meshBuffer[sceneData.MeshIndex].meshProperties.UVFlip.y == 1.0f)
   {
        FinalUV.y = 1.0f - FinalUV.y;
   }
   if(meshBuffer[sceneData.MeshIndex].meshProperties.UVFlip.x == 1.0f)
   {
        FinalUV.x = 1.0f - FinalUV.x;
   }

   vec3 albedo = pow(material.Albedo, vec3(2.2));
   if (material.AlbedoMapID != 0)
   {
       albedo = pow(texture(TextureMap[material.AlbedoMapID], FinalUV).rgb, vec3(2.2));
   }

   float metallic =  material.Matallic;
   if (material.MetallicMapID != 0)
   {
     metallic = texture(TextureMap[material.MetallicMapID], FinalUV).r;
   }

   float roughness =  material.Roughness;
   if (material.RoughnessMapID != 0)
   {
     roughness = texture(TextureMap[material.RoughnessMapID], FinalUV).r;
   }

   float ao = material.AmbientOcclusion;
   if (material.AmbientOcclusionMapID != 0)
   {
     ao = texture(TextureMap[material.AmbientOcclusionMapID], FinalUV).r;
   }

   vec3 emission = material.Emission;
   if (material.EmissionMapID != 0)
   {
       emission = texture(TextureMap[material.EmissionMapID], FinalUV).rgb;
   }

   mat3 TBN = getTBNFromMap(vertex);
   vec3 N = vertex.Normal;

   vec3 ViewPos  = sceneData.CameraPos;
   vec3 FragPos  = vertex.Position;
   vec3 viewDir = normalize(ViewPos - FragPos);
   if (material.NormalMapID != 0)
   {
        ViewPos  = TBN * sceneData.CameraPos;
        FragPos  = TBN * vertex.Position;

        if(material.DepthMapID != 0)
        {
            FinalUV = ParallaxMapping(vertex, material, FinalUV, viewDir);       
        }
        N = texture(TextureMap[material.NormalMapID], FinalUV).rgb;
        N = normalize(N * 2.0 - 1.0);
        N = normalize(TBN * N);
   }

    vec3 V = normalize(sceneData.CameraPos - FragPos);
    vec3 R = reflect(-V, N); 

    vec3 F0 = vec3(0.04); 
    F0 = mix(F0, albedo, metallic);

    vec3 Lo = vec3(0.0);
    Lo += CalcDirectionalLight(F0, V, N, albedo, roughness, metallic);
    Lo += CalcPointLight(vertex, F0, V, N, albedo, roughness, metallic);
    Lo += CalcSpotLight(vertex, F0, V, N, albedo, roughness, metallic);

    vec3 F = fresnelSchlickRoughness(max(dot(N, V), 0.0), F0, roughness);
    vec3 kS = F;
    vec3 kD = 1.0 - kS;
    kD *= 1.0 - metallic;	  
    
    vec3 irradiance = vec3(0.0f);
    if(rayHitInfo.reflectCount <= sceneData.MaxRefeflectCount)
    {
       uint seed = tea(gl_LaunchIDEXT.y * gl_LaunchSizeEXT.x + gl_LaunchIDEXT.x, sceneData.frame);
       float r1        = rnd(seed)*1.10f;
       float r2        = rnd(seed)*1.10f;
       float r3        = rnd(seed)*1.10f;
       if(gl_InstanceCustomIndexEXT == 0)
       {
       debugPrintfEXT("R1: %f \n", r1 );
       debugPrintfEXT("R2: %f \n", r2 );
       debugPrintfEXT("R3: %f \n", r3 );
}
        vec3 hitPos = gl_WorldRayOriginEXT + gl_WorldRayDirectionEXT * gl_RayTmaxEXT;
        vec3 origin   = hitPos.xyz + N * 0.001f;
        vec3 rayDir   = reflect(origin, N + vec3(r1, r2, r3));

        rayHitInfo.reflectCount++;
        traceRayEXT(topLevelAS, gl_RayFlagsNoneEXT, 0xff, 0, 0, 0, origin, 0.001f, rayDir, 10000.0f, 0);
		irradiance += rayHitInfo.color; 
    }
    irradiance /= rayHitInfo.reflectCount;
    rayHitInfo.reflectCount = 0;

    
    vec3 specular = vec3(0.0f);    
    if(metallic > 0.0f &&
       rayHitInfo.reflectCount <= sceneData.MaxRefeflectCount) 
    {
       uint seed = tea(gl_LaunchIDEXT.y * gl_LaunchSizeEXT.x + gl_LaunchIDEXT.x, sceneData.frame);
       float r1        = rnd(seed)*0.5f;
       float r2        = rnd(seed)*0.5f;
       float r3        = rnd(seed)*0.5f;

        vec3 hitPos = gl_WorldRayOriginEXT + gl_WorldRayDirectionEXT * gl_RayTmaxEXT;
        vec3 origin   = hitPos.xyz + N * 0.001f;
        vec3 rayDir   = reflect(origin, N + material.Roughness * vec3(r1, r2, r3));

        rayHitInfo.reflectCount++;
        traceRayEXT(topLevelAS, gl_RayFlagsNoneEXT, 0xff, 0, 0, 0, origin, 0.001f, rayDir, 10000.0f, 0);
		specular += rayHitInfo.color; 
	}
    specular /= rayHitInfo.reflectCount;
    rayHitInfo.reflectCount = 0;

    vec3 diffuse = irradiance * albedo;
    vec3 ambient = emission + ((kD * diffuse + specular) * ao);
    
    vec3 color = ambient + Lo;
    color = color / (color + vec3(1.0f));
    color = pow(color, vec3(1.0f/2.2f)); 
    rayHitInfo.color = color;

   debugPrintfEXT("R: %f \n", rayHitInfo.color.r );
}

Note that any driver/OS reserves the right to kill your GPU access if your shader takes too long to execute or otherwise behaves pathologically. How long that is varies, and I don’t know exactly how that interacts with ray tracing. But I image there’s a soft limit on how much recursive ray tracing you can do before the system decides that your process is behaving pathologically.

If both your irradiance reflection and specular reflection operations are generating more rays, doing both will generate moreXmore rays. That could easily cross some boundary condition. If they’re generating rays, try to reduce the number that they generate.

Tried reducing the reflections to one each one.
And tried doing it on NES Resolution(256 × 240 pixels) but no luck.

Also tried 120x20 still no luck.

Found somethign that sorta works.

    vec3 irradiance = vec3(0.0f);
    if(rayHitInfo.reflectCount  == 0) 
    {
       uint seed = tea(gl_LaunchIDEXT.y * gl_LaunchSizeEXT.x + gl_LaunchIDEXT.x, sceneData.frame);
       float r1        = rnd(seed)*1.10f;
       float r2        = rnd(seed)*1.10f;
       float r3        = rnd(seed)*1.10f;

        vec3 hitPos = gl_WorldRayOriginEXT + gl_WorldRayDirectionEXT * gl_RayTmaxEXT;
        vec3 origin   = hitPos.xyz + N * 0.001f;
        vec3 rayDir   = reflect(origin, N + vec3(r1, r2, r3));

        rayHitInfo.reflectCount++;
        traceRayEXT(topLevelAS, gl_RayFlagsNoneEXT, 0xff, 0, 0, 0, origin, 0.001f, rayDir, 10000.0f, 0);
		irradiance += rayHitInfo.color; 
    }
    irradiance /= rayHitInfo.reflectCount;
   // rayHitInfo.reflectCount = 0;

    
    vec3 specular = vec3(0.0f);    
    if(metallic > 0.0f &&
       rayHitInfo.reflectCount == 1) 
    {
       uint seed = tea(gl_LaunchIDEXT.y * gl_LaunchSizeEXT.x + gl_LaunchIDEXT.x, sceneData.frame);
       float r1        = rnd(seed)*0.5f;
       float r2        = rnd(seed)*0.5f;
       float r3        = rnd(seed)*0.5f;

        vec3 hitPos = gl_WorldRayOriginEXT + gl_WorldRayDirectionEXT * gl_RayTmaxEXT;
        vec3 origin   = hitPos.xyz + N * 0.001f;
        vec3 rayDir   = reflect(origin, N + material.Roughness * vec3(r1, r2, r3));

        rayHitInfo.reflectCount++;
        traceRayEXT(topLevelAS, gl_RayFlagsNoneEXT, 0xff, 0, 0, 0, origin, 0.001f, rayDir, 10000.0f, 0);
		specular += rayHitInfo.color; 
	}
    specular /= rayHitInfo.reflectCount;
    rayHitInfo.reflectCount = 0;