Intersection is detected only when a camera is close to the object

Every frame I check if a ray intersects a triangle. But in practice program tell me that triangle is intersected only when I move camera on to it’s position. I don’t even have to put cursor on a object to get this message(just be close enough). My code is inspired by this PDF.. I tried to set ray origin to x:0.0, y:0.0, z:0.0 coordinates and using TEST_CULL macro but it doesn’t change behaviour of a program. I’ve also tried approach with Ray-OBB but program behaves exactly the same as before except that camera doesn’t have to be directly on triangle position just a little bite in front. What am I doing wrong?

main.cpp

#include <glad/glad.h>
#include <GLFW/glfw3.h>
#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/type_ptr.hpp>

#include "VBO.h"
#include "VAO.h"
#include "EBO.h"
#include "Camera.h"
#include "MousePicker.h"
#include "Shader.h"

#define EPSILON 0.000001

void framebuffer_size_callback(GLFWwindow* window, int width, int height);
void mouse_callback(GLFWwindow* window, double xpos, double ypos);
void scroll_callback(GLFWwindow* window, double xoffset, double yoffset);
void processInput(GLFWwindow* window);

bool rayTriangleIntersect(glm::vec3 orygin, glm::vec3 dir, glm::vec3 v0, glm::vec3 v1, glm::vec3 v2, float &t, float &u, float &v);

const unsigned int SCR_WIDTH = 800;
const unsigned int SCR_HEIGHT = 600;

Camera camera(glm::vec3(0.0f, 0.0f, 3.0f));
float lastX = SCR_WIDTH / 2.0f;
float lastY = SCR_HEIGHT / 2.0f;
bool firstMouse = true;

float deltaTime = 0.0f; 
float lastFrame = 0.0f;

int main()
{
    glfwInit();
    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);

    GLFWwindow* window = glfwCreateWindow(800, 600, "Hello Window!", NULL, NULL);
    if (window == NULL)
    {
        std::cout << "Failed to initialize GLFW" << std::endl;
        glfwTerminate();
        return -1;
    }

    glfwMakeContextCurrent(window);

    if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress))
    {
        std::cout << "Failed to initialize GLAD" << std::endl;
        return -1;
    }

    glViewport(0, 0, 800, 600);
    glfwSetFramebufferSizeCallback(window, framebuffer_size_callback);
    glfwSetCursorPosCallback(window, mouse_callback);
    glfwSetScrollCallback(window, scroll_callback);

    glEnable(GL_DEPTH_TEST);

    float vertices[] = {
    -0.5f, -0.5f, 0.0f,
     0.5f, -0.5f, 0.0f,
     0.0f,  0.5f, 0.0f
    };

    Shader shaderProgram("vertexShader.vs", "fragmentShader.fs");

    VAO VAO1;
    VAO1.Bind();
    VBO VBO(vertices, sizeof(vertices));
    VAO1.LinkAttrib(VBO, 0, 3, GL_FLOAT, 3 * sizeof(float), (void*)0);

    shaderProgram.use();

    glm::mat4 projection = glm::mat4(1.0f);
    MousePicker picker(&camera, &projection, window);

    float u, v, t;

    while (!glfwWindowShouldClose(window))
    {   
        float currentFrame = glfwGetTime();
        deltaTime = currentFrame - lastFrame;
        lastFrame = currentFrame;

        processInput(window);
        glClearColor(0.2f, 0.3f, 0.3f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

        picker.update();

        shaderProgram.use();

        projection = glm::perspective(glm::radians(camera.Zoom), (float)SCR_WIDTH / (float)SCR_HEIGHT, 0.1f, 100.0f);
        shaderProgram.setMat4("projection", projection);

        glm::mat4 view = camera.GetViewMatrix();
        shaderProgram.setMat4("view", view);

        glm::vec3 rayCoords = picker.getCurrentRay();

        if (rayTriangleIntersect(camera.Position, rayCoords, glm::vec3(-0.5f, -0.5f, 0.0f), glm::vec3(0.5f, -0.5f, 0.0f), glm::vec3(0.0f, 0.5f, 0.0f), t, u, v))
        {
            std::cout << "Ray intersects triangle" << std::endl;
        }

        //std::cout << rayCoords.x << "  " << rayCoords.y << "  " << rayCoords.z << std::endl;
        //std::cout << camera.Position.x << " " << camera.Position.y << " " << camera.Position.z << std::endl;

        VAO1.Bind();
        glDrawArrays(GL_TRIANGLES, 0, 3);

        glfwSwapBuffers(window);
        glfwPollEvents();
    }
    VAO1.Unbind();
    VBO.Unbind();
    VAO1.Delete();
    VBO.Delete();
    shaderProgram.~Shader();
    glfwTerminate();
}

bool rayTriangleIntersect(glm::vec3 orygin, glm::vec3 dir, glm::vec3 v0, glm::vec3 v1, glm::vec3 v2, float &t, float &u, float &v)
{
    glm::vec3 E1 = v1 - v0;
    glm::vec3 E2 = v2 - v0;
    glm::vec3 T = orygin - v0;

    glm::vec3 P = glm::cross(dir, E2);
    glm::vec3 Q;
    float det = glm::dot(E1, P);
    float inv_det;

    #ifdef TEST_CULL
    if (det < EPSILON)
        return false;

    u = glm::dot(T, P);
    if (u < 0.0f || u > det)
        return false;

    Q = glm::cross(T, E1);
    v = glm::dot(dir, Q);
    if (v < 0.0f || u + v > det)
        return false;

    t = glm::dot(E2, Q);
    inv_det = 1.0 / det;
    t *= inv_det;
    u *= inv_det;
    v *= inv_det;
    #else
    if (det > -EPSILON && det < EPSILON)
        return false;   
    inv_det = 1.0f / det;
    T = orygin - v0;
    u = glm::dot(T, P) * inv_det;
    if (u < 0.0 || u > 1.0)
        return false;
    
    Q = glm::cross(T, E1);
    v = glm::dot(dir, Q) * inv_det;
    if (v < 0.0 || u + v > 1.0)
        return false;

    t = glm::dot(E2, Q) * inv_det;
#endif
    return true;
}

void framebuffer_size_callback(GLFWwindow* window, int width, int height)
{
    glViewport(0, 0, width, height);
}

void processInput(GLFWwindow* window)
{
    static bool lkey = false;
    static bool lkey2 = false;

    if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS)
        glfwSetWindowShouldClose(window, true);

    if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS)
        camera.ProcessKeyboard(FORWARD, deltaTime);
    if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
        camera.ProcessKeyboard(BACKWARD, deltaTime);
    if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS)
        camera.ProcessKeyboard(LEFT, deltaTime);
    if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS)
        camera.ProcessKeyboard(RIGHT, deltaTime);

    if (glfwGetKey(window, GLFW_KEY_R) == GLFW_PRESS && !lkey)
    {
        lkey = true;

        int polygonMode;
        glGetIntegerv(GL_POLYGON_MODE, &polygonMode);

        if (polygonMode == GL_LINE)
            glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
        if (polygonMode == GL_FILL)
            glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
    }
    else if (glfwGetKey(window, GLFW_KEY_R) == GLFW_RELEASE && lkey)
        lkey = false;

    if (glfwGetKey(window, GLFW_KEY_F) == GLFW_PRESS && !lkey2)
    {
        lkey2 = true;
        glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_DISABLED);
    }
    else if (glfwGetKey(window, GLFW_KEY_F) == GLFW_RELEASE && lkey2)
        lkey2 = false;
}

void mouse_callback(GLFWwindow* window, double xpos, double ypos)
{
    if (firstMouse)
    {
        lastX = xpos;
        lastY = ypos;
        firstMouse = false;
    }

    float xoffset = xpos - lastX;
    float yoffset = lastY - ypos;

    lastX = xpos;
    lastY = ypos;

    camera.ProcessMouseMovement(xoffset, yoffset);
}

void scroll_callback(GLFWwindow* window, double xoffset, double yoffset)
{
    camera.ProcessMouseScroll(yoffset);
}

MousePicker.cpp

#include "MousePicker.h"

MousePicker::MousePicker(Camera* camera, glm::mat4 *projection, GLFWwindow* window)
{
    this->camera = camera;
    this->window = window;
    this->projectionMatrix = projection;
    viewMatrix = camera->GetViewMatrix();
}

glm::vec3 MousePicker::getCurrentRay()
{
    return this->currentRay;
}

void MousePicker::update()
{
    viewMatrix = camera->GetViewMatrix();
    currentRay = calculateMouseRay();
}

glm::vec3 MousePicker::calculateMouseRay()
{
    glfwGetCursorPos(window, &this->mouseX, &this->mouseY);
    glm::vec3 normalizedCoords = this->getNormalizedCoords(mouseX, mouseY);
    glm::vec4 rayClip = glm::vec4(normalizedCoords.x, normalizedCoords.y, -1.0f, 1.0f);
    glm::vec4 eyeCoords = this->toEyeCoords(rayClip);
    glm::vec3 worldRay = this->toWorldCoords(eyeCoords);

    return worldRay;
}

glm::vec3 MousePicker::getNormalizedCoords(double xPos, double yPos)
{
    int width, height;

    glfwGetWindowSize(window, &width, &height);

    double x = (2.0f * xPos) / (double)(width) - 1.0f;
    double y = 1.0f - (2.0f * yPos) / (double)(height);

    return glm::vec3(width, height, 1.0f);
}

glm::vec4 MousePicker::toEyeCoords(glm::vec4 rayClip)
{
    glm::mat4 invertedProjection = glm::inverse(*projectionMatrix);
    glm::vec4 eyeCoords = invertedProjection * rayClip;

    return glm::vec4(eyeCoords.x, eyeCoords.y, -1.0f, 0.0f);
}

glm::vec3 MousePicker::toWorldCoords(glm::vec4 eyeCoords)
{
    glm::vec4 rayWorld = glm::inverse(viewMatrix) * eyeCoords;
    glm::vec3 finalCoords = glm::vec3(rayWorld.x, rayWorld.y, rayWorld.z);
    finalCoords = glm::normalize(finalCoords);

    return finalCoords;
}

Note that GLM has glm::intersectRayTriangle in glm/gtx/intersect.hpp.

Typically, the picking process goes like this:

  1. Get the (integer) mouse coordinates xm,ym.

  2. Convert to window coordinates:

xw=xm+0.5, yw=h-0.5 -ym

where h is the window height. This accounts for the fact that mouse coordinates normally have their origin at the top-left corner of the window. The 0.5 offset uses the pixel’s centre rather than a corner (rasterisation is based upon pixel centres).

  1. Convert to NDC using the viewport transformation:

xNDC = 2*(xw-x)/w - 1
yNDC = 2*(yw-y)/h - 1

where x,y,w,h are the viewport dimensions given to glViewport.

  1. Transform the vectors [xNDC,yNDC,-1,1] and [xNDC,yNDC,1,1] to object space using the inverse of the model-view-projection matrix.

  2. Divide each vector by its w coordinate. You now have the points where the ray intersects the near and far planes.

  3. Perform the ray-triangle intersection calculation. Ideally you get 5 parameters: barycentric coordinates a,b,c for the point within the triangle (a+b+c=1), barycentric coordinates s,t for the point along the line segment (s+t=1). If any of a,b,c are negative, the ray doesn’t intersect the triangle. If either of s,t are negative, the intersection is outside the near or far planes. For glm::intersectRayTriangle, you get a “distance” value which (IIRC) will be in [0,1] and corresponds to t.

If you don’t care about the near/far plane test, you can use the viewpoint (transformed to object space) instead of transforming [xNDC,yNDC,-1,1]. That will result in a distance of 0 at the viewpoint rather than at the near plane. You still need to ignore intersections with negative distance as those are behind the viewer.

For the ray direction, you can use any two vectors of the form [xNDC,yNDC,z,1], transform to object space, divide by w, and subtract. The choice of z only affects the origin and scale of the returned distance value.

Thanks for your answer. I tried comply to your advice about getting ray coords and replaced function rayTriangleIntersect with build-in glm funciton but now program is displaying all the time message about ray triangle intersection. I also don’t understand a few things. Firstly why converting to NDC for yNDC is done by 2*(yw-y)/h - 1? Many tutorials uses equation 1 - 2*(yw-y)/h. Secondly if I convert mouse coords from screen to object space why in glm funciton as ray originI paste camera world coordinates?

Here is my my modified MousePicker class

#include "MousePicker.h"

MousePicker::MousePicker(Camera* camera, glm::mat4* projection, GLFWwindow* window)
{
	this->camera = camera;
	this->window = window;
	this->projectionMatrix = projection;
	viewMatrix = camera->GetViewMatrix();
}

glm::vec3 MousePicker::getCurrentRay()
{
	return this->currentRay;
}

void MousePicker::update()
{
	viewMatrix = camera->GetViewMatrix();
	currentRay = calculateMouseRay();
}

glm::vec3 MousePicker::calculateMouseRay()
{
	glfwGetCursorPos(window, &this->mouseX, &this->mouseY);
	glm::vec3 normalizedCoords = this->getNormalizedCoords(mouseX, mouseY);
	glm::vec4 nearPlan = glm::vec4(normalizedCoords.x, normalizedCoords.y, -1.0f, 1.0f);
	glm::vec4 farPlan = glm::vec4(normalizedCoords.x, normalizedCoords.y, 1.0f, 1.0f);
	nearPlan = this->toObjectSpace(nearPlan);
	farPlan = this->toObjectSpace(farPlan);

	nearPlan /= nearPlan.w;
	farPlan /= farPlan.w;
	glm::vec3 dir = farPlan - nearPlan;

	return glm::normalize(dir);
}

glm::vec4 MousePicker::toObjectSpace(glm::vec4 NDC)
{
	glm::mat4 ReversedViewProj = glm::inverse(camera->GetViewMatrix() * *projectionMatrix);
	glm::vec4 objCoords = ReversedViewProj * NDC;

	return objCoords;
}

glm::vec3 MousePicker::getNormalizedCoords(double xPos, double yPos)
{
	int width, height;
	glfwGetWindowSize(window, &width, &height);

	mouseX = xPos + 0.5;
	mouseY = height - 0.5 - yPos;

	double x = (2.0f * mouseX) / (double)(width) - 1.0;
	double y = (2.0f * mouseY) / (double)(height) - 1.0;

	return glm::vec3(x, y, 1.0f);
}

And what I changed in main.cpp

picker.update();
		glm::vec3 rayCoords = picker.getCurrentRay();

		if (glm::intersectRayTriangle(camera.Position, rayCoords, glm::vec3(-0.5f, -0.5f, 0.0f), glm::vec3(0.5f, -0.5f, 0.0f), glm::vec3(0.0f, 0.5f, 0.0f), baryPos, distance));
		{
			std::cout << "Ray intersects triangle" << std::endl;
		}

They’re merging steps 2 and 3, i.e. going directly from mouse coordinates to NDC. Sometimes the window coordinates are relevant (e.g. if you’re reading the framebuffer).

The main thing I notice is that the order of multiplications is incorrect:

The overall transformation from world space (or object space) to clip space is projection * view.

Finally i solved a problem. I deleted class MousePicker and replaced it witch a function I found on a github (a little bit modified). The whole code is below thank you for help!!!

#include <iostream>

#include <glad/glad.h>
#include <GLFW/glfw3.h>
#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <glm/gtx/intersect.hpp>

#include "VBO.h"
#include "VAO.h"
#include "EBO.h"
#include "Camera.h"
#include "Shader.h"

void framebuffer_size_callback(GLFWwindow* window, int width, int height);
void mouse_callback(GLFWwindow* window, double xpos, double ypos);
void scroll_callback(GLFWwindow* window, double xoffset, double yoffset);
void processInput(GLFWwindow* window);

void ScreenPosToWorldRay(
	int mouseX, int mouseY,             // Mouse position, in pixels, from bottom-left corner of the window
	int screenWidth, int screenHeight,  // Window size, in pixels
	glm::mat4 ViewMatrix,               // Camera position and orientation
	glm::mat4 ProjectionMatrix,         // Camera parameters (ratio, field of view, near and far planes)
	glm::mat4 ModelMatrix,
	glm::vec3& out_origin,              // Ouput : Origin of the ray. /!\ Starts at the near plane, so if you want the ray to start at the camera's position instead, ignore this.
	glm::vec3& out_direction            // Ouput : Direction, in world space, of the ray that goes "through" the mouse.
);

const unsigned int SCR_WIDTH = 800;
const unsigned int SCR_HEIGHT = 600;

Camera camera(glm::vec3(0.0f, 0.0f, 3.0f));
float lastX = SCR_WIDTH / 2.0f;
float lastY = SCR_HEIGHT / 2.0f;
bool firstMouse = true;

float deltaTime = 0.0f;	
float lastFrame = 0.0f;

int main()
{
	glfwInit();
	glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
	glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
	glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);

	GLFWwindow* window = glfwCreateWindow(800, 600, "Hello Window!", NULL, NULL);
	if (window == NULL)
	{
		std::cout << "Failed to initialize GLFW" << std::endl;
		glfwTerminate();
		return -1;
	}

	glfwMakeContextCurrent(window);

	if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress)) 
	{
		std::cout << "Failed to initialize GLAD" << std::endl;
		return -1;
	}

	glViewport(0, 0, 800, 600);
	glfwSetFramebufferSizeCallback(window, framebuffer_size_callback);
	glfwSetCursorPosCallback(window, mouse_callback);
	glfwSetScrollCallback(window, scroll_callback);

	//glfwSetInputMode(window, GLFW_CURSOR, GLFW_CURSOR_DISABLED);

	glEnable(GL_DEPTH_TEST);

	float vertices[] = {
	-0.5f, -0.5f, 0.0f,
	 0.5f, -0.5f, 0.0f,
	 0.0f,  0.5f, 0.0f
	};

	Shader shaderProgram("vertexShader.vs", "fragmentShader.fs");

	VAO VAO1;
	VAO1.Bind();
	VBO VBO(vertices, sizeof(vertices));
	VAO1.LinkAttrib(VBO, 0, 3, GL_FLOAT, 3 * sizeof(float), (void*)0);

	shaderProgram.use();
	//camera.Position = glm::vec3(0.0f, 5.0f, 3.0f);
	glm::mat4 model(1.0f);
	model = glm::rotate(model, glm::radians(50.0f), glm::vec3(1.0f, 0.0f, 1.0f));
	shaderProgram.setMat4("model", model);

	while (!glfwWindowShouldClose(window))
	{
		float currentFrame = glfwGetTime();
		deltaTime = currentFrame - lastFrame;
		lastFrame = currentFrame;

		processInput(window);
		glClearColor(0.2f, 0.3f, 0.3f, 1.0f);
		glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

		shaderProgram.use();

		glm::mat4 projection = glm::perspective(glm::radians(camera.Zoom), (float)SCR_WIDTH / (float)SCR_HEIGHT, 0.1f, 100.0f);
		shaderProgram.setMat4("projection", projection);
		
		glm::mat4 view = camera.GetViewMatrix();
		shaderProgram.setMat4("view", view);

		if (glfwGetMouseButton(window, GLFW_MOUSE_BUTTON_LEFT))
		{
			glm::vec3 origin, direction;

			double mouseX, mouseY;
			glfwGetCursorPos(window, &mouseX, &mouseY);
			int width, height;
			glfwGetWindowSize(window, &width, &height);

			ScreenPosToWorldRay(mouseX, mouseY, width, height, view, projection, model, origin, direction);
			direction *= 1000.0f;

			glm::vec2 barPos;
			float t;

			if (glm::intersectRayTriangle(origin, direction, glm::vec3(-0.5f, -0.5f, 0.0f), glm::vec3(0.5f, -0.5f, 0.0f), glm::vec3(0.0f, 0.5f, 0.0f), barPos, t))
			{
				//std::cout << "Origin of a ray: " << origin.x << "  " << origin.y << "  " << origin.z << std::endl;
				//std::cout << barPos.x << "  " << barPos.y << std::endl;
				std::cout << t << std::endl;
			}
		}

		VAO1.Bind();
		glDrawArrays(GL_TRIANGLES, 0, 3);

		glfwSwapBuffers(window);
		glfwPollEvents();
	}

	VAO1.Unbind();
	VBO.Unbind();
	VAO1.Delete();
	VBO.Delete();
	shaderProgram.~Shader();
	glfwTerminate();
}

void ScreenPosToWorldRay(
	int mouseX, int mouseY,             
	int screenWidth, int screenHeight,  
	glm::mat4 ViewMatrix,              
	glm::mat4 ProjectionMatrix,
	glm::mat4 ModelMatrix,
	glm::vec3& out_origin,             
	glm::vec3& out_direction	         
) {
	mouseX += 0.5;
	mouseY = screenHeight - 0.5 - mouseY;

	glm::vec4 lRayStart_NDC(
		((float)mouseX / (float)screenWidth - 0.5f) * 2.0f, 
		((float)mouseY / (float)screenHeight - 0.5f) * 2.0f, 
		-1.0, 
		1.0f
	);
	glm::vec4 lRayEnd_NDC(((float)mouseX / (float)screenWidth - 0.5f) * 2.0f, ((float)mouseY / (float)screenHeight - 0.5f) * 2.0f, 1.0, 1.0f);

	glm::mat4 M = glm::inverse(ProjectionMatrix * ViewMatrix * ModelMatrix);
	glm::vec4 lRayStart_world = M * lRayStart_NDC; lRayStart_world/=lRayStart_world.w;
	glm::vec4 lRayEnd_world   = M * lRayEnd_NDC  ; lRayEnd_world  /=lRayEnd_world.w;


	glm::vec3 lRayDir_world(lRayEnd_world - lRayStart_world);
	lRayDir_world = glm::normalize(lRayDir_world);

	//out_origin = glm::vec3(camera.Position);
	out_origin = glm::vec3(lRayStart_world);
	out_direction = glm::normalize(lRayDir_world);
}

void framebuffer_size_callback(GLFWwindow* window, int width, int height)
{
	glViewport(0, 0, width, height);
}

void processInput(GLFWwindow* window)
{
	static bool lkey = false;

	if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS)
		glfwSetWindowShouldClose(window, true);

	if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS)
		camera.ProcessKeyboard(FORWARD, deltaTime);
	if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
		camera.ProcessKeyboard(BACKWARD, deltaTime);
	if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS)
		camera.ProcessKeyboard(LEFT, deltaTime);
	if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS)
		camera.ProcessKeyboard(RIGHT, deltaTime);

	if (glfwGetKey(window, GLFW_KEY_R) == GLFW_PRESS && !lkey)
	{
		lkey = true;

		int polygonMode;
		glGetIntegerv(GL_POLYGON_MODE, &polygonMode);

		if (polygonMode == GL_LINE)
			glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
		if (polygonMode == GL_FILL)
			glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
	}
	else if (glfwGetKey(window, GLFW_KEY_R) == GLFW_RELEASE && lkey)
	{
		lkey = false;
	}

}

void mouse_callback(GLFWwindow* window, double xpos, double ypos)
{
	if (firstMouse)
	{
		lastX = xpos;
		lastY = ypos;
		firstMouse = false;
	}

	float xoffset = xpos - lastX;
	float yoffset = lastY - ypos;

	lastX = xpos;
	lastY = ypos;

	camera.ProcessMouseMovement(xoffset, yoffset);
}

void scroll_callback(GLFWwindow* window, double xoffset, double yoffset)
{
	camera.ProcessMouseScroll(yoffset);
}

But I also have last question? Can you explain me this peace of code:

glm::vec4 lRayStart_NDC(
		((float)mouseX / (float)screenWidth - 0.5f) * 2.0f, 
		((float)mouseY / (float)screenHeight - 0.5f) * 2.0f, 
		-1.0, 
		1.0f
	);

I don’t understand why is it like that any resources witch can explain me this process?

mouseX / screenWidth is the normalised X position in the range [0,1]. Subtracting 0.5 converts the range to [-0.5, 0.5]. Multiplying by 2 converts it to [-1,1]. Likewise for Y.

More generally, f(x)=(x-0.5)*2 or f(x)=x*2-1 converts [0,1] to [-1,1].

Note that the calculation is only correct if the viewport covers the entire window. See my earlier post for the more general formula. The viewport bounds can be queried with glGetIntegerv(GL_VIEWPORT, ...), but it’s usually better to just store the values locally when setting the viewport.

Final results below everything works

void ScreenPosToWorldRay(
	GLFWwindow* window,
	glm::mat4 ViewMatrix,              
	glm::mat4 ProjectionMatrix,
	glm::mat4 ModelMatrix,
	glm::vec3& out_origin,             
	glm::vec3& out_direction	         
) {
	double mouseX, mouseY;
	glfwGetCursorPos(window, &mouseX, &mouseY);

	int width, height;
	glfwGetWindowSize(window, &width, &height);

	mouseX += 0.5;
	mouseY = height - 0.5 - mouseY;

	int viewPortDimensions[4];
	glGetIntegerv(GL_VIEWPORT, viewPortDimensions);

	glm::vec4 lRayStart_NDC(
		(((float)mouseX - (float)viewPortDimensions[0]) / (float)viewPortDimensions[2] - 0.5f) * 2.0f,
		(((float)mouseY - (float)viewPortDimensions[1]) / (float)viewPortDimensions[3] - 0.5f) * 2.0f,
		-1.0,
		1.0f
	);
	glm::vec4 lRayEnd_NDC(
		(((float)mouseX-(float)viewPortDimensions[0]) / (float)viewPortDimensions[2] - 0.5f) * 2.0f,
		(((float)mouseY - (float)viewPortDimensions[1]) / (float)viewPortDimensions[3] - 0.5f) * 2.0f, 1.0, 1.0f);

	glm::mat4 M = glm::inverse(ProjectionMatrix * ViewMatrix * ModelMatrix);
	glm::vec4 lRayStart_world = M * lRayStart_NDC; lRayStart_world/=lRayStart_world.w;
	glm::vec4 lRayEnd_world   = M * lRayEnd_NDC  ; lRayEnd_world  /=lRayEnd_world.w;


	glm::vec3 lRayDir_world(lRayEnd_world - lRayStart_world);
	lRayDir_world = glm::normalize(lRayDir_world);

	//out_origin = glm::vec3(camera.Position); // start of the ray at camera posiiton
	out_origin = glm::vec3(lRayStart_world); // start of the ray at near plan
	out_direction = glm::normalize(lRayDir_world);
}