Use of Pixel Buffer Objects with multithreading

Hi, I’ve been looking for every post and thread about using PBOs to read pixels from the buffer, but couldn’t find the answer to my problem. I’m writing a 2D program that simulates an autonomous car and I’m working on its Image Sensor, that should interpret the environment and detect road lines, ecc. My idea was to map the screen with a PBO on a separate thread than the GL render thread (if it is possible). The problem is, that I don’t know how to create and use the PBO on the other thread. I tried to but the pointer returning from the glMapBuffer method was null. Here’s my code:

Basically the read_pixels() method works in the gl thread right now.

main.cpp


#define _USE_MATH_DEFINES

#include <iostream>
#include <cstdio>
#include <math.h>
#include <array>

#include "vector.h"
#include "car.h"
#include "main.h"
#include "gl_utils.h"
#include "road_piece.h"
#include "road_piece_g.h"
#include "road_builder.h"
#include "lane_piece.h"
#include "lane_piece_g.h"
#include "line_piece.h"
#include "line_piece_g.h"
#include "car_g.h"
#include "image_sensor.h"
#include "image_sensor_g.h"

using namespace std;
using namespace std::literals::chrono_literals;
using namespace SmartCar;

// animation information
float DELTA = 0.010;
nanoseconds timestep(10ms);
float gravity = 9.81f;

/* car graphics information */
// body
float car_body_base = 2.0f;
float car_body_height = 3.0f;
colors::Color car_body_color = white;

// speed vector
float speed_vector_length = 30.0f;
colors::Color speed_vector_color = magenta;

// target vector
float target_vector_length = 30.0f;
colors::Color target_vector_color = magenta;

// path
float path_thickness = 3.0f;
colors::Color path_color = blue;

/* destination */
float destination_radius = 1.5f;
colors::Color destination_color = green;

/* road graphics information */
colors::Color road_color = black;
colors::Color line_color = white;

/* opengl */
GLFWwindow* window;

/* other variables */
Car* car;

vector<RoadPiece*> roads;

bool dragging = false;
Point current_mouse_position;
Point last_mouse_position;
Vector drag_vector;
float scale_factor = 5.0f;

// CAR IMPLEMENTATIONS
// 1. SENSORS
//	1. ULTRASONIC SENSOR
//	2. IMAGE SENSOR
//	3. RADAR SENSOR
//	4. LIDAR SENSOR

colors::Color c = red;

GLuint pboId;
int DATA_SIZE = 1920 * 1080 * 4;
unsigned char* ptr;

int main(int argc, char* argv[])
{
	// opengl
	if (!glfwInit())
	{
		::exit(EXIT_FAILURE);
	}

	glfwWindowHint(GLFW_SAMPLES, 8);
	window = glfwCreateWindow(1920, 1080, "Smart Car", NULL, NULL);

	glfwSetErrorCallback(error_callback);
	glfwSetKeyCallback(window, key_callback);
	glfwSetScrollCallback(window, scroll_callback);
	glfwSetMouseButtonCallback(window, mouse_callback);
	glfwSetCursorPosCallback(window, cursor_position_callback);
	glfwMakeContextCurrent(window);
	glfwSwapInterval(1);

	glGenBuffers = (PFNGLGENBUFFERSPROC)glfwGetProcAddress("glGenBuffers");
	glBindBuffer = (PFNGLBINDBUFFERPROC)glfwGetProcAddress("glBindBuffer");
	glBufferData = (PFNGLBUFFERDATAPROC)glfwGetProcAddress("glBufferData");
	glBufferSubData = (PFNGLBUFFERSUBDATAPROC)glfwGetProcAddress("glBufferSubData");
	glDeleteBuffers = (PFNGLDELETEBUFFERSPROC)glfwGetProcAddress("glDeleteBuffers");
	glGetBufferParameteriv = (PFNGLGETBUFFERPARAMETERIVPROC)glfwGetProcAddress("glGetBufferParameteriv");
	glMapBuffer = (PFNGLMAPBUFFERPROC)glfwGetProcAddress("glMapBuffer");
	glUnmapBuffer = (PFNGLUNMAPBUFFERPROC)glfwGetProcAddress("glUnmapBuffer");

	glGenBuffers(1, &pboId);
	glBindBuffer(GL_PIXEL_PACK_BUFFER, pboId);
	glBufferData(GL_PIXEL_PACK_BUFFER, DATA_SIZE, NULL, GL_STREAM_READ);
	glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);

	// initialization
	init_roads();
	init_car();

	while (!glfwWindowShouldClose(window))
	{
		float ratio;
		int width, height;

		glfwGetFramebufferSize(window, &width, &height);
		ratio = width / (float)height;

		glViewport(0, 0, width, height);
		glClear(GL_COLOR_BUFFER_BIT);

		glMatrixMode(GL_PROJECTION);
		glLoadIdentity();
		glOrtho(0, 1920, 1080, 0, 1, -1);

		glMatrixMode(GL_MODELVIEW);
		glLoadIdentity();

		glScalef(scale_factor, scale_factor, 0);
		glTranslatef(drag_vector.x, drag_vector.y, 0);

		draw();

		glfwSwapBuffers(window);
		glfwPollEvents();
	}

	glfwDestroyWindow(window);
	glfwTerminate();
	::exit(EXIT_SUCCESS);
}

void init_roads()
{
	RoadBuilder rb;

	rb.start(rb.create_road_piece(true, Point(500, 500), 3 * M_PI_2, 200.0f, M_PI_4, 5.0f, 2));
	rb.add_road_piece(true, 400.0f, 0.0f);
	rb.add_road_piece(true, 200.0f, M_PI_4);
	rb.add_road_piece(false, 200.0f, M_PI_4);
	rb.add_road_piece(true, 200.0f, 0.0f);
	rb.add_road_piece(true, 200.0f, M_PI_4);

	roads = rb.finish();

	cout << "roads initialized" << endl;
}

void init_car()
{
	car = new Car();

	car->car_g = new CarG();

	CarG* car_g = car->car_g;
	car_g->car = car;
	car_g->position = new Point(800, 700);
	car_g->angle = 3 * M_PI_2;
	car_g->width = 2.0f;
	car_g->length = 3.0f;

	car->speed = 0.0f;
	car->turning_speed = M_PI / 5.0f;
	car->acceleration = 0.0f;
	car->ttg_friction = 0.5;

	car->image_sensor = new ImageSensor();

	ImageSensor* image_sensor = car->image_sensor;
	image_sensor->car = car;

	image_sensor->image_sensor_g = new ImageSensorG();

	ImageSensorG* image_sensor_g = image_sensor->image_sensor_g;
	image_sensor_g->image_sensor = image_sensor;
	image_sensor_g->zero = car_g->position;
	image_sensor_g->position = (*car_g->get_center() - *car_g->position);
	image_sensor_g->angle = 3 * M_PI_2;
	image_sensor_g->portion = M_PI_2;
	image_sensor_g->radius = 150.0f;

	car->image_sensor->car = car;

	cout << "car initialized" << endl;
}

void error_callback(int error, const char* description)
{
	fprintf(stderr, "Error: %s
", description);
}

static void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods)
{
	if (key == GLFW_KEY_ESCAPE && action == GLFW_PRESS)
	{
		glfwSetWindowShouldClose(window, GLFW_TRUE);
	}
	if (key == GLFW_KEY_SPACE && action == GLFW_PRESS)
	{
		car->start();
	}
}

static void cursor_position_callback(GLFWwindow* window, double xpos, double ypos)
{
	if (dragging)
	{
		last_mouse_position = current_mouse_position;
		current_mouse_position.set(xpos, ypos);

		drag_vector += current_mouse_position - last_mouse_position;
	}
}

static void mouse_callback(GLFWwindow* window, int button, int action, int mods)
{
	double xpos, ypos;

	if (button == GLFW_MOUSE_BUTTON_LEFT) 
	{
		if (action == GLFW_PRESS)
		{
			dragging = true;
			glfwGetCursorPos(window, &xpos, &ypos);
			current_mouse_position.set(xpos, ypos);
		}
		else if (action == GLFW_RELEASE)
		{
			dragging = false;
		}
	}

}

void scroll_callback(GLFWwindow* window, double xoffset, double yoffset)
{
	if (yoffset > 0) scale_factor *= 1.1;
	if (yoffset < 0) scale_factor /= 1.1;
}

void draw()
{
	car->car_g->draw();

	for (int i = 0; i < roads.size(); i++) (*roads[i]).road_g->draw();

	for (int i1 = 0; i1 <= 100; i1 += 20)
	{
		for (int i2 = 0; i2 <= 100; i2 += 20)
		{
			draw_point(car->image_sensor->image_sensor_g->get_at(i1, i2), 0.5f, red);
		}
	}

	draw_square(Point(0, 0), 100.0f, 0.0f, red);

	read_pixels();
}

void read_pixels()
{
	glBindBuffer(GL_PIXEL_PACK_BUFFER, pboId);
	glReadPixels(0, 0, 1920, 1080, GL_RGBA, GL_UNSIGNED_BYTE, 0);
	ptr = (unsigned char*)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);

	if (ptr)
	{
		int r = (int)ptr[0];
		int g = (int)ptr[1];
		int b = (int)ptr[2];

		if (r == 255 && g == 0 && b == 0)
		{
			cout << "red" << endl;
		}
	}

	glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
}

float get_rand(float min, float max)
{
	float random = ((float)rand()) / (float)RAND_MAX;
	float diff = max - min;
	float r = random * diff;

	return min + r;
}

In the ImageSensor class the thread method is use(), as you can see there are some “traces” of the PBO but I commented them as nothing worked

image_sensor.cpp


#include <iostream>
#include <GL\glew.h>
#include <GLFW\glfw3.h>

#include "image_sensor.h"
#include "image_sensor_g.h"
#include "gl_utils.h"
#include "car.h"
#include "car_g.h"

using namespace std;
using namespace SmartCar;

ImageSensor::ImageSensor() 
{
	//glGenBuffers(1, &pboId);
	//glBindBuffer(GL_PIXEL_PACK_BUFFER, pboId);
	//glBufferData(GL_PIXEL_PACK_BUFFER, DATA_SIZE, NULL, GL_STREAM_READ);
	//glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
}

void ImageSensor::use()
{
	image_sensor_g->lock.lock();

	float r = image_sensor_g->radius;
	float a = image_sensor_g->angle;

	float i1 = 0.0f;
	float i2 = 0.0f;

	float d1 = 100.0f / r;
	float d2 = 0.0f;

	/*
	for (; i1 < 100.0f; i1 += d1)
	{
		d2 = 10000.0f / (r * i1);

		for (; i2 < 100.0f; i2 += d2)
		{
			Point p1 = image_sensor_g->get_at(i1, i2);
			Point p2 = Point((int)p1.x, (int)p1.y);

			glReadPixels(p2.x, p2.y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, 0);
			ptr = (unsigned char*)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);

			if (ptr)
			{
				int r = (int)ptr[0];
				int g = (int)ptr[1];
				int b = (int)ptr[2];

				if ((int)ptr[0] == 255 && (int)ptr[1] == 255 && (int)ptr[2] == 255)
				{
					cout << "FOUND" << endl;
					//car->image_sensor->image_sensor_g->white_points.push_back(p2);
				}
			}
		}

		i2 = 0.0f;
	}
	

	glBindBuffer(GL_PIXEL_PACK_BUFFER, pboId);
	glReadPixels(0, 0, 1920, 1080, GL_RGBA, GL_UNSIGNED_BYTE, 0);
	ptr = (unsigned char*)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY);

	if (ptr)
	{
		int r = (int)ptr[0];
		int g = (int)ptr[1];
		int b = (int)ptr[2];

		if (r == 255 && g == 0 && b == 0)
		{
			cout << "red" << endl;
		}
	}

	glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
        */

	image_sensor_g->lock.unlock();
}

I hope I’ve been clear enough, thanks in advance!

In order to use OpenGL commands, you need a context. Contexts are associated with threads, and a context cannot be current in more than one thread at a time. So you either need to create multiple contexts which share data, or only use OpenGL commands from the main thread (you can map the buffer in the main thread then have other threads read the mapped region).

With GLFW, contexts are tied to windows, so if you want multiple contexts you need multiple windows (unnecessary windows can be hidden by calling glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE) prior to creation). glfwMakeContextCurrent() needs to be called to bind a context to a thread, so that OpenGL commands issued from that thread will use the specified context.