How to add to a PyOpenGL VBO from another thread?

The problem

I’m just trying to make a game like minecraft, but I can’t add to a vbo from another thread. I’ve created a basic batch renderer with a texture coord pointer too. But when I implement threading, the strange thing is that the window just hangs closes after the data has been added to the VBO

Code

As given in my other VBO question, I have implemented:

# imports
import glfw, numpy
from OpenGL.GL import *
from ctypes import *
import threading
from core.texture_manager import *

event = threading.Event()

glfw.init()

class ThreadedVBOManager:
    def __init__(self, renderer):
        self.renderer = renderer
        glfw.make_context_current(None)
        self.thread = threading.Thread(target=self.run, daemon=True)
        self.thread.start()
        event.wait()
        glfw.make_context_current(self.renderer.window)
    
    def run(self):
        glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
        self.window = glfw.create_window(300, 300, "Window 2", None, self.renderer.window)
        glfw.make_context_current(self.window)
        event.set()

        while not glfw.window_should_close(self.window):
            for i in self.renderer.to_add:
                glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo)

                self.renderer.vertices.extend(i[0])
                self.renderer.texCoords.extend(i[1])
                self.renderer.vertices_added = len(self.renderer.vertices)
                glBufferData(GL_ARRAY_BUFFER, self.renderer.vertices_added * 4, None, GL_STATIC_DRAW)

                glBufferSubData(GL_ARRAY_BUFFER, 0, i[0].nbytes, (GLfloat * i[0].nbytes)(*i[0]))
                glFlush()
                glVertexPointer(3, GL_FLOAT, 0, None)
                glTexCoordPointer(3, GL_FLOAT, 0, None)
                glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo_1)
                glBufferSubData(GL_ARRAY_BUFFER, 0, i[1].nbytes, (GLfloat * i[1].nbytes)(*i[1]))
                glFlush()

            glfw.poll_events()
            glfw.swap_buffers(self.window)

class TerrainRenderer:
    def __init__(self, window):
        self.window = window

        self.vertices = []
        self.texCoords = []

        self.to_add = []

        self.vertices_added = 0

        self.vbo, self.vbo_1 = glGenBuffers (2)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo)
        glBufferData(GL_ARRAY_BUFFER, 12 * 4, None, GL_STATIC_DRAW)
        self.vbo_manager = ThreadedVBOManager(self)

        self.texture_manager = TextureAtlas()

        glEnable(GL_TEXTURE_2D)
        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
        glEnableClientState(GL_TEXTURE_COORD_ARRAY)
        glEnableClientState (GL_VERTEX_ARRAY)

    def render(self):
        glClear (GL_COLOR_BUFFER_BIT)

        glEnable(GL_TEXTURE_2D)
        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
        glBindBuffer (GL_ARRAY_BUFFER, self.vbo)
        glVertexPointer (3, GL_FLOAT, 0, None)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo_1)
        glTexCoordPointer(2, GL_FLOAT, 0, None)

        glDrawArrays (GL_QUADS, 0, self.vertices_added)
        glDisable(GL_TEXTURE_2D)
        glDisable(GL_BLEND)

    def add(self, posList, texCoords):
        self.to_add.append((numpy.array(posList), numpy.array(texCoords)))

    def update_vbo(self):
        pass

Sometimes it gives this error too:

OpenGL.error.GLError: GLError(
        err = 1281,
        description = b'invalid value',
        baseOperation = glBufferSubData,
        pyArgs = (
                GL_ARRAY_BUFFER,
                0,
                64,
                <core.renderer.c_float_Array_64 object at 0x000001964F159840>,
        ),
        cArgs = (
                GL_ARRAY_BUFFER,
                0,
                64,
                <core.renderer.c_float_Array_64 object at 0x000001964F159840>,
        ),
        cArguments = (
                GL_ARRAY_BUFFER,
                0,
                64,
                <core.renderer.c_float_Array_64 object at 0x000001964F159840>,
        )
)

In order for a thread to make OpenGL calls an OpenGL context must be current (wglMakeCurrent / glxMakeCurrent) on that thread. An OpenGL context can be current on at most one thread at a time. An application can create more than one OpenGL context and make those current on different threads, additionally it is possible to for contexts to share most OpenGL objects (exception are container objects) - see the first section of the wiki for more details.

Also, note that wglMakeCurrent / glxMakeCurrent are fairly expensive operations so it is generally not a good idea to bounce a single context around making it current (at different times) on a bunch of threads each frame.
Have you profiled your application to determine where it spends most of its time?

So can I use wglShareLists to share lists between different contexts?

I tried doing that, but it just crashes the application instead. It also shows the background color and other things too.

If you want OpenGL contexts to share objects you have to establish that when creating the contexts. Since you are using GLFW you should check their documentation on how to do that, see here.

Before you go too far down this path: profile your application to determine where your bottleneck is and then consider if the type of multi threading you are attempting here can solve it. I didn’t read your code closely, but it looks to me like you want to perform some vertex data upload on a thread, however your draw thread is rendering with that same vertex data concurrently. If your bottleneck is the data upload this does not seem like something you can solve with multiple threads without also making significant other changes. To understand why consider that the render thread (or at the very least the GPU) has to wait for the data upload to be complete before it can draw those vertices, so upload and rendering are still happening in sequence and not concurrently.
To benefit from concurrent data uploads to the GPU your rendering needs to operate on different (already uploaded) data until the data transfer is complete.

Also, consider using persistent&coherent mapping of buffers. This will allow you to pass the pointer to the buffer to your writing thread, instead of requiring the writer thread to make OpenGL calls directly. You’ll still need inter-thread synchronization so that you’ve finished writing before issuing commands that read.

And of course, this assumes that PyOpenGL allows for such things…

I used:

# map the buffer
self.mapped_pointer = glMapBuffer(GL_ARRAY_BUFFER, GL_WRITE_ONLY)
self.mapped_pointer = cast(self.mapped_pointer, POINTER(c_float))

And in the starting of the run() function of the threaded manager:

vbo_ptr, vbo_ptr_1 = glGenBuffers(2)
glBindBuffer(GL_ARRAY_BUFFER, vbo_ptr)
glBindBuffer(GL_ARRAY_BUFFER, vbo_ptr_1)

But now it gives:

OpenGL.error.GLError: GLError(
        err = 1282,
        description = b'invalid operation',
        baseOperation = glMapBuffer,
        cArguments = (GL_ARRAY_BUFFER, GL_WRITE_ONLY)
)

You can’t map a buffer until you’ve allocated storage for it with glBufferData (or glBufferStorage for persistent mapping). I’m not sure what this code is meant to accomplish, but if it’s supposed to let you map either buffer, it doesn’t do that.

I’m making a game like minecraft in python, and I need to add data and texCoords to a VBO from another thread. Can you help me with that?

I updated the code to:

# imports
import glfw, numpy
from OpenGL.GL import *
from ctypes import *
from core.texture_manager import *
import threading

glfw.init()
event = threading.Event()

class VBOManager:
    def __init__(self, renderer):
        self.renderer = renderer
        glfw.make_context_current(None)
        self.thread = threading.Thread(target=self.run)
        self.thread.start()
        event.wait()
        glfw.make_context_current(self.renderer.window)
    
    def run(self):
        glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
        self.window = glfw.create_window(300, 300, "Window 2", None, self.renderer.window)
        glfw.make_context_current(self.window)
        event.set()

        while not glfw.window_should_close(self.renderer.window):
            while self.renderer.rendering:
                pass
            self.renderer.allow_rendering = False

            for i in range(0,self.renderer.to_add_count):
                if not len(self.renderer.to_add) == 0:
                    try:
                        self.renderer.vertices.extend(self.renderer.to_add[0][0])
                        self.renderer.texCoords.extend(self.renderer.to_add[0][1])
                        self.renderer.vertices_added = len(self.renderer.vertices)

                        self.renderer.to_add.remove(self.renderer.to_add[0])
                    except IndexError:
                        pass

            glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo)
            glBufferData(GL_ARRAY_BUFFER, len(self.renderer.vertices) * 4, (c_float * len(self.renderer.vertices))(*self.renderer.vertices), GL_STATIC_DRAW)
            glFlush()
            glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo_1)
            glBufferData(GL_ARRAY_BUFFER, len(self.renderer.texCoords) * 4, (c_float * len(self.renderer.texCoords))(*self.renderer.texCoords), GL_STATIC_DRAW)
            glFlush()

            glfw.poll_events()
            glfw.swap_buffers(self.window)      

            self.renderer.allow_rendering = True  

class TerrainRenderer:
    def __init__(self, window):
        self.window = window

        self.vertices = []
        self.texCoords = []

        self.to_add = []
        self.to_add_count = 1024 * 1024

        self.vertices_added = 0

        self.rendering = False
        self.allow_rendering = True

        self.vbo, self.vbo_1 = glGenBuffers (2)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo)
        glBufferData(GL_ARRAY_BUFFER, 12 * 4, (c_float * 12)(0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0), GL_STATIC_DRAW)
        glBindBuffer(GL_ARRAY_BUFFER, self.vbo_1)
        glBufferData(GL_ARRAY_BUFFER, 12 * 4, None, GL_DYNAMIC_DRAW)

        self.vbo_manager = VBOManager(self)

        self.texture_manager = TextureAtlas()

        glEnable(GL_TEXTURE_2D)
        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
        glEnableClientState(GL_TEXTURE_COORD_ARRAY)
        glEnableClientState (GL_VERTEX_ARRAY)

    def render(self):
        if self.allow_rendering:
            self.rendering = True
            self.vbo_manager.run()

            glClear (GL_COLOR_BUFFER_BIT)

            glEnable(GL_TEXTURE_2D)
            glEnable(GL_BLEND)
            glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
            glBindBuffer (GL_ARRAY_BUFFER, self.vbo)
            glVertexPointer (3, GL_FLOAT, 0, None)
            glBindBuffer(GL_ARRAY_BUFFER, self.vbo_1)
            glTexCoordPointer(2, GL_FLOAT, 0, None)

            glDrawArrays (GL_QUADS, 0, len(self.vertices) * 4)
            glDisable(GL_TEXTURE_2D)
            glDisable(GL_BLEND)

            self.rendering = False

    def add(self, posList, texCoords):
        self.to_add.append((numpy.array(posList), numpy.array(texCoords)))
        return len(self.to_add) - 1

    def make_urgent_update(self, index):
        i = index
        self.renderer.vertices.extend(i[0])
        self.renderer.texCoords.extend(i[1])
        self.renderer.to_add.remove(i)

        glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo)
        glBufferData(GL_ARRAY_BUFFER, len(self.renderer.vertices) * 4, (c_float * len(self.renderer.vertices))(*self.renderer.vertices), GL_STATIC_DRAW)
        glFlush()
        glBindBuffer(GL_ARRAY_BUFFER, self.renderer.vbo_1)
        glBufferData(GL_ARRAY_BUFFER, len(self.renderer.texCoords) * 4, (c_float * len(self.renderer.texCoords))(*self.renderer.texCoords), GL_STATIC_DRAW)
        glFlush()

    def update_vbo(self):
        pass

But now it just shows nothing. It doesn’t show any errors. I tried printing the length of the vertices array, and it does.

I’ve also checked for errors EVERYWHERE, and nothing seems to be going wrong.