Fix window resize memory leak (nvidia is not cleaning up on glDeleteTextures)

2 files changed, 50 insertions, 24 deletions

diff --git a/src/main.cpp b/src/main.cpp
index 22ad039..5e4cd74 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -568,6 +568,10 @@ CMainApplication::CMainApplication( int argc, char *argv[] )
 
 	cursor_scale_uniform[0] = 0.0f;
 	cursor_scale_uniform[1] = 0.0f;
+
+#ifdef DEBUG
+	m_bDebugOpenGL = true;
+#endif
 };
 
 
@@ -808,9 +812,9 @@ bool CMainApplication::BInitGL()
 {
 	if( m_bDebugOpenGL )
 	{
+		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS);
 		glDebugMessageCallback( (GLDEBUGPROC)DebugCallback, nullptr);
 		glDebugMessageControl( GL_DONT_CARE, GL_DONT_CARE, GL_DONT_CARE, 0, nullptr, GL_TRUE );
-		glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS);
 	}
 
 	if( !CreateAllShaders() )
@@ -845,9 +849,8 @@ bool CMainApplication::BInitGL()
 
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE );
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE );
-    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR_MIPMAP_LINEAR);
+    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
-    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
  
     float fLargest = 0.0f;
     glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY_EXT, &fLargest);
@@ -985,6 +988,7 @@ bool CMainApplication::HandleInput()
 {
 	SDL_Event sdlEvent;
 	bool bRet = false;
+	bool zoom_resize = false;
 
 	while ( SDL_PollEvent( &sdlEvent ) != 0 )
 	{
@@ -1005,7 +1009,7 @@ bool CMainApplication::HandleInput()
 			if( sdlEvent.key.keysym.sym == SDLK_q )
 			{
 				zoom -= 0.01f;
-				window_resized = true;
+				zoom_resize = true;
 
 				std::stringstream strstr;
 				strstr << "/tmp/vr-video-player_" << src_window_id;
@@ -1015,7 +1019,7 @@ bool CMainApplication::HandleInput()
 			if( sdlEvent.key.keysym.sym == SDLK_e )
 			{
 				zoom += 0.01f;
-				window_resized = true;
+				zoom_resize = true;
 
 				std::stringstream strstr;
 				strstr << "/tmp/vr-video-player_" << src_window_id;
@@ -1055,6 +1059,9 @@ bool CMainApplication::HandleInput()
 		SetupScene();
 	}
 
+	if(!window_resized && zoom_resize)
+		SetupScene();
+
 	// Process SteamVR events
 	vr::VREvent_t event;
 	while( m_pHMD->PollNextEvent( &event, sizeof( event ) ) )
diff --git a/src/window_texture.c b/src/window_texture.c
index a69b191..f1818c9 100644
--- a/src/window_texture.c
+++ b/src/window_texture.c
@@ -27,8 +27,8 @@ int window_texture_init(WindowTexture *window_texture, Display *display, Window
     return window_texture_on_resize(window_texture);
 }
 
-static void window_texture_cleanup(WindowTexture *self) {
-    if(self->texture_id) {
+static void window_texture_cleanup(WindowTexture *self, int delete_texture) {
+    if(delete_texture && self->texture_id) {
         glDeleteTextures(1, &self->texture_id);
         self->texture_id = 0;
     }
@@ -47,13 +47,18 @@ static void window_texture_cleanup(WindowTexture *self) {
 
 void window_texture_deinit(WindowTexture *self) {
     XCompositeUnredirectWindow(self->display, self->window, CompositeRedirectAutomatic);
-    window_texture_cleanup(self);
+    window_texture_cleanup(self, 1);
 }
 
 int window_texture_on_resize(WindowTexture *self) {
-    window_texture_cleanup(self);
+    window_texture_cleanup(self, 0);
 
     int result = 0;
+    GLXFBConfig *configs = NULL;
+    Pixmap pixmap = None;
+    GLXPixmap glx_pixmap = None;
+    GLuint texture_id = 0;
+    int glx_pixmap_bound = 0;
 
     const int pixmap_config[] = {
         GLX_BIND_TO_TEXTURE_RGBA_EXT, True,
@@ -79,15 +84,15 @@ int window_texture_on_resize(WindowTexture *self) {
         return 1;
     }
 
+    GLXFBConfig config;
     int c;
-    GLXFBConfig *configs = glXChooseFBConfig(self->display, 0, pixmap_config, &c);
+    configs = glXChooseFBConfig(self->display, 0, pixmap_config, &c);
     if(!configs) {
         fprintf(stderr, "Failed to choose fb config\n");
         return 1;
     }
 
     int found = 0;
-    GLXFBConfig config;
     for (int i = 0; i < c; i++) {
         config = configs[i];
         XVisualInfo *visual = glXGetVisualFromFBConfig(self->display, config);
@@ -109,33 +114,36 @@ int window_texture_on_resize(WindowTexture *self) {
         goto cleanup;
     }
 
-    self->pixmap = XCompositeNameWindowPixmap(self->display, self->window);
-    if(!self->pixmap) {
+    pixmap = XCompositeNameWindowPixmap(self->display, self->window);
+    if(!pixmap) {
         result = 2;
         goto cleanup;
     }
 
-    self->glx_pixmap = glXCreatePixmap(self->display, config, self->pixmap, pixmap_attribs);
-    if(!self->glx_pixmap) {
+    glx_pixmap = glXCreatePixmap(self->display, config, pixmap, pixmap_attribs);
+    if(!glx_pixmap) {
         result = 3;
         goto cleanup;
     }
 
-    glGenTextures(1, &self->texture_id);
-    glBindTexture(GL_TEXTURE_2D, self->texture_id);
     if(self->texture_id == 0) {
-        result = 4;
-        goto cleanup;
+        glGenTextures(1, &texture_id);
+        if(texture_id == 0) {
+            result = 4;
+            goto cleanup;
+        }
+        glBindTexture(GL_TEXTURE_2D, texture_id);
+    } else {
+        glBindTexture(GL_TEXTURE_2D, self->texture_id);
     }
 
-    glXBindTexImageEXT(self->display, self->glx_pixmap, GLX_FRONT_EXT, NULL);
-    /*glGenerateMipmap(GL_TEXTURE_2D)*/
+    glXBindTexImageEXT(self->display, glx_pixmap, GLX_FRONT_EXT, NULL);
+    glx_pixmap_bound = 1;
 
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE );
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE );
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
-    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);//GL_LINEAR_MIPMAP_LINEAR );
-    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
+    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
  
     float fLargest = 0.0f;
     glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY_EXT, &fLargest);
@@ -143,8 +151,19 @@ int window_texture_on_resize(WindowTexture *self) {
 
     glBindTexture(GL_TEXTURE_2D, 0);
 
-    cleanup:
     XFree(configs);
+    self->pixmap = pixmap;
+    self->glx_pixmap = glx_pixmap;
+    if(texture_id != 0)
+        self->texture_id = texture_id;
+    return 0;
+
+    cleanup:
+    if(texture_id != 0)     glDeleteTextures(1, &texture_id);
+    if(glx_pixmap)          glXDestroyPixmap(self->display, self->glx_pixmap);
+    if(glx_pixmap_bound)    glXReleaseTexImageEXT(self->display, self->glx_pixmap, GLX_FRONT_EXT);
+    if(pixmap)              XFreePixmap(self->display, self->pixmap);
+    if(configs)             XFree(configs);
     return result;
 }