From 37f30c5ba5f670e087b206e80da622dcff09d903 Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Thu, 16 Jul 2020 13:17:34 +0200
Subject: Fix window resize memory leak (nvidia is not cleaning up on
 glDeleteTextures)

---
 window_texture.c | 57 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 19 deletions(-)

(limited to 'window_texture.c')

diff --git a/window_texture.c b/window_texture.c
index 96e82f7..fd3e6ca 100644
--- a/window_texture.c
+++ b/window_texture.c
@@ -27,8 +27,8 @@ int window_texture_init(WindowTexture *window_texture, Display *display, Window
     return window_texture_on_resize(window_texture);
 }
 
-static void window_texture_cleanup(WindowTexture *self) {
-    if(self->texture_id) {
+static void window_texture_cleanup(WindowTexture *self, int delete_texture) {
+    if(delete_texture && self->texture_id) {
         glDeleteTextures(1, &self->texture_id);
         self->texture_id = 0;
     }
@@ -47,13 +47,18 @@ static void window_texture_cleanup(WindowTexture *self) {
 
 void window_texture_deinit(WindowTexture *self) {
     XCompositeUnredirectWindow(self->display, self->window, CompositeRedirectAutomatic);
-    window_texture_cleanup(self);
+    window_texture_cleanup(self, 1);
 }
 
 int window_texture_on_resize(WindowTexture *self) {
-    window_texture_cleanup(self);
+    window_texture_cleanup(self, 0);
 
     int result = 0;
+    GLXFBConfig *configs = NULL;
+    Pixmap pixmap = None;
+    GLXPixmap glx_pixmap = None;
+    GLuint texture_id = 0;
+    int glx_pixmap_bound = 0;
 
     const int pixmap_config[] = {
         GLX_BIND_TO_TEXTURE_RGBA_EXT, True,
@@ -79,15 +84,15 @@ int window_texture_on_resize(WindowTexture *self) {
         return 1;
     }
 
+    GLXFBConfig config;
     int c;
-    GLXFBConfig *configs = glXChooseFBConfig(self->display, 0, pixmap_config, &c);
+    configs = glXChooseFBConfig(self->display, 0, pixmap_config, &c);
     if(!configs) {
         fprintf(stderr, "Failed to choose fb config\n");
         return 1;
     }
 
     int found = 0;
-    GLXFBConfig config;
     for (int i = 0; i < c; i++) {
         config = configs[i];
         XVisualInfo *visual = glXGetVisualFromFBConfig(self->display, config);
@@ -109,33 +114,36 @@ int window_texture_on_resize(WindowTexture *self) {
         goto cleanup;
     }
 
-    self->pixmap = XCompositeNameWindowPixmap(self->display, self->window);
-    if(!self->pixmap) {
+    pixmap = XCompositeNameWindowPixmap(self->display, self->window);
+    if(!pixmap) {
         result = 2;
         goto cleanup;
     }
 
-    self->glx_pixmap = glXCreatePixmap(self->display, config, self->pixmap, pixmap_attribs);
-    if(!self->glx_pixmap) {
+    glx_pixmap = glXCreatePixmap(self->display, config, pixmap, pixmap_attribs);
+    if(!glx_pixmap) {
         result = 3;
         goto cleanup;
     }
 
-    glGenTextures(1, &self->texture_id);
-    glBindTexture(GL_TEXTURE_2D, self->texture_id);
     if(self->texture_id == 0) {
-        result = 4;
-        goto cleanup;
+        glGenTextures(1, &texture_id);
+        if(texture_id == 0) {
+            result = 4;
+            goto cleanup;
+        }
+        glBindTexture(GL_TEXTURE_2D, texture_id);
+    } else {
+        glBindTexture(GL_TEXTURE_2D, self->texture_id);
     }
 
-    glXBindTexImageEXT(self->display, self->glx_pixmap, GLX_FRONT_EXT, NULL);
-    /*glGenerateMipmap(GL_TEXTURE_2D)*/
+    glXBindTexImageEXT(self->display, glx_pixmap, GLX_FRONT_EXT, NULL);
+    glx_pixmap_bound = 1;
 
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE );
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE );
     glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
-    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);//GL_LINEAR_MIPMAP_LINEAR );
-    glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
+    glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
  
     float fLargest = 0.0f;
     glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY_EXT, &fLargest);
@@ -143,8 +151,19 @@ int window_texture_on_resize(WindowTexture *self) {
 
     glBindTexture(GL_TEXTURE_2D, 0);
 
-    cleanup:
     XFree(configs);
+    self->pixmap = pixmap;
+    self->glx_pixmap = glx_pixmap;
+    if(texture_id != 0)
+        self->texture_id = texture_id;
+    return 0;
+
+    cleanup:
+    if(texture_id != 0)     glDeleteTextures(1, &texture_id);
+    if(glx_pixmap)          glXDestroyPixmap(self->display, self->glx_pixmap);
+    if(glx_pixmap_bound)    glXReleaseTexImageEXT(self->display, self->glx_pixmap, GLX_FRONT_EXT);
+    if(pixmap)              XFreePixmap(self->display, self->pixmap);
+    if(configs)             XFree(configs);
     return result;
 }
 
-- 
cgit v1.2.3