egl: fix transparent window

5 files changed, 111 insertions, 53 deletions

diff --git a/include/mgl/gl.h b/include/mgl/gl.h
index 397d608..dace8ef 100644
--- a/include/mgl/gl.h
+++ b/include/mgl/gl.h
@@ -39,7 +39,7 @@ typedef struct {
     EGLDisplay (*eglGetDisplay)(EGLNativeDisplayType display_id);
     unsigned int (*eglInitialize)(EGLDisplay dpy, int32_t *major, int32_t *minor);
     unsigned int (*eglTerminate)(EGLDisplay dpy);
-    unsigned int (*eglChooseConfig)(EGLDisplay dpy, const int32_t *attrib_list, EGLConfig *configs, int32_t config_size, int32_t *num_config);
+    unsigned int (*eglGetConfigs)(EGLDisplay dpy, EGLConfig *configs, int32_t config_size, int32_t *num_config);
     EGLSurface (*eglCreateWindowSurface)(EGLDisplay dpy, EGLConfig config, EGLNativeWindowType win, const int32_t *attrib_list);
     EGLContext (*eglCreateContext)(EGLDisplay dpy, EGLConfig config, EGLContext share_context, const int32_t *attrib_list);
     unsigned int (*eglMakeCurrent)(EGLDisplay dpy, EGLSurface draw, EGLSurface read, EGLContext ctx);
diff --git a/include/mgl/gl_macro.h b/include/mgl/gl_macro.h
index bfeffce..d6b54f0 100644
--- a/include/mgl/gl_macro.h
+++ b/include/mgl/gl_macro.h
@@ -103,5 +103,9 @@
 #define EGL_NONE                                0x3038
 #define EGL_CONTEXT_CLIENT_VERSION              0x3098
 #define EGL_NATIVE_VISUAL_ID                    0x302E
+#define EGL_COLOR_BUFFER_TYPE                   0x303F
+#define EGL_RGB_BUFFER                          0x308E
+#define EGL_SURFACE_TYPE                        0x3033
+#define EGL_WINDOW_BIT                          0x0004
 
 #endif /* MGL_GL_MACRO_H */
diff --git a/src/gl.c b/src/gl.c
index 77e867a..6d2c0fe 100644
--- a/src/gl.c
+++ b/src/gl.c
@@ -134,7 +134,7 @@ static int mgl_gl_load_egl(mgl_gl *self) {
         { (void**)&self->eglGetDisplay, "eglGetDisplay" },
         { (void**)&self->eglInitialize, "eglInitialize" },
         { (void**)&self->eglTerminate, "eglTerminate" },
-        { (void**)&self->eglChooseConfig, "eglChooseConfig" },
+        { (void**)&self->eglGetConfigs, "eglGetConfigs" },
         { (void**)&self->eglCreateWindowSurface, "eglCreateWindowSurface" },
         { (void**)&self->eglCreateContext, "eglCreateContext" },
         { (void**)&self->eglMakeCurrent, "eglMakeCurrent" },
diff --git a/src/mgl.c b/src/mgl.c
index 6fd26a7..5a43427 100644
--- a/src/mgl.c
+++ b/src/mgl.c
@@ -99,16 +99,6 @@ int mgl_init(void) {
 
 void mgl_deinit(void) {
     if(init_count == 1) {
-        if(prev_xioerror) {
-            XSetIOErrorHandler(prev_xioerror);
-            prev_xioerror = NULL;
-        }
-
-        if(prev_xerror) {
-            XSetErrorHandler(prev_xerror);
-            prev_xerror = NULL;
-        }
-
         if(context.connection) {
             XCloseDisplay(context.connection);
             context.connection = NULL;
@@ -120,6 +110,16 @@ void mgl_deinit(void) {
             mgl_gl_unload(&context.gl);
         }
 
+        if(prev_xioerror) {
+            XSetIOErrorHandler(prev_xioerror);
+            prev_xioerror = NULL;
+        }
+
+        if(prev_xerror) {
+            XSetErrorHandler(prev_xerror);
+            prev_xerror = NULL;
+        }
+
         context.current_window = NULL;
     }
 
diff --git a/src/window/window.c b/src/window/window.c
index 5c44fa8..06b5a6b 100644
--- a/src/window/window.c
+++ b/src/window/window.c
@@ -54,6 +54,13 @@ static bool x11_events_circular_buffer_pop(x11_events_circular_buffer *self, mgl
     return true;
 }
 
+static bool xvisual_match_alpha(Display *dpy, XVisualInfo *visual_info, bool alpha) {
+    XRenderPictFormat *pict_format = XRenderFindVisualFormat(dpy, visual_info->visual);
+    if(!pict_format)
+        return false;
+    return (alpha && pict_format->direct.alphaMask > 0) || (!alpha && pict_format->direct.alphaMask == 0);
+}
+
 #define MAX_MONITORS 12
 
 typedef struct {
@@ -88,20 +95,14 @@ static bool glx_context_choose(mgl_context *context, x11_context_glx *glx, bool
         if(!glx->visual_info)
             continue;
 
-        XRenderPictFormat *pict_format = XRenderFindVisualFormat(context->connection, glx->visual_info->visual);
-        if(!pict_format) {
+        if(xvisual_match_alpha(context->connection, glx->visual_info, alpha)) {
+            glx->fbconfig = glx->fbconfigs[i];
+            break;
+        } else {
             XFree(glx->visual_info);
             glx->visual_info = NULL;
-            continue;
+            glx->fbconfig = NULL;
         }
-
-        glx->fbconfig = glx->fbconfigs[i];
-        if((alpha && pict_format->direct.alphaMask > 0) || (!alpha && pict_format->direct.alphaMask == 0))
-            break;
-
-        XFree(glx->visual_info);
-        glx->visual_info = NULL;
-        glx->fbconfig = NULL;
     }
 
     if(!glx->visual_info) {
@@ -203,10 +204,91 @@ typedef struct {
     EGLDisplay   egl_display;
     EGLSurface   egl_surface;
     EGLContext   egl_context;
+    EGLConfig    *configs;
     EGLConfig    ecfg;
     XVisualInfo *visual_info;
 } x11_context_egl;
 
+static int32_t egl_get_config_attrib(x11_context_egl *egl, EGLConfig ecfg, int32_t attribute_name) {
+    mgl_context *context = mgl_get_context();
+    int32_t value = 0;
+    context->gl.eglGetConfigAttrib(egl->egl_display, ecfg, attribute_name, &value);
+    return value;
+}
+
+static bool egl_context_choose(mgl_context *context, x11_context_egl *egl, bool alpha) {
+    egl->configs = NULL;
+    egl->ecfg = NULL;
+    egl->visual_info = NULL;
+
+    int32_t num_configs = 0;
+    context->gl.eglGetConfigs(egl->egl_display, NULL, 0, &num_configs);
+    if(num_configs == 0) {
+        fprintf(stderr, "mgl error: no configs found\n");
+        return false;
+    }
+
+    egl->configs = (EGLConfig*)calloc(num_configs, sizeof(EGLConfig));
+    if(!egl->configs) {
+        fprintf(stderr, "mgl error: failed to allocate %d configs\n", (int)num_configs);
+        return false;
+    }
+
+    context->gl.eglGetConfigs(egl->egl_display, egl->configs, num_configs, &num_configs);
+    for(int i = 0; i < num_configs; i++) {
+        egl->ecfg = egl->configs[i];
+
+        if(egl_get_config_attrib(egl, egl->ecfg, EGL_COLOR_BUFFER_TYPE) != EGL_RGB_BUFFER)
+            continue;
+
+        if(!(egl_get_config_attrib(egl, egl->ecfg, EGL_SURFACE_TYPE) & EGL_WINDOW_BIT))
+            continue;
+
+        if(egl_get_config_attrib(egl, egl->ecfg, EGL_RED_SIZE) != 8)
+            continue;
+
+        if(egl_get_config_attrib(egl, egl->ecfg, EGL_GREEN_SIZE) != 8)
+            continue;
+
+        if(egl_get_config_attrib(egl, egl->ecfg, EGL_BLUE_SIZE) != 8)
+            continue;
+
+        if(egl_get_config_attrib(egl, egl->ecfg, EGL_ALPHA_SIZE) != (alpha ? 8 : 0))
+            continue;
+
+        XVisualInfo vi = {0};
+        vi.visualid = egl_get_config_attrib(egl, egl->ecfg, EGL_NATIVE_VISUAL_ID);
+        if(!vi.visualid)
+            continue;
+
+        int vis_count = 0;
+        egl->visual_info = XGetVisualInfo(context->connection, VisualIDMask, &vi, &vis_count);
+        if(!egl->visual_info)
+            continue;
+
+        if(xvisual_match_alpha(context->connection, egl->visual_info, alpha)) {
+            break;
+        } else {
+            XFree(egl->visual_info);
+            egl->visual_info = NULL;
+            egl->ecfg = NULL;
+        }
+    }
+
+    if(!egl->visual_info) {
+        if(egl->configs) {
+            free(egl->configs);
+            egl->configs = NULL;
+        }
+        egl->ecfg = NULL;
+
+        fprintf(stderr, "mgl error: no appropriate visual found\n");
+        return false;
+    }
+
+    return true;
+}
+
 static void x11_context_egl_deinit(x11_context_egl *self) {
     mgl_context *context = mgl_get_context();
 
@@ -236,20 +318,6 @@ static bool x11_context_egl_init(x11_context_egl *self, bool alpha) {
     mgl_context *context = mgl_get_context();
     memset(self, 0, sizeof(*self));
 
-    int32_t num_config = 0;
-
-    const int32_t attr[] = {
-        EGL_BUFFER_SIZE, alpha ? 32 : 24,
-        EGL_RED_SIZE, 8,
-        EGL_GREEN_SIZE, 8,
-        EGL_BLUE_SIZE, 8,
-        EGL_ALPHA_SIZE, alpha ? 8 : 0,
-        // TODO:
-        //EGL_DEPTH_SIZE, 0,
-        EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT,
-        EGL_NONE, EGL_NONE
-    };
-
     const int32_t ctxattr[] = {
         EGL_CONTEXT_CLIENT_VERSION, 2,
         EGL_NONE, EGL_NONE
@@ -269,10 +337,8 @@ static bool x11_context_egl_init(x11_context_egl *self, bool alpha) {
         x11_context_egl_deinit(self);
         return false;
     }
-    
-    // TODO: Iterate all egl configs (eglGetConfigs) and match visuals to choose the correct one
-    if(!context->gl.eglChooseConfig(self->egl_display, attr, &self->ecfg, 1, &num_config) || num_config != 1) {
-        fprintf(stderr, "gsr error: gsr_egl_create_window failed: failed to find a matching config\n");
+
+    if(!egl_context_choose(context, self, alpha)) {
         x11_context_egl_deinit(self);
         return false;
     }
@@ -284,18 +350,6 @@ static bool x11_context_egl_init(x11_context_egl *self, bool alpha) {
         return false;
     }
 
-    // TODO: Error check
-    int visual_id = 0;
-    context->gl.eglGetConfigAttrib(self->egl_display, self->ecfg, EGL_NATIVE_VISUAL_ID, &visual_id);
-
-    XVisualInfo vi = {0};
-    vi.visualid = visual_id;
-    int count = 0;
-    self->visual_info = XGetVisualInfo(context->connection, VisualIDMask, &vi, &count);
-    if(self->visual_info) {
-        // TODO: Check if the visual really is transparent with XRenderFindVisualFormat and alphaMask value, the same way its done for GLX
-    }
-
     return true;
 }