Fix pipewire capture on amd (fallback to no drm modifiers if eglCreateImage fails), might happen because of a buggy desktop portal/pipewire implementation

4 files changed, 39 insertions, 6 deletions

diff --git a/TODO b/TODO
index 88e90e8..8556754 100644
--- a/TODO
+++ b/TODO
@@ -157,3 +157,5 @@ HDR support on x11?
 Move most kms data to kms client. We dont need root access for everything that is server from kms server right now, such as hdr metadata and drm plane properties. Only the drm plane fd really needs root access.
 
 Show rotated window size in monitor list when using incorrect monitor name.
+
+Desktop portal capture on kde plasma makes notifications not show up unless the notification is set as urgent. How to fix this? do we have to make our own notification system?
diff --git a/include/pipewire.h b/include/pipewire.h
index 4e486fc..4a6330d 100644
--- a/include/pipewire.h
+++ b/include/pipewire.h
@@ -84,6 +84,7 @@ typedef struct {
 
     bool started;
     bool stopped;
+    bool no_modifiers_fallback;
 
     uint64_t modifiers[GSR_PIPEWIRE_MAX_MODIFIERS];
     size_t num_modifiers;
diff --git a/src/capture/kms.c b/src/capture/kms.c
index 7fa69c6..a148169 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -43,6 +43,8 @@ typedef struct {
 
     unsigned int input_texture_id;
     unsigned int cursor_texture_id;
+
+    bool no_modifiers_fallback;
 } gsr_capture_kms;
 
 static void gsr_capture_kms_cleanup_kms_fds(gsr_capture_kms *self) {
@@ -399,11 +401,23 @@ static int gsr_capture_kms_capture(gsr_capture *cap, AVStream *video_stream, AVF
         modifiers[i] = drm_fd->modifier;
     }
 
+    EGLImage image = NULL;
     intptr_t img_attr[44];
-    setup_dma_buf_attrs(img_attr, drm_fd->pixel_format, drm_fd->width, drm_fd->height,
-        fds, offsets, pitches, modifiers, drm_fd->num_dma_bufs, true);
 
-    EGLImage image = self->params.egl->eglCreateImage(self->params.egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+    if(self->no_modifiers_fallback) {
+        setup_dma_buf_attrs(img_attr, drm_fd->pixel_format, drm_fd->width, drm_fd->height, fds, offsets, pitches, modifiers, drm_fd->num_dma_bufs, false);
+    } else {
+        setup_dma_buf_attrs(img_attr, drm_fd->pixel_format, drm_fd->width, drm_fd->height, fds, offsets, pitches, modifiers, drm_fd->num_dma_bufs, true);
+        while(self->params.egl->eglGetError() != EGL_SUCCESS){}
+        image = self->params.egl->eglCreateImage(self->params.egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+        if(!image || self->params.egl->eglGetError() != EGL_SUCCESS) {
+            fprintf(stderr, "gsr error: gsr_capture_kms_capture: failed to create egl image with modifiers, trying without modifiers\n");
+            self->no_modifiers_fallback = true;
+            setup_dma_buf_attrs(img_attr, drm_fd->pixel_format, drm_fd->width, drm_fd->height, fds, offsets, pitches, modifiers, drm_fd->num_dma_bufs, false);
+            image = self->params.egl->eglCreateImage(self->params.egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+        }
+    }
+
     self->params.egl->glBindTexture(GL_TEXTURE_2D, self->input_texture_id);
     self->params.egl->glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);
     self->params.egl->eglDestroyImage(self->params.egl->egl_display, image);
diff --git a/src/pipewire.c b/src/pipewire.c
index db3122a..fae83cd 100644
--- a/src/pipewire.c
+++ b/src/pipewire.c
@@ -643,11 +643,27 @@ bool gsr_pipewire_map_texture(gsr_pipewire *self, unsigned int texture_id, unsig
         modifiers[i] = self->format.info.raw.modifier;
     }
 
+    EGLImage image = NULL;
     intptr_t img_attr[44];
-    setup_dma_buf_attrs(img_attr, spa_video_format_to_drm_format(self->format.info.raw.format), self->format.info.raw.size.width, self->format.info.raw.size.height,
-        fds, offsets, pitches, modifiers, self->dmabuf_num_planes, true);
 
-    EGLImage image = self->egl->eglCreateImage(self->egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+    if(self->no_modifiers_fallback) {
+        setup_dma_buf_attrs(img_attr, spa_video_format_to_drm_format(self->format.info.raw.format), self->format.info.raw.size.width, self->format.info.raw.size.height,
+            fds, offsets, pitches, modifiers, self->dmabuf_num_planes, false);
+    } else {
+        setup_dma_buf_attrs(img_attr, spa_video_format_to_drm_format(self->format.info.raw.format), self->format.info.raw.size.width, self->format.info.raw.size.height,
+            fds, offsets, pitches, modifiers, self->dmabuf_num_planes, true);
+
+        while(self->egl->eglGetError() != EGL_SUCCESS){}
+        image = self->egl->eglCreateImage(self->egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+        if(!image || self->egl->eglGetError() != EGL_SUCCESS) {
+            fprintf(stderr, "gsr error: gsr_pipewire_map_texture: failed to create egl image with modifiers, trying without modifiers\n");
+            self->no_modifiers_fallback = true;
+            setup_dma_buf_attrs(img_attr, spa_video_format_to_drm_format(self->format.info.raw.format), self->format.info.raw.size.width, self->format.info.raw.size.height,
+                fds, offsets, pitches, modifiers, self->dmabuf_num_planes, false);
+            image = self->egl->eglCreateImage(self->egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+        }
+    }
+
     self->egl->glBindTexture(GL_TEXTURE_2D, texture_id);
     self->egl->glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);
     self->egl->eglDestroyImage(self->egl->egl_display, image);