Fix wayland capture on amd/intel

6 files changed, 105 insertions, 76 deletions

diff --git a/TODO b/TODO
index 247ee92..83126ee 100644
--- a/TODO
+++ b/TODO
@@ -79,5 +79,7 @@ Test kms_cuda on hyprland and other wlroots based compositor to see if it works.
 Support "screen" (all monitors) capture on wayland. This should be done by getting all drm fds and multiple EGL_DMA_BUF_PLANEX_FD_EXT to create one egl image with all fds combined.
 
 Support pipewire screen capture?
-Support screen rotation in nvidia wayland.
-Support wlroots dmabuf screen recording, because it doesn't require root access unlike kms grab.
\ No newline at end of file
+Support screen rotation in amd/intel/nvidia wayland.
+Support wlroots dmabuf screen recording, because it doesn't require root access unlike kms grab.
+Capture cursor on amd/intel wayland without xwayland.
+When nvidia supports hardware cursor then capture the cursor. Right now the cursor is captured because it's a software cursor so it's composed on the dma buf.
\ No newline at end of file
diff --git a/build.sh b/build.sh
index 5648f6a..4295d3f 100755
--- a/build.sh
+++ b/build.sh
@@ -45,14 +45,14 @@ build_gsr() {
     $CC -c src/cursor.c $opts $includes
     $CC -c src/utils.c $opts $includes
     $CC -c src/library_loader.c $opts $includes
-    $CC -c external/wlr-export-dmabuf-unstable-v1-protocol.c $opts $includes
+    #$CC -c external/wlr-export-dmabuf-unstable-v1-protocol.c $opts $includes
     $CXX -c src/sound.cpp $opts $includes
     $CXX -c src/main.cpp $opts $includes
     $CXX -o gpu-screen-recorder -O2 capture.o nvfbc.o kms_client.o egl.o cuda.o xnvctrl.o overclock.o window_texture.o shader.o \
-        color_conversion.o cursor.o utils.o library_loader.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o wlr-export-dmabuf-unstable-v1-protocol.o sound.o main.o $libs $opts
+        color_conversion.o cursor.o utils.o library_loader.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o sound.o main.o $libs $opts
 }
 
 #build_wayland_protocol
 build_gsr_kms_server
 build_gsr
-echo "Successfully built gpu-screen-recorder"
\ No newline at end of file
+echo "Successfully built gpu-screen-recorder"
diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c
index dc24224..4ed11fa 100644
--- a/kms/server/kms_server.c
+++ b/kms/server/kms_server.c
@@ -92,24 +92,25 @@ static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) {
 
     for(uint32_t i = 0; i < props->count_props; ++i) {
         drmModePropertyPtr prop = drmModeGetProperty(drmfd, props->props[i]);
-        if(prop) {
-            if(strcmp(prop->name, "type") == 0) {
-                const uint64_t current_enum_value = props->prop_values[i];
-                bool is_cursor = false;
-
-                for(int j = 0; j < prop->count_enums; ++j) {
-                    if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) {
-                        is_cursor = true;
-                        break;
-                    }
+        if(!prop)
+            continue;
 
-                }
+        const uint32_t type = prop->flags & (DRM_MODE_PROP_LEGACY_TYPE | DRM_MODE_PROP_EXTENDED_TYPE);
+        if((type & DRM_MODE_PROP_ENUM) && strcmp(prop->name, "type") == 0) {
+            const uint64_t current_enum_value = props->prop_values[i];
+            bool is_cursor = false;
 
-                drmModeFreeProperty(prop);
-                return is_cursor;
+            for(int j = 0; j < prop->count_enums; ++j) {
+                if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) {
+                    is_cursor = true;
+                    break;
+                }
             }
+
             drmModeFreeProperty(prop);
+            return is_cursor;
         }
+        drmModeFreeProperty(prop);
     }
 
     drmModeFreeObjectProperties(props);
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index 7569cd2..b171421 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -119,13 +119,12 @@ static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
     if(monitor_callback_userdata->monitor_to_capture_len != monitor->name_len || memcmp(monitor_callback_userdata->monitor_to_capture, monitor->name, monitor->name_len) != 0)
         return;
 
-    const int connector_index = monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
-    if(connector_index < MAX_CONNECTOR_IDS) {
-        monitor_callback_userdata->cap_kms->monitor_id.connector_ids[connector_index] = monitor->connector_id;
+    if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS) {
+        monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = monitor->connector_id;
         ++monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
     }
 
-    if(connector_index == MAX_CONNECTOR_IDS)
+    if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids == MAX_CONNECTOR_IDS)
         fprintf(stderr, "gsr warning: reached max connector ids\n");
 }
 
diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c
index 908b537..2c836b3 100644
--- a/src/capture/kms_vaapi.c
+++ b/src/capture/kms_vaapi.c
@@ -119,11 +119,12 @@ static bool drm_create_codec_context(gsr_capture_kms_vaapi *cap_kms, AVCodecCont
 
 typedef struct {
     gsr_capture_kms_vaapi *cap_kms;
-    const Atom randr_connector_id_atom;
+    Atom randr_connector_id_atom;
     const char *monitor_to_capture;
     int monitor_to_capture_len;
     int num_monitors;
     int rotation;
+    bool wayland;
 } MonitorCallbackUserdata;
 
 static bool properties_has_atom(Atom *props, int nprop, Atom atom) {
@@ -135,43 +136,51 @@ static bool properties_has_atom(Atom *props, int nprop, Atom atom) {
 }
 
 static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
+    (void)monitor;
     MonitorCallbackUserdata *monitor_callback_userdata = userdata;
     ++monitor_callback_userdata->num_monitors;
 
-    if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0)
-        monitor_callback_userdata->rotation = monitor->crt_info->rotation;
-
     if(monitor_callback_userdata->monitor_to_capture_len != monitor->name_len || memcmp(monitor_callback_userdata->monitor_to_capture, monitor->name, monitor->name_len) != 0)
         return;
 
-    monitor_callback_userdata->rotation = monitor->crt_info->rotation;
-    for(int i = 0; i < monitor->crt_info->noutput && monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS; ++i) {
-        int nprop = 0;
-        Atom *props = XRRListOutputProperties(monitor_callback_userdata->cap_kms->dpy, monitor->crt_info->outputs[i], &nprop);
-        if(!props)
-            continue;
-
-        if(!properties_has_atom(props, nprop, monitor_callback_userdata->randr_connector_id_atom)) {
-            XFree(props);
-            continue;
-        }
-
-        Atom type = 0;
-        int format = 0;
-        unsigned long bytes_after = 0;
-        unsigned long nitems = 0;
-        unsigned char *prop = NULL;
-        XRRGetOutputProperty(monitor_callback_userdata->cap_kms->dpy, monitor->crt_info->outputs[i],
-            monitor_callback_userdata->randr_connector_id_atom,
-            0, 128, false, false, AnyPropertyType,
-            &type, &format, &nitems, &bytes_after, &prop);
-
-        if(type == XA_INTEGER && format == 32) {
-            monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = *(long*)prop;
+    if(monitor_callback_userdata->wayland) {
+        if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS) {
+            monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = monitor->connector_id;
             ++monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
         }
+    } else {
+        if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0)
+            monitor_callback_userdata->rotation = monitor->crt_info->rotation;
+
+        monitor_callback_userdata->rotation = monitor->crt_info->rotation;
+        for(int i = 0; i < monitor->crt_info->noutput && monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids < MAX_CONNECTOR_IDS; ++i) {
+            int nprop = 0;
+            Atom *props = XRRListOutputProperties(monitor_callback_userdata->cap_kms->dpy, monitor->crt_info->outputs[i], &nprop);
+            if(!props)
+                continue;
+
+            if(!properties_has_atom(props, nprop, monitor_callback_userdata->randr_connector_id_atom)) {
+                XFree(props);
+                continue;
+            }
+
+            Atom type = 0;
+            int format = 0;
+            unsigned long bytes_after = 0;
+            unsigned long nitems = 0;
+            unsigned char *prop = NULL;
+            XRRGetOutputProperty(monitor_callback_userdata->cap_kms->dpy, monitor->crt_info->outputs[i],
+                monitor_callback_userdata->randr_connector_id_atom,
+                0, 128, false, false, AnyPropertyType,
+                &type, &format, &nitems, &bytes_after, &prop);
+
+            if(type == XA_INTEGER && format == 32) {
+                monitor_callback_userdata->cap_kms->monitor_id.connector_ids[monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids] = *(long*)prop;
+                ++monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids;
+            }
 
-        XFree(props);
+            XFree(props);
+        }
     }
 
     if(monitor_callback_userdata->cap_kms->monitor_id.num_connector_ids == MAX_CONNECTOR_IDS)
@@ -188,18 +197,30 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
     void *connection = cap_kms->params.wayland ? (void*)cap_kms->params.card_path : (void*)cap_kms->dpy;
     const gsr_connection_type connection_type = cap_kms->params.wayland ? GSR_CONNECTION_DRM : GSR_CONNECTION_X11;
 
-    const Atom randr_connector_id_atom = XInternAtom(cap_kms->dpy, "CONNECTOR_ID", False);
-    cap_kms->monitor_id.num_connector_ids = 0;
     MonitorCallbackUserdata monitor_callback_userdata = {
-        cap_kms, randr_connector_id_atom,
+        cap_kms, None,
         cap_kms->params.display_to_capture, strlen(cap_kms->params.display_to_capture),
         0,
-        X11_ROT_0
+        X11_ROT_0,
+        true
     };
-    for_each_active_monitor_output(connection, connection_type, monitor_callback, &monitor_callback_userdata);
 
-    cap_kms->screen_size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
-    cap_kms->screen_size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
+    if(cap_kms->params.wayland) {
+        cap_kms->monitor_id.num_connector_ids = 0;
+        for_each_active_monitor_output(connection, connection_type, monitor_callback, &monitor_callback_userdata);
+
+        cap_kms->screen_size.x = 0;
+        cap_kms->screen_size.y = 0;
+    } else {
+        const Atom randr_connector_id_atom = XInternAtom(cap_kms->dpy, "CONNECTOR_ID", False);
+        cap_kms->monitor_id.num_connector_ids = 0;
+        monitor_callback_userdata.randr_connector_id_atom = randr_connector_id_atom;
+        monitor_callback_userdata.wayland = false;
+        for_each_active_monitor_output(connection, connection_type, monitor_callback, &monitor_callback_userdata);
+
+        cap_kms->screen_size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
+        cap_kms->screen_size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy));
+    }
 
     gsr_monitor monitor;
     if(strcmp(cap_kms->params.display_to_capture, "screen") == 0) {
@@ -244,13 +265,15 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
         return -1;
     }
 
-    if(gsr_cursor_init(&cap_kms->cursor, &cap_kms->egl, cap_kms->dpy) != 0) {
-        gsr_capture_kms_vaapi_stop(cap, video_codec_context);
-        return -1;
-    }
+    if(cap_kms->dpy) {
+        if(gsr_cursor_init(&cap_kms->cursor, &cap_kms->egl, cap_kms->dpy) != 0) {
+            gsr_capture_kms_vaapi_stop(cap, video_codec_context);
+            return -1;
+        }
 
-    gsr_cursor_change_window_target(&cap_kms->cursor, DefaultRootWindow(cap_kms->dpy));
-    gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
+        gsr_cursor_change_window_target(&cap_kms->cursor, DefaultRootWindow(cap_kms->dpy));
+        gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
+    }
 
     return 0;
 }
@@ -267,9 +290,11 @@ static void gsr_capture_kms_vaapi_tick(gsr_capture *cap, AVCodecContext *video_c
     // TODO:
     cap_kms->egl.glClear(GL_COLOR_BUFFER_BIT);
 
-    while(XPending(cap_kms->dpy)) {
-        XNextEvent(cap_kms->dpy, &cap_kms->xev);
-        gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
+    if(cap_kms->dpy) {
+        while(XPending(cap_kms->dpy)) {
+            XNextEvent(cap_kms->dpy, &cap_kms->xev);
+            gsr_cursor_update(&cap_kms->cursor, &cap_kms->xev);
+        }
     }
 
     if(!cap_kms->created_hw_frame) {
@@ -596,14 +621,16 @@ static void gsr_capture_kms_vaapi_stop(gsr_capture *cap, AVCodecContext *video_c
         }
     }
 
-    if(cap_kms->input_texture) {
-        cap_kms->egl.glDeleteTextures(1, &cap_kms->input_texture);
-        cap_kms->input_texture = 0;
-    }
+    if(cap_kms->egl.egl_context) {
+        if(cap_kms->input_texture) {
+            cap_kms->egl.glDeleteTextures(1, &cap_kms->input_texture);
+            cap_kms->input_texture = 0;
+        }
 
-    cap_kms->egl.glDeleteTextures(2, cap_kms->target_textures);
-    cap_kms->target_textures[0] = 0;
-    cap_kms->target_textures[1] = 0;
+        cap_kms->egl.glDeleteTextures(2, cap_kms->target_textures);
+        cap_kms->target_textures[0] = 0;
+        cap_kms->target_textures[1] = 0;
+    }
 
     for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
         if(cap_kms->kms_response.fds[i].fd > 0)
@@ -656,7 +683,7 @@ gsr_capture* gsr_capture_kms_vaapi_create(const gsr_capture_kms_vaapi_params *pa
     }
 
     Display *display = XOpenDisplay(NULL);
-    if(!display) {
+    if(!params->wayland && !display) {
         fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_create failed: XOpenDisplay failed\n");
         free(cap);
         free(cap_kms);
diff --git a/src/main.cpp b/src/main.cpp
index 212aff5..7723bc2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1449,7 +1449,7 @@ int main(int argc, char **argv) {
             kms_params.display_to_capture = capture_target;
             kms_params.gpu_inf = gpu_inf;
             kms_params.card_path = card_path;
-            kms_params.wayland = false;//wayland;
+            kms_params.wayland = wayland;
             capture = gsr_capture_kms_vaapi_create(&kms_params);
             if(!capture)
                 _exit(1);