Support wlroots screen capture, which doesn't require kms root access and also captures cursor

1 files changed, 88 insertions, 44 deletions

diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index fc7da5b..284ec77 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -34,6 +34,8 @@ typedef struct {
     
     gsr_kms_client kms_client;
     gsr_kms_response kms_response;
+    gsr_kms_response_fd wayland_kms_data;
+    bool using_wayland_capture;
 
     vec2i capture_pos;
     vec2i capture_size;
@@ -131,22 +133,39 @@ static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
 static int gsr_capture_kms_cuda_start(gsr_capture *cap, AVCodecContext *video_codec_context) {
     gsr_capture_kms_cuda *cap_kms = cap->priv;
 
-    if(gsr_kms_client_init(&cap_kms->kms_client, cap_kms->params.card_path) != 0) {
+    if(!gsr_egl_load(&cap_kms->egl, NULL, true)) {
+        fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to load opengl\n");
+        gsr_capture_kms_cuda_stop(cap, video_codec_context);
         return -1;
     }
-    cap_kms->monitor_id.num_connector_ids = 0;
-    MonitorCallbackUserdata monitor_callback_userdata = {
-        cap_kms,
-        cap_kms->params.display_to_capture, strlen(cap_kms->params.display_to_capture),
-        0
-    };
-    for_each_active_monitor_output((void*)cap_kms->params.card_path, GSR_CONNECTION_DRM, monitor_callback, &monitor_callback_userdata);
 
     gsr_monitor monitor;
-    if(!get_monitor_by_name((void*)cap_kms->params.card_path, GSR_CONNECTION_DRM, cap_kms->params.display_to_capture, &monitor)) {
-        fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture);
-        gsr_capture_kms_cuda_stop(cap, video_codec_context);
-        return -1;
+    cap_kms->monitor_id.num_connector_ids = 0;
+    if(gsr_egl_start_capture(&cap_kms->egl, cap_kms->params.display_to_capture)) {
+        if(!get_monitor_by_name(&cap_kms->egl, GSR_CONNECTION_WAYLAND, cap_kms->params.display_to_capture, &monitor)) {
+            fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture);
+            gsr_capture_kms_cuda_stop(cap, video_codec_context);
+            return -1;
+        }
+        cap_kms->using_wayland_capture = true;
+    } else {
+        if(gsr_kms_client_init(&cap_kms->kms_client, cap_kms->params.card_path) != 0) {
+            gsr_capture_kms_cuda_stop(cap, video_codec_context);
+            return -1;
+        }
+
+        MonitorCallbackUserdata monitor_callback_userdata = {
+            cap_kms,
+            cap_kms->params.display_to_capture, strlen(cap_kms->params.display_to_capture),
+            0
+        };
+        for_each_active_monitor_output((void*)cap_kms->params.card_path, GSR_CONNECTION_DRM, monitor_callback, &monitor_callback_userdata);
+
+        if(!get_monitor_by_name((void*)cap_kms->params.card_path, GSR_CONNECTION_DRM, cap_kms->params.display_to_capture, &monitor)) {
+            fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture);
+            gsr_capture_kms_cuda_stop(cap, video_codec_context);
+            return -1;
+        }
     }
 
     cap_kms->capture_pos = monitor.pos;
@@ -155,12 +174,6 @@ static int gsr_capture_kms_cuda_start(gsr_capture *cap, AVCodecContext *video_co
     video_codec_context->width = max_int(2, even_number_ceil(cap_kms->capture_size.x));
     video_codec_context->height = max_int(2, even_number_ceil(cap_kms->capture_size.y));
 
-    if(!gsr_egl_load(&cap_kms->egl, NULL, true)) {
-        fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to load opengl\n");
-        gsr_capture_kms_cuda_stop(cap, video_codec_context);
-        return -1;
-    }
-
     /* Disable vsync */
     cap_kms->egl.eglSwapInterval(cap_kms->egl.egl_display, 0);
 
@@ -183,8 +196,6 @@ static uint32_t fourcc(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
     return (d << 24) | (c << 16) | (b << 8) | a;
 }
 
-#define FOURCC_NV12 842094158
-
 static void gsr_capture_kms_cuda_tick(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame) {
     gsr_capture_kms_cuda *cap_kms = cap->priv;
 
@@ -278,8 +289,18 @@ static bool gsr_capture_kms_register_egl_image_in_cuda(gsr_capture_kms_cuda *cap
         return false;
     }
 
+    if(res != CUDA_SUCCESS) {
+        fprintf(stderr, "cuda error 1\n");
+    }
+
     res = cap_kms->cuda.cuGraphicsResourceSetMapFlags(cap_kms->cuda_graphics_resource, CU_GRAPHICS_MAP_RESOURCE_FLAGS_READ_ONLY);
+    if(res != CUDA_SUCCESS) {
+        fprintf(stderr, "cuda error 2\n");
+    }
     res = cap_kms->cuda.cuGraphicsSubResourceGetMappedArray(&cap_kms->mapped_array, cap_kms->cuda_graphics_resource, 0, 0);
+    if(res != CUDA_SUCCESS) {
+        fprintf(stderr, "cuda error 3\n");
+    }
     res = cap_kms->cuda.cuCtxPopCurrent_v2(&old_ctx);
     return true;
 }
@@ -310,31 +331,49 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
     }
     cap_kms->kms_response.num_fds = 0;
 
-    if(gsr_kms_client_get_kms(&cap_kms->kms_client, &cap_kms->kms_response) != 0) {
-        fprintf(stderr, "gsr error: gsr_capture_kms_cuda_capture: failed to get kms, error: %d (%s)\n", cap_kms->kms_response.result, cap_kms->kms_response.err_msg);
-        return -1;
-    }
+    gsr_kms_response_fd *drm_fd = NULL;
+    if(cap_kms->using_wayland_capture) {
+        gsr_egl_update(&cap_kms->egl);
+        cap_kms->wayland_kms_data.fd = cap_kms->egl.fd;
+        cap_kms->wayland_kms_data.width = cap_kms->egl.width;
+        cap_kms->wayland_kms_data.height = cap_kms->egl.height;
+        cap_kms->wayland_kms_data.pitch = cap_kms->egl.pitch;
+        cap_kms->wayland_kms_data.offset = cap_kms->egl.offset;
+        cap_kms->wayland_kms_data.pixel_format = cap_kms->egl.pixel_format;
+        cap_kms->wayland_kms_data.modifier = cap_kms->egl.modifier;
+        cap_kms->wayland_kms_data.connector_id = 0;
+        cap_kms->wayland_kms_data.is_combined_plane = false;
+
+        if(cap_kms->wayland_kms_data.fd <= 0)
+            return -1;
+
+        drm_fd = &cap_kms->wayland_kms_data;
+    } else {
+        if(gsr_kms_client_get_kms(&cap_kms->kms_client, &cap_kms->kms_response) != 0) {
+            fprintf(stderr, "gsr error: gsr_capture_kms_cuda_capture: failed to get kms, error: %d (%s)\n", cap_kms->kms_response.result, cap_kms->kms_response.err_msg);
+            return -1;
+        }
 
-    if(cap_kms->kms_response.num_fds == 0) {
-        static bool error_shown = false;
-        if(!error_shown) {
-            error_shown = true;
-            fprintf(stderr, "gsr error: no drm found, capture will fail\n");
+        if(cap_kms->kms_response.num_fds == 0) {
+            static bool error_shown = false;
+            if(!error_shown) {
+                error_shown = true;
+                fprintf(stderr, "gsr error: no drm found, capture will fail\n");
+            }
+            return -1;
         }
-        return -1;
-    }
 
-    gsr_kms_response_fd *drm_fd = NULL;
-    for(int i = 0; i < cap_kms->monitor_id.num_connector_ids; ++i) {
-        drm_fd = find_drm_by_connector_id(&cap_kms->kms_response, cap_kms->monitor_id.connector_ids[i]);
-        if(drm_fd)
-            break;
-    }
+        for(int i = 0; i < cap_kms->monitor_id.num_connector_ids; ++i) {
+            drm_fd = find_drm_by_connector_id(&cap_kms->kms_response, cap_kms->monitor_id.connector_ids[i]);
+            if(drm_fd)
+                break;
+        }
 
-    if(!drm_fd) {
-        drm_fd = find_first_combined_drm(&cap_kms->kms_response);
-        if(!drm_fd)
-            drm_fd = find_largest_drm(&cap_kms->kms_response);
+        if(!drm_fd) {
+            drm_fd = find_first_combined_drm(&cap_kms->kms_response);
+            if(!drm_fd)
+                drm_fd = find_largest_drm(&cap_kms->kms_response);
+        }
     }
 
     if(!drm_fd)
@@ -342,9 +381,6 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
 
     //bool capture_is_combined_plane = drm_fd->is_combined_plane || ((int)drm_fd->width == cap_kms->screen_size.x && (int)drm_fd->height == cap_kms->screen_size.y);
 
-    //gsr_egl_cleanup_frame(&cap_kms->egl);
-    //gsr_egl_update(&cap_kms->egl);
-
     const intptr_t img_attr[] = {
         //EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
         EGL_LINUX_DRM_FOURCC_EXT,       fourcc('A', 'R', '2', '4'),//cap_kms->egl.pixel_format, ARGB8888
@@ -358,7 +394,13 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
         EGL_NONE
     };
 
+    while(cap_kms->egl.glGetError()) {}
+    while(cap_kms->egl.eglGetError() != EGL_SUCCESS){}
     EGLImage image = cap_kms->egl.eglCreateImage(cap_kms->egl.egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
+    if(cap_kms->egl.glGetError() != 0 || cap_kms->egl.eglGetError() != EGL_SUCCESS) {
+        fprintf(stderr, "egl error!\n");
+    }
+
     gsr_capture_kms_register_egl_image_in_cuda(cap_kms, image);
     cap_kms->egl.eglDestroyImage(cap_kms->egl.egl_display, image);
 
@@ -385,6 +427,8 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
 
     gsr_capture_kms_unload_cuda_graphics(cap_kms);
 
+    gsr_egl_cleanup_frame(&cap_kms->egl);
+
     for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
         if(cap_kms->kms_response.fds[i].fd > 0)
             close(cap_kms->kms_response.fds[i].fd);