From 26571094aa67d2f4829ea0fc1649aed37d8fbaf5 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 12 May 2023 05:03:12 +0200 Subject: kms_vaapi: fix regression, screen rotation not working when capturing screen --- TODO | 3 ++- kms/server/kms_server.c | 35 ++++++++++++++++++++++++++++++++ src/capture/kms_vaapi.c | 53 +++++++++++++++++++++++++++++-------------------- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/TODO b/TODO index e98c227..2cf6b70 100644 --- a/TODO +++ b/TODO @@ -64,4 +64,5 @@ Enable opus/flac again. It's broken right now when merging audio inputs. The aud Support vp8/vp9/av1. This is especially important on amd which on some distros (such as Manjaro) where hardware accelerated h264/hevc is disabled in the mesa package. -Support screen (all monitors) capture on amd/intel when no combined plane is found. \ No newline at end of file +Support screen (all monitors) capture on amd/intel when no combined plane is found. +Use separate plane (which has offset and pitch) from combined plane instead of the combined plane. \ No newline at end of file diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c index 76bf718..3c6e2d5 100644 --- a/kms/server/kms_server.c +++ b/kms/server/kms_server.c @@ -14,6 +14,7 @@ #include #include +#include #define MAX_CONNECTORS 32 @@ -85,6 +86,37 @@ static bool connector_get_property_by_name(int drmfd, drmModeConnectorPtr props, return false; } +static bool plane_is_cursor_plane(int drmfd, uint32_t plane_id) { + drmModeObjectPropertiesPtr props = drmModeObjectGetProperties(drmfd, plane_id, DRM_MODE_OBJECT_PLANE); + if(!props) + return false; + + for(uint32_t i = 0; i < props->count_props; ++i) { + drmModePropertyPtr prop = drmModeGetProperty(drmfd, props->props[i]); + if(prop) { + if(strcmp(prop->name, "type") == 0) { + const uint64_t current_enum_value = props->prop_values[i]; + bool is_cursor = false; + + for(int j = 0; j < prop->count_enums; ++j) { + if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) { + is_cursor = true; + break; + } + + } + + drmModeFreeProperty(prop); + return is_cursor; + } + drmModeFreeProperty(prop); + } + } + + drmModeFreeObjectProperties(props); + return false; +} + /* Returns 0 if not found */ static uint32_t get_connector_by_crtc_id(const connector_to_crtc_map *c2crtc_map, uint32_t crtc_id) { for(int i = 0; i < c2crtc_map->num_maps; ++i) { @@ -147,6 +179,9 @@ static int kms_get_plane_ids(gsr_drm *drm) { continue; } + if(plane_is_cursor_plane(drm->drmfd, plane->plane_id)) + continue; + // TODO: Fallback to getfb(1)? drmModeFB2Ptr drmfb = drmModeGetFB2(drm->drmfd, plane->fb_id); if(drmfb) { diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c index 31cc492..765da68 100644 --- a/src/capture/kms_vaapi.c +++ b/src/capture/kms_vaapi.c @@ -45,6 +45,7 @@ typedef struct { gsr_kms_client kms_client; gsr_kms_response kms_response; + vec2i screen_size; vec2i capture_pos; vec2i capture_size; bool screen_capture; @@ -136,6 +137,10 @@ static bool properties_has_atom(Atom *props, int nprop, Atom atom) { static void monitor_callback(const XRROutputInfo *output_info, const XRRCrtcInfo *crt_info, const XRRModeInfo *mode_info, void *userdata) { MonitorCallbackUserdata *monitor_callback_userdata = userdata; ++monitor_callback_userdata->num_monitors; + + if(strcmp(monitor_callback_userdata->monitor_to_capture, "screen") == 0) + monitor_callback_userdata->rotation = crt_info->rotation; + if(monitor_callback_userdata->monitor_to_capture_len != output_info->nameLen || memcmp(monitor_callback_userdata->monitor_to_capture, output_info->name, output_info->nameLen) != 0) return; @@ -190,12 +195,14 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c }; for_each_active_monitor_output(cap_kms->dpy, monitor_callback, &monitor_callback_userdata); + cap_kms->screen_size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy)); + cap_kms->screen_size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy)); + gsr_monitor monitor; if(strcmp(cap_kms->params.display_to_capture, "screen") == 0) { monitor.pos.x = 0; monitor.pos.y = 0; - monitor.size.x = WidthOfScreen(DefaultScreenOfDisplay(cap_kms->dpy)); - monitor.size.y = HeightOfScreen(DefaultScreenOfDisplay(cap_kms->dpy)); + monitor.size = cap_kms->screen_size; cap_kms->screen_capture = true; } else if(!get_monitor_by_name(cap_kms->dpy, cap_kms->params.display_to_capture, &monitor)) { fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_start: failed to find monitor by name \"%s\"\n", cap_kms->params.display_to_capture); @@ -412,6 +419,22 @@ static gsr_kms_response_fd* find_first_combined_drm(gsr_kms_response *kms_respon return NULL; } +static gsr_kms_response_fd* find_largest_drm(gsr_kms_response *kms_response) { + if(kms_response->num_fds == 0) + return NULL; + + int64_t largest_size = 0; + gsr_kms_response_fd *largest_drm = &kms_response->fds[0]; + for(int i = 0; i < kms_response->num_fds; ++i) { + const int64_t size = (int64_t)kms_response->fds[i].width * (int64_t)kms_response->fds[i].height; + if(size > largest_size) { + largest_size = size; + largest_drm = &kms_response->fds[i]; + } + } + return largest_drm; +} + static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) { gsr_capture_kms_vaapi *cap_kms = cap->priv; @@ -442,16 +465,9 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) { gsr_kms_response_fd *drm_fd = NULL; if(cap_kms->screen_capture) { drm_fd = find_first_combined_drm(&cap_kms->kms_response); - if(drm_fd) { - capture_is_combined_plane = true; - } else { - static bool error_shown = false; - if(!error_shown) { - error_shown = true; - fprintf(stderr, "gsr warning: no combined drm found, screen capture will capture the first monitor found instead\n"); - } - drm_fd = &cap_kms->kms_response.fds[0]; - } + if(!drm_fd) + drm_fd = find_largest_drm(&cap_kms->kms_response); + capture_is_combined_plane = drm_fd->is_combined_plane || ((int)drm_fd->width == cap_kms->screen_size.x && (int)drm_fd->height == cap_kms->screen_size.y); } else { for(int i = 0; i < cap_kms->monitor_id.num_connector_ids; ++i) { drm_fd = find_drm_by_connector_id(&cap_kms->kms_response, cap_kms->monitor_id.connector_ids[i]); @@ -464,16 +480,9 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) { if(!drm_fd) { drm_fd = find_first_combined_drm(&cap_kms->kms_response); - if(drm_fd) { - capture_is_combined_plane = true; - } else { - static bool error_shown = false; - if(!error_shown) { - error_shown = true; - fprintf(stderr, "gsr error: no drm found for monitor and no combined drm found, capture will fail\n"); - } - return -1; - } + if(!drm_fd) + drm_fd = find_largest_drm(&cap_kms->kms_response); + capture_is_combined_plane = drm_fd->is_combined_plane || ((int)drm_fd->width == cap_kms->screen_size.x && (int)drm_fd->height == cap_kms->screen_size.y); } } -- cgit v1.2.3