Refactor kms_vaapi and kms_cuda

Also fixes color metadata, color range and hdr on nvidia wayland.

1 files changed, 15 insertions, 141 deletions

diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c
index d32dd39..941467a 100644
--- a/src/capture/xcomposite_vaapi.c
+++ b/src/capture/xcomposite_vaapi.c
@@ -14,13 +14,13 @@
 #include <va/va_drmcommon.h>
 
 typedef struct {
+    gsr_capture_base base;
     gsr_capture_xcomposite_vaapi_params params;
     XEvent xev;
 
     bool should_stop;
     bool stop_is_error;
     bool window_resized;
-    bool created_hw_frame;
     bool follow_focused_initialized;
 
     Window window;
@@ -33,10 +33,6 @@ typedef struct {
     VADisplay va_dpy;
     VADRMPRIMESurfaceDescriptor prime;
 
-    unsigned int target_textures[2];
-
-    gsr_color_conversion color_conversion;
-
     Atom net_active_window_atom;
 } gsr_capture_xcomposite_vaapi;
 
@@ -113,9 +109,11 @@ static bool drm_create_codec_context(gsr_capture_xcomposite_vaapi *cap_xcomp, AV
 
 #define DRM_FORMAT_MOD_INVALID 0xffffffffffffffULL
 
-static int gsr_capture_xcomposite_vaapi_start(gsr_capture *cap, AVCodecContext *video_codec_context) {
+static int gsr_capture_xcomposite_vaapi_start(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame *frame) {
     gsr_capture_xcomposite_vaapi *cap_xcomp = cap->priv;
 
+    cap_xcomp->base.video_codec_context = video_codec_context;
+
     if(cap_xcomp->params.follow_focused) {
         cap_xcomp->net_active_window_atom = XInternAtom(cap_xcomp->params.egl->x11.dpy, "_NET_ACTIVE_WINDOW", False);
         if(!cap_xcomp->net_active_window_atom) {
@@ -185,17 +183,16 @@ static int gsr_capture_xcomposite_vaapi_start(gsr_capture *cap, AVCodecContext *
         return -1;
     }
 
+    if(!gsr_capture_base_setup_vaapi_textures(&cap_xcomp->base, frame, cap_xcomp->params.egl, cap_xcomp->va_dpy, &cap_xcomp->prime, cap_xcomp->params.color_range)) {
+        gsr_capture_xcomposite_vaapi_stop(cap, video_codec_context);
+        return -1;
+    }
+
     cap_xcomp->window_resize_timer = clock_get_monotonic_seconds();
     return 0;
 }
 
-static uint32_t fourcc(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
-    return (d << 24) | (c << 16) | (b << 8) | a;
-}
-
-#define FOURCC_NV12 842094158
-
-static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame) {
+static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *video_codec_context) {
     gsr_capture_xcomposite_vaapi *cap_xcomp = cap->priv;
 
     cap_xcomp->params.egl->glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
@@ -281,7 +278,7 @@ static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *
     }
 
     const double window_resize_timeout = 1.0; // 1 second
-    if(!cap_xcomp->created_hw_frame || (cap_xcomp->window_resized && clock_get_monotonic_seconds() - cap_xcomp->window_resize_timer >= window_resize_timeout)) {
+    if(cap_xcomp->window_resized && clock_get_monotonic_seconds() - cap_xcomp->window_resize_timer >= window_resize_timeout) {
         cap_xcomp->window_resized = false;
 
         if(window_texture_on_resize(&cap_xcomp->window_texture) != 0) {
@@ -302,124 +299,7 @@ static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *
         cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(cap_xcomp->texture_size.x)));
         cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(cap_xcomp->texture_size.y)));
 
-        if(!cap_xcomp->created_hw_frame) {
-            cap_xcomp->created_hw_frame = true;
-            av_frame_free(frame);
-            *frame = av_frame_alloc();
-            if(!frame) {
-                fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: failed to allocate frame\n");
-                cap_xcomp->should_stop = true;
-                cap_xcomp->stop_is_error = true;
-                return;
-            }
-            (*frame)->format = video_codec_context->pix_fmt;
-            (*frame)->width = video_codec_context->width;
-            (*frame)->height = video_codec_context->height;
-            (*frame)->color_range = video_codec_context->color_range;
-            (*frame)->color_primaries = video_codec_context->color_primaries;
-            (*frame)->color_trc = video_codec_context->color_trc;
-            (*frame)->colorspace = video_codec_context->colorspace;
-            (*frame)->chroma_location = video_codec_context->chroma_sample_location;
-
-            int res = av_hwframe_get_buffer(video_codec_context->hw_frames_ctx, *frame, 0);
-            if(res < 0) {
-                fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: av_hwframe_get_buffer failed: %d\n", res);
-                cap_xcomp->should_stop = true;
-                cap_xcomp->stop_is_error = true;
-                return;
-            }
-
-            VASurfaceID target_surface_id = (uintptr_t)(*frame)->data[3];
-
-            VAStatus va_status = vaExportSurfaceHandle(cap_xcomp->va_dpy, target_surface_id, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, VA_EXPORT_SURFACE_WRITE_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS, &cap_xcomp->prime);
-            if(va_status != VA_STATUS_SUCCESS) {
-                fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: vaExportSurfaceHandle failed, error: %d\n", va_status);
-                cap_xcomp->should_stop = true;
-                cap_xcomp->stop_is_error = true;
-                return;
-            }
-            vaSyncSurface(cap_xcomp->va_dpy, target_surface_id);
-
-            if(cap_xcomp->prime.fourcc == FOURCC_NV12) {
-                cap_xcomp->params.egl->glGenTextures(2, cap_xcomp->target_textures);
-                for(int i = 0; i < 2; ++i) {
-                    const uint32_t formats[2] = { fourcc('R', '8', ' ', ' '), fourcc('G', 'R', '8', '8') };
-                    const int layer = i;
-                    const int plane = 0;
-
-                    const int div[2] = {1, 2}; // divide UV texture size by 2 because chroma is half size
-                    //const uint64_t modifier = cap_kms->prime.objects[cap_kms->prime.layers[layer].object_index[plane]].drm_format_modifier;
-
-                    const intptr_t img_attr[] = {
-                        EGL_LINUX_DRM_FOURCC_EXT,       formats[i],
-                        EGL_WIDTH,                      cap_xcomp->prime.width / div[i],
-                        EGL_HEIGHT,                     cap_xcomp->prime.height / div[i],
-                        EGL_DMA_BUF_PLANE0_FD_EXT,      cap_xcomp->prime.objects[cap_xcomp->prime.layers[layer].object_index[plane]].fd,
-                        EGL_DMA_BUF_PLANE0_OFFSET_EXT,  cap_xcomp->prime.layers[layer].offset[plane],
-                        EGL_DMA_BUF_PLANE0_PITCH_EXT,   cap_xcomp->prime.layers[layer].pitch[plane],
-                        // TODO:
-                        //EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, modifier & 0xFFFFFFFFULL,
-                        //EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, modifier >> 32ULL,
-                        EGL_NONE
-                    };
-
-                    while(cap_xcomp->params.egl->eglGetError() != EGL_SUCCESS){}
-                    EGLImage image = cap_xcomp->params.egl->eglCreateImage(cap_xcomp->params.egl->egl_display, 0, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);
-                    if(!image) {
-                        fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: failed to create egl image from drm fd for output drm fd, error: %d\n", cap_xcomp->params.egl->eglGetError());
-                        cap_xcomp->should_stop = true;
-                        cap_xcomp->stop_is_error = true;
-                        return;
-                    }
-
-                    cap_xcomp->params.egl->glBindTexture(GL_TEXTURE_2D, cap_xcomp->target_textures[i]);
-                    cap_xcomp->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-                    cap_xcomp->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-                    cap_xcomp->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-                    cap_xcomp->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-
-                    while(cap_xcomp->params.egl->glGetError()) {}
-                    while(cap_xcomp->params.egl->eglGetError() != EGL_SUCCESS){}
-                    cap_xcomp->params.egl->glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);
-                    if(cap_xcomp->params.egl->glGetError() != 0 || cap_xcomp->params.egl->eglGetError() != EGL_SUCCESS) {
-                        // TODO: Get the error properly
-                        fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: failed to bind egl image to gl texture, error: %d\n", cap_xcomp->params.egl->eglGetError());
-                        cap_xcomp->should_stop = true;
-                        cap_xcomp->stop_is_error = true;
-                        cap_xcomp->params.egl->eglDestroyImage(cap_xcomp->params.egl->egl_display, image);
-                        cap_xcomp->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
-                        return;
-                    }
-
-                    cap_xcomp->params.egl->eglDestroyImage(cap_xcomp->params.egl->egl_display, image);
-                    cap_xcomp->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
-                }
-
-                gsr_color_conversion_params color_conversion_params = {0};
-                color_conversion_params.color_range = cap_xcomp->params.color_range;
-                color_conversion_params.egl = cap_xcomp->params.egl;
-                color_conversion_params.source_color = GSR_SOURCE_COLOR_RGB;
-                color_conversion_params.destination_color = GSR_DESTINATION_COLOR_NV12;
-
-                color_conversion_params.destination_textures[0] = cap_xcomp->target_textures[0];
-                color_conversion_params.destination_textures[1] = cap_xcomp->target_textures[1];
-                color_conversion_params.num_destination_textures = 2;
-
-                if(gsr_color_conversion_init(&cap_xcomp->color_conversion, &color_conversion_params) != 0) {
-                    fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: failed to create color conversion\n");
-                    cap_xcomp->should_stop = true;
-                    cap_xcomp->stop_is_error = true;
-                    return;
-                }
-            } else {
-                fprintf(stderr, "gsr error: gsr_capture_xcomposite_vaapi_tick: unexpected fourcc %u for output drm fd, expected nv12\n", cap_xcomp->prime.fourcc);
-                cap_xcomp->should_stop = true;
-                cap_xcomp->stop_is_error = true;
-                return;
-            }
-        }
-
-        gsr_color_conversion_clear(&cap_xcomp->color_conversion);
+        gsr_color_conversion_clear(&cap_xcomp->base.color_conversion);
     }
 }
 
@@ -443,7 +323,7 @@ static int gsr_capture_xcomposite_vaapi_capture(gsr_capture *cap, AVFrame *frame
     const int target_x = max_int(0, frame->width / 2 - cap_xcomp->texture_size.x / 2);
     const int target_y = max_int(0, frame->height / 2 - cap_xcomp->texture_size.y / 2);
 
-    gsr_color_conversion_draw(&cap_xcomp->color_conversion, window_texture_get_opengl_texture_id(&cap_xcomp->window_texture),
+    gsr_color_conversion_draw(&cap_xcomp->base.color_conversion, window_texture_get_opengl_texture_id(&cap_xcomp->window_texture),
         (vec2i){target_x, target_y}, cap_xcomp->texture_size,
         (vec2i){0, 0}, cap_xcomp->texture_size,
         0.0f, false);
@@ -458,8 +338,6 @@ static int gsr_capture_xcomposite_vaapi_capture(gsr_capture *cap, AVFrame *frame
 static void gsr_capture_xcomposite_vaapi_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
     gsr_capture_xcomposite_vaapi *cap_xcomp = cap->priv;
 
-    gsr_color_conversion_deinit(&cap_xcomp->color_conversion);
-
     for(uint32_t i = 0; i < cap_xcomp->prime.num_objects; ++i) {
         if(cap_xcomp->prime.objects[i].fd > 0) {
             close(cap_xcomp->prime.objects[i].fd);
@@ -467,18 +345,14 @@ static void gsr_capture_xcomposite_vaapi_stop(gsr_capture *cap, AVCodecContext *
         }
     }
 
-    if(cap_xcomp->params.egl->egl_context) {
-        cap_xcomp->params.egl->glDeleteTextures(2, cap_xcomp->target_textures);
-        cap_xcomp->target_textures[0] = 0;
-        cap_xcomp->target_textures[1] = 0;
-    }
-
     window_texture_deinit(&cap_xcomp->window_texture);
 
     if(video_codec_context->hw_device_ctx)
         av_buffer_unref(&video_codec_context->hw_device_ctx);
     if(video_codec_context->hw_frames_ctx)
         av_buffer_unref(&video_codec_context->hw_frames_ctx);
+
+    gsr_capture_base_stop(&cap_xcomp->base, cap_xcomp->params.egl);
 }
 
 static void gsr_capture_xcomposite_vaapi_destroy(gsr_capture *cap, AVCodecContext *video_codec_context) {