1 files changed, 34 insertions, 33 deletions

diff --git a/src/encoder/video/cuda.c b/src/encoder/video/cuda.c
index 2568bc7..6d26cdd 100644
--- a/src/encoder/video/cuda.c
+++ b/src/encoder/video/cuda.c
@@ -12,6 +12,8 @@ typedef struct {
 
     unsigned int target_textures[2];
 
+    AVBufferRef *device_ctx;
+
     gsr_cuda cuda;
     CUgraphicsResource cuda_graphics_resources[2];
     CUarray mapped_arrays[2];
@@ -19,47 +21,46 @@ typedef struct {
 } gsr_video_encoder_cuda;
 
 static bool gsr_video_encoder_cuda_setup_context(gsr_video_encoder_cuda *self, AVCodecContext *video_codec_context) {
-    AVBufferRef *device_ctx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA);
-    if(!device_ctx) {
+    self->device_ctx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA);
+    if(!self->device_ctx) {
         fprintf(stderr, "gsr error: gsr_video_encoder_cuda_setup_context failed: failed to create hardware device context\n");
         return false;
     }
 
-    AVHWDeviceContext *hw_device_context = (AVHWDeviceContext*)device_ctx->data;
+    AVHWDeviceContext *hw_device_context = (AVHWDeviceContext*)self->device_ctx->data;
     AVCUDADeviceContext *cuda_device_context = (AVCUDADeviceContext*)hw_device_context->hwctx;
     cuda_device_context->cuda_ctx = self->cuda.cu_ctx;
-    if(av_hwdevice_ctx_init(device_ctx) < 0) {
+    if(av_hwdevice_ctx_init(self->device_ctx) < 0) {
         fprintf(stderr, "gsr error: gsr_video_encoder_cuda_setup_context failed: failed to create hardware device context\n");
-        av_buffer_unref(&device_ctx);
+        av_buffer_unref(&self->device_ctx);
         return false;
     }
 
-    AVBufferRef *frame_context = av_hwframe_ctx_alloc(device_ctx);
+    AVBufferRef *frame_context = av_hwframe_ctx_alloc(self->device_ctx);
     if(!frame_context) {
         fprintf(stderr, "gsr error: gsr_video_encoder_cuda_setup_context failed: failed to create hwframe context\n");
-        av_buffer_unref(&device_ctx);
+        av_buffer_unref(&self->device_ctx);
         return false;
     }
 
     AVHWFramesContext *hw_frame_context = (AVHWFramesContext*)frame_context->data;
     hw_frame_context->width = video_codec_context->width;
     hw_frame_context->height = video_codec_context->height;
-    hw_frame_context->sw_format = self->params.hdr ? AV_PIX_FMT_P010LE : AV_PIX_FMT_NV12;
+    hw_frame_context->sw_format = self->params.color_depth == GSR_COLOR_DEPTH_10_BITS ? AV_PIX_FMT_P010LE : AV_PIX_FMT_NV12;
     hw_frame_context->format = video_codec_context->pix_fmt;
-    hw_frame_context->device_ref = device_ctx;
-    hw_frame_context->device_ctx = (AVHWDeviceContext*)device_ctx->data;
+    hw_frame_context->device_ctx = (AVHWDeviceContext*)self->device_ctx->data;
 
     if (av_hwframe_ctx_init(frame_context) < 0) {
         fprintf(stderr, "gsr error: gsr_video_encoder_cuda_setup_context failed: failed to initialize hardware frame context "
                         "(note: ffmpeg version needs to be > 4.0)\n");
-        av_buffer_unref(&device_ctx);
+        av_buffer_unref(&self->device_ctx);
         //av_buffer_unref(&frame_context);
         return false;
     }
 
     self->cuda_stream = cuda_device_context->stream;
-    video_codec_context->hw_device_ctx = av_buffer_ref(device_ctx);
     video_codec_context->hw_frames_ctx = av_buffer_ref(frame_context);
+    av_buffer_unref(&frame_context);
     return true;
 }
 
@@ -108,7 +109,7 @@ static bool gsr_video_encoder_cuda_setup_textures(gsr_video_encoder_cuda *self,
     const int div[2] = {1, 2}; // divide UV texture size by 2 because chroma is half size
 
     for(int i = 0; i < 2; ++i) {
-        self->target_textures[i] = gl_create_texture(self->params.egl, video_codec_context->width / div[i], video_codec_context->height / div[i], !self->params.hdr ? internal_formats_nv12[i] : internal_formats_p010[i], formats[i]);
+        self->target_textures[i] = gl_create_texture(self->params.egl, video_codec_context->width / div[i], video_codec_context->height / div[i], self->params.color_depth == GSR_COLOR_DEPTH_8_BITS ? internal_formats_nv12[i] : internal_formats_p010[i], formats[i]);
         if(self->target_textures[i] == 0) {
             fprintf(stderr, "gsr error: gsr_video_encoder_cuda_setup_textures: failed to create opengl texture\n");
             return false;
@@ -125,22 +126,22 @@ static bool gsr_video_encoder_cuda_setup_textures(gsr_video_encoder_cuda *self,
 static void gsr_video_encoder_cuda_stop(gsr_video_encoder_cuda *self, AVCodecContext *video_codec_context);
 
 static bool gsr_video_encoder_cuda_start(gsr_video_encoder *encoder, AVCodecContext *video_codec_context, AVFrame *frame) {
-    gsr_video_encoder_cuda *encoder_cuda = encoder->priv;
+    gsr_video_encoder_cuda *self = encoder->priv;
 
-    // TODO: Force set overclock to false if wayland
-    if(!gsr_cuda_load(&encoder_cuda->cuda, encoder_cuda->params.egl->x11.dpy, encoder_cuda->params.overclock)) {
+    const bool overclock = gsr_egl_get_display_server(self->params.egl) == GSR_DISPLAY_SERVER_X11 ? self->params.overclock : false;
+    if(!gsr_cuda_load(&self->cuda, self->params.egl->x11.dpy, overclock)) {
         fprintf(stderr, "gsr error: gsr_video_encoder_cuda_start: failed to load cuda\n");
-        gsr_video_encoder_cuda_stop(encoder_cuda, video_codec_context);
+        gsr_video_encoder_cuda_stop(self, video_codec_context);
         return false;
     }
 
-    if(!gsr_video_encoder_cuda_setup_context(encoder_cuda, video_codec_context)) {
-        gsr_video_encoder_cuda_stop(encoder_cuda, video_codec_context);
+    if(!gsr_video_encoder_cuda_setup_context(self, video_codec_context)) {
+        gsr_video_encoder_cuda_stop(self, video_codec_context);
         return false;
     }
 
-    if(!gsr_video_encoder_cuda_setup_textures(encoder_cuda, video_codec_context, frame)) {
-        gsr_video_encoder_cuda_stop(encoder_cuda, video_codec_context);
+    if(!gsr_video_encoder_cuda_setup_textures(self, video_codec_context, frame)) {
+        gsr_video_encoder_cuda_stop(self, video_codec_context);
         return false;
     }
 
@@ -152,10 +153,10 @@ void gsr_video_encoder_cuda_stop(gsr_video_encoder_cuda *self, AVCodecContext *v
     self->target_textures[0] = 0;
     self->target_textures[1] = 0;
 
-    if(video_codec_context->hw_device_ctx)
-        av_buffer_unref(&video_codec_context->hw_device_ctx);
     if(video_codec_context->hw_frames_ctx)
         av_buffer_unref(&video_codec_context->hw_frames_ctx);
+    if(self->device_ctx)
+        av_buffer_unref(&self->device_ctx);
 
     if(self->cuda.cu_ctx) {
         for(int i = 0; i < 2; ++i) {
@@ -170,8 +171,8 @@ void gsr_video_encoder_cuda_stop(gsr_video_encoder_cuda *self, AVCodecContext *v
     gsr_cuda_unload(&self->cuda);
 }
 
-static void gsr_video_encoder_cuda_copy_textures_to_frame(gsr_video_encoder *encoder, AVFrame *frame) {
-    gsr_video_encoder_cuda *encoder_cuda = encoder->priv;
+static void gsr_video_encoder_cuda_copy_textures_to_frame(gsr_video_encoder *encoder, AVFrame *frame, gsr_color_conversion *color_conversion) {
+    gsr_video_encoder_cuda *self = encoder->priv;
     const int div[2] = {1, 2}; // divide UV texture size by 2 because chroma is half size
     for(int i = 0; i < 2; ++i) {
         CUDA_MEMCPY2D memcpy_struct;
@@ -183,26 +184,26 @@ static void gsr_video_encoder_cuda_copy_textures_to_frame(gsr_video_encoder *enc
         memcpy_struct.dstY = 0;
         memcpy_struct.dstMemoryType = CU_MEMORYTYPE_DEVICE;
 
-        memcpy_struct.srcArray = encoder_cuda->mapped_arrays[i];
+        memcpy_struct.srcArray = self->mapped_arrays[i];
         memcpy_struct.srcPitch = frame->width / div[i];
         memcpy_struct.dstDevice = (CUdeviceptr)frame->data[i];
         memcpy_struct.dstPitch = frame->linesize[i];
-        memcpy_struct.WidthInBytes = frame->width * (encoder_cuda->params.hdr ? 2 : 1);
+        memcpy_struct.WidthInBytes = frame->width * (self->params.color_depth == GSR_COLOR_DEPTH_10_BITS ? 2 : 1);
         memcpy_struct.Height = frame->height / div[i];
         // TODO: Remove this copy if possible
-        encoder_cuda->cuda.cuMemcpy2DAsync_v2(&memcpy_struct, encoder_cuda->cuda_stream);
+        self->cuda.cuMemcpy2DAsync_v2(&memcpy_struct, self->cuda_stream);
     }
 
     // TODO: needed?
-    encoder_cuda->cuda.cuStreamSynchronize(encoder_cuda->cuda_stream);
+    self->cuda.cuStreamSynchronize(self->cuda_stream);
 }
 
 static void gsr_video_encoder_cuda_get_textures(gsr_video_encoder *encoder, unsigned int *textures, int *num_textures, gsr_destination_color *destination_color) {
-    gsr_video_encoder_cuda *encoder_cuda = encoder->priv;
-    textures[0] = encoder_cuda->target_textures[0];
-    textures[1] = encoder_cuda->target_textures[1];
+    gsr_video_encoder_cuda *self = encoder->priv;
+    textures[0] = self->target_textures[0];
+    textures[1] = self->target_textures[1];
     *num_textures = 2;
-    *destination_color = encoder_cuda->params.hdr ? GSR_DESTINATION_COLOR_P010 : GSR_DESTINATION_COLOR_NV12;
+    *destination_color = self->params.color_depth == GSR_COLOR_DEPTH_10_BITS ? GSR_DESTINATION_COLOR_P010 : GSR_DESTINATION_COLOR_NV12;
 }
 
 static void gsr_video_encoder_cuda_destroy(gsr_video_encoder *encoder, AVCodecContext *video_codec_context) {