Fix vram leak for reals

9 files changed, 72 insertions, 15 deletions

diff --git a/include/capture/capture.h b/include/capture/capture.h
index edcc14d..82a9555 100644
--- a/include/capture/capture.h
+++ b/include/capture/capture.h
@@ -14,6 +14,7 @@ struct gsr_capture {
     void (*tick)(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame); /* can be NULL */
     bool (*should_stop)(gsr_capture *cap, bool *err); /* can be NULL */
     int (*capture)(gsr_capture *cap, AVFrame *frame);
+    void (*capture_end)(gsr_capture *cap, AVFrame *frame); /* can be NULL */
     void (*destroy)(gsr_capture *cap, AVCodecContext *video_codec_context);
 
     void *priv; /* can be NULL */
@@ -24,6 +25,7 @@ int gsr_capture_start(gsr_capture *cap, AVCodecContext *video_codec_context);
 void gsr_capture_tick(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame);
 bool gsr_capture_should_stop(gsr_capture *cap, bool *err);
 int gsr_capture_capture(gsr_capture *cap, AVFrame *frame);
+void gsr_capture_end(gsr_capture *cap, AVFrame *frame);
 /* Calls |gsr_capture_stop| as well */
 void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context);
 
diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c
index 7bc555d..f42567a 100644
--- a/kms/server/kms_server.c
+++ b/kms/server/kms_server.c
@@ -185,6 +185,26 @@ static void map_crtc_to_connector_ids(gsr_drm *drm, connector_to_crtc_map *c2crt
     drmModeFreeResources(resources);
 }
 
+static void drm_mode_cleanup_handles(int drmfd, drmModeFB2Ptr drmfb) {
+    for(int i = 0; i < 4; ++i) {
+        if(!drmfb->handles[i])
+            continue;
+
+        bool already_closed = false;
+        for(int j = 0; j < i; ++j) {
+            if(drmfb->handles[i] == drmfb->handles[j]) {
+                already_closed = true;
+                break;
+            }
+        }
+
+        if(already_closed)
+            continue;
+
+        drmCloseBufferHandle(drmfd, drmfb->handles[i]);
+    }
+}
+
 static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crtc_map *c2crtc_map) {
     int result = -1;
 
@@ -194,7 +214,7 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
 
     for(uint32_t i = 0; i < drm->planes->count_planes && response->num_fds < GSR_KMS_MAX_PLANES; ++i) {
         drmModePlanePtr plane = NULL;
-        drmModeFBPtr drmfb = NULL;
+        drmModeFB2Ptr drmfb = NULL;
 
         plane = drmModeGetPlane(drm->drmfd, drm->planes->planes[i]);
         if(!plane) {
@@ -207,8 +227,7 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
         if(!plane->fb_id)
             goto next;
 
-        // TODO: drmModeGetFB2 can't be used because it causes a vram leak when the fb_fd is sent amd/intel.. why?
-        drmfb = drmModeGetFB(drm->drmfd, plane->fb_id);
+        drmfb = drmModeGetFB2(drm->drmfd, plane->fb_id);
         if(!drmfb) {
             // Commented out for now because we get here if the cursor is moved to another monitor and we dont care about the cursor
             //response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
@@ -217,23 +236,23 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
             goto next;
         }
 
-        if(!drmfb->handle) {
+        if(!drmfb->handles[0]) {
             response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
             snprintf(response->err_msg, sizeof(response->err_msg), "drmfb handle is NULL");
             fprintf(stderr, "kms server error: %s\n", response->err_msg);
-            goto next;
+            goto cleanup_handles;
         }
 
         // TODO: Check if dimensions have changed by comparing width and height to previous time this was called.
         // TODO: Support other plane formats than rgb (with multiple planes, such as direct YUV420 on wayland).
 
         int fb_fd = -1;
-        const int ret = drmPrimeHandleToFD(drm->drmfd, drmfb->handle, O_RDONLY, &fb_fd);
+        const int ret = drmPrimeHandleToFD(drm->drmfd, drmfb->handles[0], O_RDONLY, &fb_fd);
         if(ret != 0 || fb_fd == -1) {
             response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
             snprintf(response->err_msg, sizeof(response->err_msg), "failed to get fd from drm handle, error: %s", strerror(errno));
             fprintf(stderr, "kms server error: %s\n", response->err_msg);
-            goto next;
+            goto cleanup_handles;
         }
 
         bool is_cursor = false;
@@ -243,10 +262,11 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
         response->fds[response->num_fds].fd = fb_fd;
         response->fds[response->num_fds].width = drmfb->width;
         response->fds[response->num_fds].height = drmfb->height;
-        response->fds[response->num_fds].pitch = drmfb->pitch;
-        response->fds[response->num_fds].offset = 0;//drmfb->offsets[0];
-        // TODO?
+        response->fds[response->num_fds].pitch = drmfb->pitches[0];
+        response->fds[response->num_fds].offset = drmfb->offsets[0];
+        // TODO:
         response->fds[response->num_fds].pixel_format = DRM_FORMAT_ARGB8888;//drmfb->pixel_format;
+        // TODO:
         response->fds[response->num_fds].modifier = 0;//drmfb->modifier;
         response->fds[response->num_fds].connector_id = get_connector_by_crtc_id(c2crtc_map, plane->crtc_id);
         response->fds[response->num_fds].is_cursor = is_cursor;
@@ -264,13 +284,19 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
         }
         ++response->num_fds;
 
+        cleanup_handles:
+        drm_mode_cleanup_handles(drm->drmfd, drmfb);
+
         next:
         if(drmfb)
-            drmModeFreeFB(drmfb);
+            drmModeFreeFB2(drmfb);
         if(plane)
             drmModeFreePlane(plane);
     }
 
+    if(response->num_fds > 0)
+        response->result = KMS_RESULT_OK;
+
     if(response->result == KMS_RESULT_OK) {
         result = 0;
     } else {
diff --git a/src/capture/capture.c b/src/capture/capture.c
index 699745a..eea0d1d 100644
--- a/src/capture/capture.c
+++ b/src/capture/capture.c
@@ -42,6 +42,18 @@ int gsr_capture_capture(gsr_capture *cap, AVFrame *frame) {
     return cap->capture(cap, frame);
 }
 
+void gsr_capture_end(gsr_capture *cap, AVFrame *frame) {
+    if(!cap->started) {
+        fprintf(stderr, "gsr error: gsr_capture_end failed: the gsr capture has not been started\n");
+        return;
+    }
+
+    if(!cap->capture_end)
+        return;
+
+    cap->capture_end(cap, frame);
+}
+
 void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context) {
     cap->destroy(cap, video_codec_context);
 }
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index 13636aa..e7f7ea5 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -424,6 +424,13 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
 
     gsr_capture_kms_unload_cuda_graphics(cap_kms);
 
+    return 0;
+}
+
+static void gsr_capture_kms_cuda_capture_end(gsr_capture *cap, AVFrame *frame) {
+    (void)frame;
+    gsr_capture_kms_cuda *cap_kms = cap->priv;
+
     gsr_egl_cleanup_frame(cap_kms->params.egl);
 
     for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
@@ -432,8 +439,6 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
         cap_kms->kms_response.fds[i].fd = 0;
     }
     cap_kms->kms_response.num_fds = 0;
-
-    return 0;
 }
 
 static void gsr_capture_kms_cuda_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
@@ -501,6 +506,7 @@ gsr_capture* gsr_capture_kms_cuda_create(const gsr_capture_kms_cuda_params *para
         .tick = gsr_capture_kms_cuda_tick,
         .should_stop = gsr_capture_kms_cuda_should_stop,
         .capture = gsr_capture_kms_cuda_capture,
+        .capture_end = gsr_capture_kms_cuda_capture_end,
         .destroy = gsr_capture_kms_cuda_destroy,
         .priv = cap_kms
     };
diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c
index 5899f9f..681f345 100644
--- a/src/capture/kms_vaapi.c
+++ b/src/capture/kms_vaapi.c
@@ -547,6 +547,13 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
 
     cap_kms->params.egl->eglSwapBuffers(cap_kms->params.egl->egl_display, cap_kms->params.egl->egl_surface);
 
+    return 0;
+}
+
+static void gsr_capture_kms_vaapi_capture_end(gsr_capture *cap, AVFrame *frame) {
+    (void)frame;
+    gsr_capture_kms_vaapi *cap_kms = cap->priv;
+
     gsr_egl_cleanup_frame(cap_kms->params.egl);
 
     for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
@@ -555,8 +562,6 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
         cap_kms->kms_response.fds[i].fd = 0;
     }
     cap_kms->kms_response.num_fds = 0;
-
-    return 0;
 }
 
 static void gsr_capture_kms_vaapi_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
@@ -647,6 +652,7 @@ gsr_capture* gsr_capture_kms_vaapi_create(const gsr_capture_kms_vaapi_params *pa
         .tick = gsr_capture_kms_vaapi_tick,
         .should_stop = gsr_capture_kms_vaapi_should_stop,
         .capture = gsr_capture_kms_vaapi_capture,
+        .capture_end = gsr_capture_kms_vaapi_capture_end,
         .destroy = gsr_capture_kms_vaapi_destroy,
         .priv = cap_kms
     };
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 32b83fc..5b62310 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -492,6 +492,7 @@ gsr_capture* gsr_capture_nvfbc_create(const gsr_capture_nvfbc_params *params) {
         .tick = gsr_capture_nvfbc_tick,
         .should_stop = NULL,
         .capture = gsr_capture_nvfbc_capture,
+        .capture_end = NULL,
         .destroy = gsr_capture_nvfbc_destroy,
         .priv = cap_nvfbc
     };
diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c
index e58decb..7e65efa 100644
--- a/src/capture/xcomposite_cuda.c
+++ b/src/capture/xcomposite_cuda.c
@@ -503,6 +503,7 @@ gsr_capture* gsr_capture_xcomposite_cuda_create(const gsr_capture_xcomposite_cud
         .tick = gsr_capture_xcomposite_cuda_tick,
         .should_stop = gsr_capture_xcomposite_cuda_should_stop,
         .capture = gsr_capture_xcomposite_cuda_capture,
+        .capture_end = NULL,
         .destroy = gsr_capture_xcomposite_cuda_destroy,
         .priv = cap_xcomp
     };
diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c
index 18bcf32..2bc21a7 100644
--- a/src/capture/xcomposite_vaapi.c
+++ b/src/capture/xcomposite_vaapi.c
@@ -508,6 +508,7 @@ gsr_capture* gsr_capture_xcomposite_vaapi_create(const gsr_capture_xcomposite_va
         .tick = gsr_capture_xcomposite_vaapi_tick,
         .should_stop = gsr_capture_xcomposite_vaapi_should_stop,
         .capture = gsr_capture_xcomposite_vaapi_capture,
+        .capture_end = NULL,
         .destroy = gsr_capture_xcomposite_vaapi_destroy,
         .priv = cap_xcomp
     };
diff --git a/src/main.cpp b/src/main.cpp
index 1f436ff..eb5f8c2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -2186,6 +2186,8 @@ int main(int argc, char **argv) {
                         fprintf(stderr, "Error: avcodec_send_frame failed, error: %s\n", av_error_to_string(ret));
                     }
                 }
+
+                gsr_capture_end(capture, frame);
                 video_pts_counter += num_frames;
             }
         }