aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2023-10-22 04:56:34 +0200
committerdec05eba <dec05eba@protonmail.com>2023-10-22 04:56:34 +0200
commit51154b71be6be742541f53a21b5c9d39d0033270 (patch)
tree348cc95ce90744c14884b2821136586b5ea8ed09
parent44d50065b92284638398b4989490ac66eb34c53e (diff)
Fix vram leak for reals
-rw-r--r--include/capture/capture.h2
-rw-r--r--kms/server/kms_server.c48
-rw-r--r--src/capture/capture.c12
-rw-r--r--src/capture/kms_cuda.c10
-rw-r--r--src/capture/kms_vaapi.c10
-rw-r--r--src/capture/nvfbc.c1
-rw-r--r--src/capture/xcomposite_cuda.c1
-rw-r--r--src/capture/xcomposite_vaapi.c1
-rw-r--r--src/main.cpp2
9 files changed, 72 insertions, 15 deletions
diff --git a/include/capture/capture.h b/include/capture/capture.h
index edcc14d..82a9555 100644
--- a/include/capture/capture.h
+++ b/include/capture/capture.h
@@ -14,6 +14,7 @@ struct gsr_capture {
void (*tick)(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame); /* can be NULL */
bool (*should_stop)(gsr_capture *cap, bool *err); /* can be NULL */
int (*capture)(gsr_capture *cap, AVFrame *frame);
+ void (*capture_end)(gsr_capture *cap, AVFrame *frame); /* can be NULL */
void (*destroy)(gsr_capture *cap, AVCodecContext *video_codec_context);
void *priv; /* can be NULL */
@@ -24,6 +25,7 @@ int gsr_capture_start(gsr_capture *cap, AVCodecContext *video_codec_context);
void gsr_capture_tick(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame **frame);
bool gsr_capture_should_stop(gsr_capture *cap, bool *err);
int gsr_capture_capture(gsr_capture *cap, AVFrame *frame);
+void gsr_capture_end(gsr_capture *cap, AVFrame *frame);
/* Calls |gsr_capture_stop| as well */
void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context);
diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c
index 7bc555d..f42567a 100644
--- a/kms/server/kms_server.c
+++ b/kms/server/kms_server.c
@@ -185,6 +185,26 @@ static void map_crtc_to_connector_ids(gsr_drm *drm, connector_to_crtc_map *c2crt
drmModeFreeResources(resources);
}
+static void drm_mode_cleanup_handles(int drmfd, drmModeFB2Ptr drmfb) {
+ for(int i = 0; i < 4; ++i) {
+ if(!drmfb->handles[i])
+ continue;
+
+ bool already_closed = false;
+ for(int j = 0; j < i; ++j) {
+ if(drmfb->handles[i] == drmfb->handles[j]) {
+ already_closed = true;
+ break;
+ }
+ }
+
+ if(already_closed)
+ continue;
+
+ drmCloseBufferHandle(drmfd, drmfb->handles[i]);
+ }
+}
+
static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crtc_map *c2crtc_map) {
int result = -1;
@@ -194,7 +214,7 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
for(uint32_t i = 0; i < drm->planes->count_planes && response->num_fds < GSR_KMS_MAX_PLANES; ++i) {
drmModePlanePtr plane = NULL;
- drmModeFBPtr drmfb = NULL;
+ drmModeFB2Ptr drmfb = NULL;
plane = drmModeGetPlane(drm->drmfd, drm->planes->planes[i]);
if(!plane) {
@@ -207,8 +227,7 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
if(!plane->fb_id)
goto next;
- // TODO: drmModeGetFB2 can't be used because it causes a vram leak when the fb_fd is sent amd/intel.. why?
- drmfb = drmModeGetFB(drm->drmfd, plane->fb_id);
+ drmfb = drmModeGetFB2(drm->drmfd, plane->fb_id);
if(!drmfb) {
// Commented out for now because we get here if the cursor is moved to another monitor and we dont care about the cursor
//response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
@@ -217,23 +236,23 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
goto next;
}
- if(!drmfb->handle) {
+ if(!drmfb->handles[0]) {
response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
snprintf(response->err_msg, sizeof(response->err_msg), "drmfb handle is NULL");
fprintf(stderr, "kms server error: %s\n", response->err_msg);
- goto next;
+ goto cleanup_handles;
}
// TODO: Check if dimensions have changed by comparing width and height to previous time this was called.
// TODO: Support other plane formats than rgb (with multiple planes, such as direct YUV420 on wayland).
int fb_fd = -1;
- const int ret = drmPrimeHandleToFD(drm->drmfd, drmfb->handle, O_RDONLY, &fb_fd);
+ const int ret = drmPrimeHandleToFD(drm->drmfd, drmfb->handles[0], O_RDONLY, &fb_fd);
if(ret != 0 || fb_fd == -1) {
response->result = KMS_RESULT_FAILED_TO_GET_PLANE;
snprintf(response->err_msg, sizeof(response->err_msg), "failed to get fd from drm handle, error: %s", strerror(errno));
fprintf(stderr, "kms server error: %s\n", response->err_msg);
- goto next;
+ goto cleanup_handles;
}
bool is_cursor = false;
@@ -243,10 +262,11 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
response->fds[response->num_fds].fd = fb_fd;
response->fds[response->num_fds].width = drmfb->width;
response->fds[response->num_fds].height = drmfb->height;
- response->fds[response->num_fds].pitch = drmfb->pitch;
- response->fds[response->num_fds].offset = 0;//drmfb->offsets[0];
- // TODO?
+ response->fds[response->num_fds].pitch = drmfb->pitches[0];
+ response->fds[response->num_fds].offset = drmfb->offsets[0];
+ // TODO:
response->fds[response->num_fds].pixel_format = DRM_FORMAT_ARGB8888;//drmfb->pixel_format;
+ // TODO:
response->fds[response->num_fds].modifier = 0;//drmfb->modifier;
response->fds[response->num_fds].connector_id = get_connector_by_crtc_id(c2crtc_map, plane->crtc_id);
response->fds[response->num_fds].is_cursor = is_cursor;
@@ -264,13 +284,19 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response, connector_to_crt
}
++response->num_fds;
+ cleanup_handles:
+ drm_mode_cleanup_handles(drm->drmfd, drmfb);
+
next:
if(drmfb)
- drmModeFreeFB(drmfb);
+ drmModeFreeFB2(drmfb);
if(plane)
drmModeFreePlane(plane);
}
+ if(response->num_fds > 0)
+ response->result = KMS_RESULT_OK;
+
if(response->result == KMS_RESULT_OK) {
result = 0;
} else {
diff --git a/src/capture/capture.c b/src/capture/capture.c
index 699745a..eea0d1d 100644
--- a/src/capture/capture.c
+++ b/src/capture/capture.c
@@ -42,6 +42,18 @@ int gsr_capture_capture(gsr_capture *cap, AVFrame *frame) {
return cap->capture(cap, frame);
}
+void gsr_capture_end(gsr_capture *cap, AVFrame *frame) {
+ if(!cap->started) {
+ fprintf(stderr, "gsr error: gsr_capture_end failed: the gsr capture has not been started\n");
+ return;
+ }
+
+ if(!cap->capture_end)
+ return;
+
+ cap->capture_end(cap, frame);
+}
+
void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context) {
cap->destroy(cap, video_codec_context);
}
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index 13636aa..e7f7ea5 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -424,6 +424,13 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
gsr_capture_kms_unload_cuda_graphics(cap_kms);
+ return 0;
+}
+
+static void gsr_capture_kms_cuda_capture_end(gsr_capture *cap, AVFrame *frame) {
+ (void)frame;
+ gsr_capture_kms_cuda *cap_kms = cap->priv;
+
gsr_egl_cleanup_frame(cap_kms->params.egl);
for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
@@ -432,8 +439,6 @@ static int gsr_capture_kms_cuda_capture(gsr_capture *cap, AVFrame *frame) {
cap_kms->kms_response.fds[i].fd = 0;
}
cap_kms->kms_response.num_fds = 0;
-
- return 0;
}
static void gsr_capture_kms_cuda_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
@@ -501,6 +506,7 @@ gsr_capture* gsr_capture_kms_cuda_create(const gsr_capture_kms_cuda_params *para
.tick = gsr_capture_kms_cuda_tick,
.should_stop = gsr_capture_kms_cuda_should_stop,
.capture = gsr_capture_kms_cuda_capture,
+ .capture_end = gsr_capture_kms_cuda_capture_end,
.destroy = gsr_capture_kms_cuda_destroy,
.priv = cap_kms
};
diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c
index 5899f9f..681f345 100644
--- a/src/capture/kms_vaapi.c
+++ b/src/capture/kms_vaapi.c
@@ -547,6 +547,13 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
cap_kms->params.egl->eglSwapBuffers(cap_kms->params.egl->egl_display, cap_kms->params.egl->egl_surface);
+ return 0;
+}
+
+static void gsr_capture_kms_vaapi_capture_end(gsr_capture *cap, AVFrame *frame) {
+ (void)frame;
+ gsr_capture_kms_vaapi *cap_kms = cap->priv;
+
gsr_egl_cleanup_frame(cap_kms->params.egl);
for(int i = 0; i < cap_kms->kms_response.num_fds; ++i) {
@@ -555,8 +562,6 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
cap_kms->kms_response.fds[i].fd = 0;
}
cap_kms->kms_response.num_fds = 0;
-
- return 0;
}
static void gsr_capture_kms_vaapi_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
@@ -647,6 +652,7 @@ gsr_capture* gsr_capture_kms_vaapi_create(const gsr_capture_kms_vaapi_params *pa
.tick = gsr_capture_kms_vaapi_tick,
.should_stop = gsr_capture_kms_vaapi_should_stop,
.capture = gsr_capture_kms_vaapi_capture,
+ .capture_end = gsr_capture_kms_vaapi_capture_end,
.destroy = gsr_capture_kms_vaapi_destroy,
.priv = cap_kms
};
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 32b83fc..5b62310 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -492,6 +492,7 @@ gsr_capture* gsr_capture_nvfbc_create(const gsr_capture_nvfbc_params *params) {
.tick = gsr_capture_nvfbc_tick,
.should_stop = NULL,
.capture = gsr_capture_nvfbc_capture,
+ .capture_end = NULL,
.destroy = gsr_capture_nvfbc_destroy,
.priv = cap_nvfbc
};
diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c
index e58decb..7e65efa 100644
--- a/src/capture/xcomposite_cuda.c
+++ b/src/capture/xcomposite_cuda.c
@@ -503,6 +503,7 @@ gsr_capture* gsr_capture_xcomposite_cuda_create(const gsr_capture_xcomposite_cud
.tick = gsr_capture_xcomposite_cuda_tick,
.should_stop = gsr_capture_xcomposite_cuda_should_stop,
.capture = gsr_capture_xcomposite_cuda_capture,
+ .capture_end = NULL,
.destroy = gsr_capture_xcomposite_cuda_destroy,
.priv = cap_xcomp
};
diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c
index 18bcf32..2bc21a7 100644
--- a/src/capture/xcomposite_vaapi.c
+++ b/src/capture/xcomposite_vaapi.c
@@ -508,6 +508,7 @@ gsr_capture* gsr_capture_xcomposite_vaapi_create(const gsr_capture_xcomposite_va
.tick = gsr_capture_xcomposite_vaapi_tick,
.should_stop = gsr_capture_xcomposite_vaapi_should_stop,
.capture = gsr_capture_xcomposite_vaapi_capture,
+ .capture_end = NULL,
.destroy = gsr_capture_xcomposite_vaapi_destroy,
.priv = cap_xcomp
};
diff --git a/src/main.cpp b/src/main.cpp
index 1f436ff..eb5f8c2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -2186,6 +2186,8 @@ int main(int argc, char **argv) {
fprintf(stderr, "Error: avcodec_send_frame failed, error: %s\n", av_error_to_string(ret));
}
}
+
+ gsr_capture_end(capture, frame);
video_pts_counter += num_frames;
}
}