Show warning when capture monitor on intel and the plane is compressed. The warning tells the user to capture on x11 instead or use -w portal option

2 files changed, 31 insertions, 1 deletions

diff --git a/TODO b/TODO
index b4dae18..71e058e 100644
--- a/TODO
+++ b/TODO
@@ -152,4 +152,6 @@ To test vulkan encode on amd set the environment variable RADV_PERFTEST=video_en
 
 Support hevc/av1 for software encoder and hdr support at the same time. Need support for yuv420p shader for that. Use libx265 for hevc and libsvtav1 for av1 (libsvtav1 is the fastest software av1 video encoder). Also support vp8/vp9 since we are not limited by hardware.
 
-Cleanup pipewire code and add more error checks.
\ No newline at end of file
+Cleanup pipewire code and add more error checks.
+
+Detect if recording monitor on intel and plane is compressed. In that case give an error and tell the user to use -w portal instead.
diff --git a/src/capture/kms.c b/src/capture/kms.c
index e7b0b59..3e66bdb 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -8,6 +8,8 @@
 #include <stdio.h>
 #include <unistd.h>
 
+#include <libdrm/drm_fourcc.h>
+
 #include <libavcodec/avcodec.h>
 #include <libavutil/mastering_display_metadata.h>
 
@@ -266,6 +268,24 @@ static vec2i swap_vec2i(vec2i value) {
     return value;
 }
 
+static bool is_plane_compressed(uint64_t modifier) {
+    switch(modifier) {
+        case I915_FORMAT_MOD_Y_TILED_CCS:
+        case I915_FORMAT_MOD_Yf_TILED_CCS:
+        case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
+        case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
+        case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
+        case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS:
+        case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS:
+        case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC:
+        case I915_FORMAT_MOD_4_TILED_MTL_RC_CCS:
+        case I915_FORMAT_MOD_4_TILED_MTL_MC_CCS:
+        case I915_FORMAT_MOD_4_TILED_MTL_RC_CCS_CC:
+            return true;
+    }
+    return false;
+}
+
 static int gsr_capture_kms_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) {
     gsr_capture_kms *self = cap->priv;
     const bool screen_plane_use_modifiers = self->params.egl->gpu_info.vendor != GSR_GPU_VENDOR_AMD;
@@ -346,6 +366,14 @@ static int gsr_capture_kms_capture(gsr_capture *cap, AVFrame *frame, gsr_color_c
         EGL_DMA_BUF_PLANE0_PITCH_EXT,  drm_fd->pitch,
     };
 
+    if(is_plane_compressed(drm_fd->modifier)) {
+        static bool compressed_plane_warning_shown = false;
+        if(!compressed_plane_warning_shown) {
+            compressed_plane_warning_shown = true;
+            fprintf(stderr, "gsr warning: gsr_capture_kms_capture: the monitor plane is compressed. The video will likely be glitched/black. Try recording on X11 instead (maybe capturing a single window) or use the \"-w portal\" capture option on Wayland.\n");
+        }
+    }
+
     if(screen_plane_use_modifiers) {
         img_attr[12] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT;
         img_attr[13] = drm_fd->modifier & 0xFFFFFFFFULL;