temp test for cosmic

4 files changed, 102 insertions, 40 deletions

diff --git a/kms/kms_shared.h b/kms/kms_shared.h
index 2dbb655..ca13e7e 100644
--- a/kms/kms_shared.h
+++ b/kms/kms_shared.h
@@ -5,7 +5,7 @@
 #include <stdbool.h>
 #include <drm_mode.h>
 
-#define GSR_KMS_PROTOCOL_VERSION 4
+#define GSR_KMS_PROTOCOL_VERSION 5
 
 #define GSR_KMS_MAX_ITEMS 8
 #define GSR_KMS_MAX_DMA_BUFS 4
@@ -39,6 +39,12 @@ struct gsr_kms_response_dma_buf {
     uint32_t offset;
 };
 
+typedef enum {
+    KMS_PLANE_TYPE_PRIMARY,
+    KMS_PLANE_TYPE_CURSOR,
+    KMS_PLANE_TYPE_OVERLAY
+} gsr_kms_plane_type;
+
 struct gsr_kms_response_item {
     gsr_kms_response_dma_buf dma_buf[GSR_KMS_MAX_DMA_BUFS];
     int num_dma_bufs;
@@ -47,7 +53,7 @@ struct gsr_kms_response_item {
     uint32_t pixel_format;
     uint64_t modifier;
     uint32_t connector_id; /* 0 if unknown */
-    bool is_cursor;
+    gsr_kms_plane_type plane_type;
     bool has_hdr_metadata;
     int x;
     int y;
diff --git a/kms/server/kms_server.c b/kms/server/kms_server.c
index 070875b..88d4b73 100644
--- a/kms/server/kms_server.c
+++ b/kms/server/kms_server.c
@@ -145,12 +145,13 @@ typedef enum {
     PLANE_PROPERTY_SRC_H      = 1 << 5,
     PLANE_PROPERTY_IS_CURSOR  = 1 << 6,
     PLANE_PROPERTY_IS_PRIMARY = 1 << 7,
+    PLANE_PROPERTY_IS_OVERLAY = 1 << 8,
 } plane_property_mask;
 
 /* Returns plane_property_mask */
-static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *y, int *src_x, int *src_y, int *src_w, int *src_h) {
-    *x = 0;
-    *y = 0;
+static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *crtc_x, int *crtc_y, int *src_x, int *src_y, int *src_w, int *src_h) {
+    *crtc_x = 0;
+    *crtc_y = 0;
     *src_x = 0;
     *src_y = 0;
     *src_w = 0;
@@ -171,10 +172,10 @@ static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *
         // SRC_* values are fixed 16.16 points
         const uint32_t type = prop->flags & (DRM_MODE_PROP_LEGACY_TYPE | DRM_MODE_PROP_EXTENDED_TYPE);
         if((type & DRM_MODE_PROP_SIGNED_RANGE) && strcmp(prop->name, "CRTC_X") == 0) {
-            *x = (int)props->prop_values[i];
+            *crtc_x = (int)props->prop_values[i];
             property_mask |= PLANE_PROPERTY_X;
         } else if((type & DRM_MODE_PROP_SIGNED_RANGE) && strcmp(prop->name, "CRTC_Y") == 0) {
-            *y = (int)props->prop_values[i];
+            *crtc_y = (int)props->prop_values[i];
             property_mask |= PLANE_PROPERTY_Y;
         } else if((type & DRM_MODE_PROP_RANGE) && strcmp(prop->name, "SRC_X") == 0) {
             *src_x = (int)(props->prop_values[i] >> 16);
@@ -197,6 +198,9 @@ static uint32_t plane_get_properties(int drmfd, uint32_t plane_id, int *x, int *
                 } else if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Cursor") == 0) {
                     property_mask |= PLANE_PROPERTY_IS_CURSOR;
                     break;
+                } else if(prop->enums[j].value == current_enum_value && strcmp(prop->enums[j].name, "Overlay") == 0) {
+                    property_mask |= PLANE_PROPERTY_IS_OVERLAY;
+                    break;
                 }
             }
         }
@@ -289,6 +293,16 @@ static int drm_prime_handles_to_fds(gsr_drm *drm, drmModeFB2Ptr drmfb, int *fb_f
     return GSR_KMS_MAX_DMA_BUFS;
 }
 
+static int plane_property_get_plane_type(plane_property_mask property_mask) {
+    if(property_mask & PLANE_PROPERTY_IS_PRIMARY)
+        return KMS_PLANE_TYPE_PRIMARY;
+    else if(property_mask & PLANE_PROPERTY_IS_CURSOR)
+        return KMS_PLANE_TYPE_CURSOR;
+    else if(property_mask & PLANE_PROPERTY_IS_OVERLAY)
+        return KMS_PLANE_TYPE_OVERLAY;
+    return -1;
+}
+
 static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response) {
     int result = -1;
 
@@ -340,9 +354,10 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response) {
         // TODO: Check if dimensions have changed by comparing width and height to previous time this was called.
         // TODO: Support other plane formats than rgb (with multiple planes, such as direct YUV420 on wayland).
 
-        int x = 0, y = 0, src_x = 0, src_y = 0, src_w = 0, src_h = 0;
-        plane_property_mask property_mask = plane_get_properties(drm->drmfd, plane->plane_id, &x, &y, &src_x, &src_y, &src_w, &src_h);
-        if(!(property_mask & PLANE_PROPERTY_IS_PRIMARY) && !(property_mask & PLANE_PROPERTY_IS_CURSOR))
+        int crtc_x = 0, crtc_y = 0, src_x = 0, src_y = 0, src_w = 0, src_h = 0;
+        const plane_property_mask property_mask = plane_get_properties(drm->drmfd, plane->plane_id, &crtc_x, &crtc_y, &src_x, &src_y, &src_w, &src_h);
+        const int plane_type = plane_property_get_plane_type(property_mask);
+        if(plane_type == -1)
             continue;
 
         int fb_fds[GSR_KMS_MAX_DMA_BUFS];
@@ -354,38 +369,50 @@ static int kms_get_fb(gsr_drm *drm, gsr_kms_response *response) {
             goto cleanup_handles;
         }
 
-        const int item_index = response->num_items;
+        gsr_kms_response_item *response_item = &response->items[response->num_items];
 
         const connector_crtc_pair *crtc_pair = get_connector_pair_by_crtc_id(&c2crtc_map, plane->crtc_id);
         if(crtc_pair && crtc_pair->hdr_metadata_blob_id) {
-            response->items[item_index].has_hdr_metadata = get_hdr_metadata(drm->drmfd, crtc_pair->hdr_metadata_blob_id, &response->items[item_index].hdr_metadata);
+            response_item->has_hdr_metadata = get_hdr_metadata(drm->drmfd, crtc_pair->hdr_metadata_blob_id, &response_item->hdr_metadata);
         } else {
-            response->items[item_index].has_hdr_metadata = false;
+            response_item->has_hdr_metadata = false;
         }
 
         for(int j = 0; j < num_fb_fds; ++j) {
-            response->items[item_index].dma_buf[j].fd = fb_fds[j];
-            response->items[item_index].dma_buf[j].pitch = drmfb->pitches[j];
-            response->items[item_index].dma_buf[j].offset = drmfb->offsets[j];
+            response_item->dma_buf[j].fd = fb_fds[j];
+            response_item->dma_buf[j].pitch = drmfb->pitches[j];
+            response_item->dma_buf[j].offset = drmfb->offsets[j];
         }
-        response->items[item_index].num_dma_bufs = num_fb_fds;
-
-        response->items[item_index].width = drmfb->width;
-        response->items[item_index].height = drmfb->height;
-        response->items[item_index].pixel_format = drmfb->pixel_format;
-        response->items[item_index].modifier = drmfb->flags & DRM_MODE_FB_MODIFIERS ? drmfb->modifier : DRM_FORMAT_MOD_INVALID;
-        response->items[item_index].connector_id = crtc_pair ? crtc_pair->connector_id : 0;
-        response->items[item_index].is_cursor = property_mask & PLANE_PROPERTY_IS_CURSOR;
-        if(property_mask & PLANE_PROPERTY_IS_CURSOR) {
-            response->items[item_index].x = x;
-            response->items[item_index].y = y;
-            response->items[item_index].src_w = 0;
-            response->items[item_index].src_h = 0;
-        } else {
-            response->items[item_index].x = src_x;
-            response->items[item_index].y = src_y;
-            response->items[item_index].src_w = src_w;
-            response->items[item_index].src_h = src_h;
+        response_item->num_dma_bufs = num_fb_fds;
+
+        response_item->width = drmfb->width;
+        response_item->height = drmfb->height;
+        response_item->pixel_format = drmfb->pixel_format;
+        response_item->modifier = drmfb->flags & DRM_MODE_FB_MODIFIERS ? drmfb->modifier : DRM_FORMAT_MOD_INVALID;
+        response_item->connector_id = crtc_pair ? crtc_pair->connector_id : 0;
+        response_item->plane_type = plane_type;
+        switch(response_item->plane_type) {
+            case KMS_PLANE_TYPE_PRIMARY: {
+                response_item->x = src_x;
+                response_item->y = src_y;
+                response_item->src_w = src_w;
+                response_item->src_h = src_h;
+                break;
+            }
+            case KMS_PLANE_TYPE_CURSOR: {
+                response_item->x = crtc_x;
+                response_item->y = crtc_y;
+                response_item->src_w = 0;
+                response_item->src_h = 0;
+                break;
+            }
+            case KMS_PLANE_TYPE_OVERLAY: {
+                response_item->x = crtc_x;
+                response_item->y = crtc_y;
+                response_item->src_w = src_w;
+                response_item->src_h = src_y;
+                break;
+            }
         }
         ++response->num_items;
 
diff --git a/project.conf b/project.conf
index 0f3f566..ff62ad5 100644
--- a/project.conf
+++ b/project.conf
@@ -5,7 +5,7 @@ version = "5.5.3"
 platforms = ["posix"]
 
 [config]
-ignore_dirs = ["kms/server", "build", "debug-build"]
+ignore_dirs = ["kms/server", "build", "debug-build", "dbus/server"]
 #error_on_warning = "true"
 
 [define]
diff --git a/src/capture/kms.c b/src/capture/kms.c
index 18858f2..552b7e5 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -264,14 +264,21 @@ static void gsr_capture_kms_on_event(gsr_capture *cap, gsr_egl *egl) {
 //     }
 // }
 
-static gsr_kms_response_item* find_drm_by_connector_id(gsr_kms_response *kms_response, uint32_t connector_id) {
+static gsr_kms_response_item* find_drm_by_connector_id_and_plane_type(gsr_kms_response *kms_response, uint32_t connector_id, gsr_kms_plane_type plane_type) {
     for(int i = 0; i < kms_response->num_items; ++i) {
-        if(kms_response->items[i].connector_id == connector_id && !kms_response->items[i].is_cursor)
+        if(kms_response->items[i].connector_id == connector_id && kms_response->items[i].plane_type == plane_type)
             return &kms_response->items[i];
     }
     return NULL;
 }
 
+static gsr_kms_response_item* find_drm_by_connector_id(gsr_kms_response *kms_response, uint32_t connector_id) {
+    gsr_kms_response_item *response_item = find_drm_by_connector_id_and_plane_type(kms_response, connector_id, KMS_PLANE_TYPE_PRIMARY);
+    if(!response_item)
+        response_item = find_drm_by_connector_id_and_plane_type(kms_response, connector_id, KMS_PLANE_TYPE_OVERLAY);
+    return response_item;
+}
+
 static gsr_kms_response_item* find_largest_drm(gsr_kms_response *kms_response) {
     if(kms_response->num_items == 0)
         return NULL;
@@ -280,7 +287,7 @@ static gsr_kms_response_item* find_largest_drm(gsr_kms_response *kms_response) {
     gsr_kms_response_item *largest_drm = &kms_response->items[0];
     for(int i = 0; i < kms_response->num_items; ++i) {
         const int64_t size = (int64_t)kms_response->items[i].width * (int64_t)kms_response->items[i].height;
-        if(size > largest_size && !kms_response->items[i].is_cursor) {
+        if(size > largest_size && kms_response->items[i].plane_type == KMS_PLANE_TYPE_PRIMARY) {
             largest_size = size;
             largest_drm = &kms_response->items[i];
         }
@@ -291,7 +298,7 @@ static gsr_kms_response_item* find_largest_drm(gsr_kms_response *kms_response) {
 static gsr_kms_response_item* find_cursor_drm(gsr_kms_response *kms_response, uint32_t connector_id) {
     gsr_kms_response_item *cursor_drm = NULL;
     for(int i = 0; i < kms_response->num_items; ++i) {
-        if(kms_response->items[i].is_cursor) {
+        if(kms_response->items[i].plane_type == KMS_PLANE_TYPE_CURSOR) {
             cursor_drm = &kms_response->items[i];
             if(kms_response->items[i].connector_id == connector_id)
                 break;
@@ -649,6 +656,28 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
         capture_pos, self->capture_size, original_frame_size,
         gsr_monitor_rotation_to_rotation(self->monitor_rotation), GSR_SOURCE_COLOR_RGB, self->external_texture_fallback, false);
 
+    if(drm_fd->plane_type != KMS_PLANE_TYPE_OVERLAY) {
+        const vec2i framebuffer_size = original_frame_size;
+        for(int i = 0; i < self->monitor_id.num_connector_ids; ++i) {
+            const gsr_kms_response_item *overlay_drm_fd = find_drm_by_connector_id_and_plane_type(&self->kms_response, self->monitor_id.connector_ids[i], KMS_PLANE_TYPE_OVERLAY);
+            if(!overlay_drm_fd)
+                continue;
+
+            self->external_texture_fallback = false;
+            image = gsr_capture_kms_create_egl_image_with_fallback(self, overlay_drm_fd);
+            if(image) {
+                gsr_capture_kms_bind_image_to_input_texture_with_fallback(self, image);
+                self->params.egl->eglDestroyImage(self->params.egl->egl_display, image);
+            }
+
+            // TODO: Lole
+            gsr_color_conversion_draw(color_conversion, self->external_texture_fallback ? self->external_input_texture_id : self->input_texture_id,
+                target_pos, output_size,
+                (vec2i){overlay_drm_fd->x, overlay_drm_fd->y}, framebuffer_size, original_frame_size,
+                gsr_monitor_rotation_to_rotation(self->monitor_rotation), GSR_SOURCE_COLOR_RGB, self->external_texture_fallback, true);
+        }
+    }
+
     if(self->params.record_cursor) {
         gsr_kms_response_item *cursor_drm_fd = find_cursor_drm_if_on_monitor(self, drm_fd->connector_id, capture_is_combined_plane);
         // The cursor is handled by x11 on x11 instead of using the cursor drm plane because on prime systems with a dedicated nvidia gpu
@@ -661,7 +690,7 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
             cursor_monitor_offset.y += self->params.region_position.y;
             render_x11_cursor(self, color_conversion, cursor_monitor_offset, target_pos, output_size);
         } else if(cursor_drm_fd) {
-            const vec2i framebuffer_size = rotate_capture_size_if_rotated(self, (vec2i){ drm_fd->src_w, drm_fd->src_h });
+            const vec2i framebuffer_size = original_frame_size;
             render_drm_cursor(self, color_conversion, cursor_drm_fd, target_pos, output_size, framebuffer_size);
         }
     }