Revert "amd/intel: use xrgb format, use drm prime 2"

This reverts commit 2f665e9d53d19f82af0973a169e6c2dc186e915f.

2 files changed, 46 insertions, 33 deletions

diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c
index ff666ed..f1630ea 100644
--- a/src/capture/kms_vaapi.c
+++ b/src/capture/kms_vaapi.c
@@ -245,35 +245,48 @@ static int gsr_capture_kms_vaapi_capture(gsr_capture *cap, AVFrame *frame) {
         cap_kms->input_surface = 0;
     }
 
-    VADRMPRIMESurfaceDescriptor buf = {0};
-    buf.fourcc = VA_FOURCC_XRGB;
+    uintptr_t dmabuf = cap_kms->dmabuf_fd;
+
+    VASurfaceAttribExternalBuffers buf = {0};
+    buf.pixel_format = VA_FOURCC_BGRX; // VA_FOURCC_XRGB
     buf.width = cap_kms->kms_size.x;
     buf.height = cap_kms->kms_size.y;
-
-    buf.num_objects = 0;
-    buf.objects[0].fd = cap_kms->dmabuf_fd;
-    buf.objects[0].size = cap_kms->kms_size.y * cap_kms->pitch;
-    buf.objects[0].drm_format_modifier = cap_kms->modifiers;
-
-    buf.num_layers = 1;
-    buf.layers[0].drm_format = cap_kms->fourcc;
-    buf.layers[0].num_planes = 1;
-    buf.layers[0].object_index[0] = 0;
-    buf.layers[0].offset[0] = cap_kms->offset;
-    buf.layers[0].pitch[0] = cap_kms->pitch;
-
-    VASurfaceAttrib attribs[2] = {0};
+    buf.data_size = cap_kms->kms_size.y * cap_kms->pitch;
+    buf.num_planes = 1;
+    buf.pitches[0] = cap_kms->pitch;
+    buf.offsets[0] = cap_kms->offset;
+    buf.buffers = &dmabuf;
+    buf.num_buffers = 1;
+    buf.flags = 0;
+    buf.private_data = 0;
+
+    VADRMFormatModifierList modifier_list = {0};
+    modifier_list.modifiers = &cap_kms->modifiers;
+    modifier_list.num_modifiers = 1;
+
+    #define VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME        0x20000000
+
+    VASurfaceAttrib attribs[3] = {0};
     attribs[0].type = VASurfaceAttribMemoryType;
     attribs[0].flags = VA_SURFACE_ATTRIB_SETTABLE;
     attribs[0].value.type = VAGenericValueTypeInteger;
-    attribs[0].value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2;
+    attribs[0].value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME; // TODO: prime1 instead?
     attribs[1].type = VASurfaceAttribExternalBufferDescriptor;
     attribs[1].flags = VA_SURFACE_ATTRIB_SETTABLE;
     attribs[1].value.type = VAGenericValueTypePointer;
     attribs[1].value.value.p = &buf;
+
+    int num_attribs = 2;
+    if(cap_kms->modifiers != DRM_FORMAT_MOD_INVALID) {
+        attribs[2].type = VASurfaceAttribDRMFormatModifiers;
+        attribs[2].flags = VA_SURFACE_ATTRIB_SETTABLE;
+        attribs[2].value.type = VAGenericValueTypePointer;
+        attribs[2].value.value.p = &modifier_list;
+        ++num_attribs;
+    }
     
     // TODO: Do we really need to create a new surface every frame?
-    VAStatus va_status = vaCreateSurfaces(cap_kms->va_dpy, VA_RT_FORMAT_RGB32, cap_kms->kms_size.x, cap_kms->kms_size.y, &cap_kms->input_surface, 1, attribs, 2);
+    VAStatus va_status = vaCreateSurfaces(cap_kms->va_dpy, VA_RT_FORMAT_RGB32, cap_kms->kms_size.x, cap_kms->kms_size.y, &cap_kms->input_surface, 1, attribs, num_attribs);
     if(va_status != VA_STATUS_SUCCESS) {
         fprintf(stderr, "gsr error: gsr_capture_kms_vaapi_capture: vaCreateSurfaces failed: %d\n", va_status);
         cap_kms->should_stop = true;
diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c
index cfa5057..f12a978 100644
--- a/src/capture/xcomposite_vaapi.c
+++ b/src/capture/xcomposite_vaapi.c
@@ -11,8 +11,6 @@
 #include <libavutil/hwcontext_vaapi.h>
 #include <libavutil/frame.h>
 #include <libavcodec/avcodec.h>
-#include <va/va.h>
-#include <va/va_drmcommon.h>
 
 typedef struct {
     gsr_capture_xcomposite_vaapi_params params;
@@ -385,28 +383,30 @@ static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *
 
         cap_xcomp->egl.eglDestroyImage(cap_xcomp->egl.egl_display, img);
 
-        VADRMPRIMESurfaceDescriptor buf = {0};
-        buf.fourcc = VA_FOURCC_XRGB;
+        uintptr_t dmabuf = cap_xcomp->dmabuf_fd;
+
+        VASurfaceAttribExternalBuffers buf = {0};
+        buf.pixel_format = VA_FOURCC_BGRX;
         buf.width = xx;
         buf.height = yy;
+        buf.data_size = yy * cap_xcomp->pitch;
+        buf.num_planes = 1;
+        buf.pitches[0] = cap_xcomp->pitch;
+        buf.offsets[0] = cap_xcomp->offset;
+        buf.buffers = &dmabuf;
+        buf.num_buffers = 1;
+        buf.flags = 0;
+        buf.private_data = 0;
 
-        buf.num_objects = 0;
-        buf.objects[0].fd = cap_xcomp->dmabuf_fd;
-        buf.objects[0].size = yy * cap_xcomp->pitch;
-        buf.objects[0].drm_format_modifier = cap_xcomp->modifiers;
+        // TODO: Use VASurfaceAttribDRMFormatModifiers to set modifier if modifier is not INVALID
 
-        buf.num_layers = 1;
-        buf.layers[0].drm_format = cap_xcomp->fourcc;
-        buf.layers[0].num_planes = 1;
-        buf.layers[0].object_index[0] = 0;
-        buf.layers[0].offset[0] = cap_xcomp->offset;
-        buf.layers[0].pitch[0] = cap_xcomp->pitch;
+        #define VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME        0x20000000
 
         VASurfaceAttrib attribs[2] = {0};
         attribs[0].type = VASurfaceAttribMemoryType;
         attribs[0].flags = VA_SURFACE_ATTRIB_SETTABLE;
         attribs[0].value.type = VAGenericValueTypeInteger;
-        attribs[0].value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2;
+        attribs[0].value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME; // TODO: prime1 instead?
         attribs[1].type = VASurfaceAttribExternalBufferDescriptor;
         attribs[1].flags = VA_SURFACE_ATTRIB_SETTABLE;
         attribs[1].value.type = VAGenericValueTypePointer;