Fix AMD single monitor rotated display being rotated in recording

If there is only one monitor connected and it's rotated then
the drm buf will also be rotated. This only the case with AMD and
only when using one monitor!

To fix this, we perform color conversion with an opengl shader
which allows us to also rotate the texture.
VAAPI supports rotation but it's not implemented by AMD at least.

Performance seems to be the same as when using VAAPI, even when
GPU usage is 100%.

6 files changed, 141 insertions, 2 deletions

diff --git a/include/capture/kms_vaapi.h b/include/capture/kms_vaapi.h
index ba5763b..a4aa3ed 100644
--- a/include/capture/kms_vaapi.h
+++ b/include/capture/kms_vaapi.h
@@ -1,8 +1,9 @@
 #ifndef GSR_CAPTURE_KMS_VAAPI_H
 #define GSR_CAPTURE_KMS_VAAPI_H
 
-#include "capture.h"
 #include "../vec2.h"
+#include "../utils.h"
+#include "capture.h"
 #include <X11/X.h>
 
 typedef struct _XDisplay Display;
@@ -10,6 +11,7 @@ typedef struct _XDisplay Display;
 typedef struct {
     Display *dpy;
     const char *display_to_capture; /* if this is "screen", then the entire x11 screen is captured (all displays). A copy is made of this */
+    gsr_gpu_info gpu_inf;
 } gsr_capture_kms_vaapi_params;
 
 gsr_capture* gsr_capture_kms_vaapi_create(const gsr_capture_kms_vaapi_params *params);
diff --git a/include/color_conversion.h b/include/color_conversion.h
new file mode 100644
index 0000000..de83d1e
--- /dev/null
+++ b/include/color_conversion.h
@@ -0,0 +1,51 @@
+#ifndef GSR_COLOR_CONVERSION_H
+#define GSR_COLOR_CONVERSION_H
+
+#include "shader.h"
+#include "vec2.h"
+
+typedef enum {
+    GSR_SOURCE_COLOR_RGB
+} gsr_source_color;
+
+typedef enum {
+    GSR_DESTINATION_COLOR_NV12
+} gsr_destination_color;
+
+typedef struct {
+    gsr_egl *egl;
+
+    gsr_source_color source_color;
+    gsr_destination_color destination_color;
+
+    unsigned int source_textures[2];
+    int num_source_textures;
+
+    unsigned int destination_textures[2];
+    int num_destination_textures;
+
+    float rotation;
+
+    vec2f position;
+    vec2f size;
+} gsr_color_conversion_params;
+
+typedef struct {
+    gsr_egl *egl;
+    gsr_shader shaders[2];
+
+    unsigned int source_textures[2];
+    unsigned int destination_textures[2];
+
+    unsigned int framebuffers[2];
+
+    unsigned int vertex_array_object_id;
+    unsigned int vertex_buffer_object_id;
+} gsr_color_conversion;
+
+int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conversion_params *params);
+void gsr_color_conversion_deinit(gsr_color_conversion *self);
+
+int gsr_color_conversion_update(gsr_color_conversion *self, int width, int height);
+
+#endif /* GSR_COLOR_CONVERSION_H */
diff --git a/include/egl.h b/include/egl.h
index 1056a5c..891c7a0 100644
--- a/include/egl.h
+++ b/include/egl.h
@@ -44,7 +44,18 @@ typedef void (*__eglMustCastToProperFunctionPointerType)(void);
 #define EGL_TRUE                                1
 #define EGL_IMAGE_PRESERVED_KHR                 0x30D2
 #define EGL_NATIVE_PIXMAP_KHR                   0x30B0
+#define EGL_LINUX_DRM_FOURCC_EXT                0x3271
+#define EGL_WIDTH                               0x3057
+#define EGL_HEIGHT                              0x3056
+#define EGL_DMA_BUF_PLANE0_FD_EXT               0x3272
+#define EGL_DMA_BUF_PLANE0_OFFSET_EXT           0x3273
+#define EGL_DMA_BUF_PLANE0_PITCH_EXT            0x3274
+#define EGL_LINUX_DMA_BUF_EXT                   0x3270
 
+#define GL_FLOAT                                0x1406
+#define GL_FALSE                                0
+#define GL_TRUE                                 1
+#define GL_TRIANGLES                            0x0004
 #define GL_TEXTURE_2D                           0x0DE1
 #define GL_RGB                                  0x1907
 #define GL_UNSIGNED_BYTE                        0x1401
@@ -67,6 +78,13 @@ typedef void (*__eglMustCastToProperFunctionPointerType)(void);
 #define GL_VENDOR                               0x1F00
 #define GL_RENDERER                             0x1F01
 
+#define GL_COMPILE_STATUS                       0x8B81
+#define GL_INFO_LOG_LENGTH                      0x8B84
+#define GL_FRAGMENT_SHADER                      0x8B30
+#define GL_VERTEX_SHADER                        0x8B31
+#define GL_COMPILE_STATUS                       0x8B81
+#define GL_LINK_STATUS                          0x8B82
+
 typedef struct {
     void *egl_library;
     void *gl_library;
@@ -108,6 +126,38 @@ typedef struct {
     void (*glTexImage2D)(unsigned int target, int level, int internalFormat, int width, int height, int border, unsigned int format, unsigned int type, const void *pixels);
     void (*glCopyImageSubData)(unsigned int srcName, unsigned int srcTarget, int srcLevel, int srcX, int srcY, int srcZ, unsigned int dstName, unsigned int dstTarget, int dstLevel, int dstX, int dstY, int dstZ, int srcWidth, int srcHeight, int srcDepth);
     void (*glClearTexImage)(unsigned int texture, unsigned int level, unsigned int format, unsigned int type, const void *data);
+    void (*glGenFramebuffers)(int n, unsigned int *framebuffers);
+    void (*glBindFramebuffer)(unsigned int target, unsigned int framebuffer);
+    void (*glDeleteFramebuffers)(int n, const unsigned int *framebuffers);
+    void (*glViewport)(int x, int y, int width, int height);
+    void (*glFramebufferTexture2D)(unsigned int target, unsigned int attachment, unsigned int textarget, unsigned int texture, int level);
+    void (*glDrawBuffers)(int n, const unsigned int *bufs);
+    unsigned int (*glCheckFramebufferStatus)(unsigned int target);
+    void (*glBindBuffer)(unsigned int target, unsigned int buffer);
+    void (*glGenBuffers)(int n, unsigned int *buffers);
+    void (*glBufferData)(unsigned int target, khronos_ssize_t size, const void *data, unsigned int usage);
+    void (*glDeleteBuffers)(int n, const unsigned int *buffers);
+    void (*glGenVertexArrays)(int n, unsigned int *arrays);
+    void (*glBindVertexArray)(unsigned int array);
+    void (*glDeleteVertexArrays)(int n, const unsigned int *arrays);
+
+    unsigned int (*glCreateProgram)(void);
+    unsigned int (*glCreateShader)(unsigned int type);
+    void (*glAttachShader)(unsigned int program, unsigned int shader);
+    void (*glBindAttribLocation)(unsigned int program, unsigned int index, const char *name);
+    void (*glCompileShader)(unsigned int shader);
+    void (*glLinkProgram)(unsigned int program);
+    void (*glShaderSource)(unsigned int shader, int count, const char *const*string, const int *length);
+    void (*glUseProgram)(unsigned int program);
+    void (*glGetProgramInfoLog)(unsigned int program, int bufSize, int *length, char *infoLog);
+    void (*glGetShaderiv)(unsigned int shader, unsigned int pname, int *params);
+    void (*glGetShaderInfoLog)(unsigned int shader, int bufSize, int *length, char *infoLog);
+    void (*glDeleteProgram)(unsigned int program);
+    void (*glDeleteShader)(unsigned int shader);
+    void (*glGetProgramiv)(unsigned int program, unsigned int pname, int *params);
+    void (*glVertexAttribPointer)(unsigned int index, int size, unsigned int type, unsigned char normalized, int stride, const void *pointer);
+    void (*glEnableVertexAttribArray)(unsigned int index);
+    void (*glDrawArrays)(unsigned int mode, int first, int count );
 } gsr_egl;
 
 bool gsr_egl_load(gsr_egl *self, Display *dpy);
diff --git a/include/shader.h b/include/shader.h
new file mode 100644
index 0000000..37f4c09
--- /dev/null
+++ b/include/shader.h
@@ -0,0 +1,19 @@
+#ifndef GSR_SHADER_H
+#define GSR_SHADER_H
+
+#include "egl.h"
+
+typedef struct {
+    gsr_egl *egl;
+    unsigned int program_id;
+} gsr_shader;
+
+/* |vertex_shader| or |fragment_shader| may be NULL */
+int gsr_shader_init(gsr_shader *self, gsr_egl *egl, const char *vertex_shader, const char *fragment_shader);
+void gsr_shader_deinit(gsr_shader *self);
+
+int gsr_shader_bind_attribute_location(gsr_shader *self, const char *attribute, int location);
+void gsr_shader_use(gsr_shader *self);
+void gsr_shader_use_none(gsr_shader *self);
+
+#endif /* GSR_SHADER_H */
diff --git a/include/utils.h b/include/utils.h
index f4925e5..fb1365b 100644
--- a/include/utils.h
+++ b/include/utils.h
@@ -5,6 +5,17 @@
 #include <stdbool.h>
 #include <X11/extensions/Xrandr.h>
 
+typedef enum {
+    GSR_GPU_VENDOR_AMD,
+    GSR_GPU_VENDOR_INTEL,
+    GSR_GPU_VENDOR_NVIDIA
+} gsr_gpu_vendor;
+
+typedef struct {
+    gsr_gpu_vendor vendor;
+    int gpu_version; /* 0 if unknown */
+} gsr_gpu_info;
+
 typedef struct {
     vec2i pos;
     vec2i size;
@@ -23,4 +34,6 @@ typedef void (*active_monitor_callback)(const XRROutputInfo *output_info, const
 void for_each_active_monitor_output(Display *display, active_monitor_callback callback, void *userdata);
 bool get_monitor_by_name(Display *display, const char *name, gsr_monitor *monitor);
 
+bool gl_get_gpu_info(Display *dpy, gsr_gpu_info *info);
+
 #endif /* GSR_UTILS_H */
diff --git a/include/vec2.h b/include/vec2.h
index d1957ef..3e33cfb 100644
--- a/include/vec2.h
+++ b/include/vec2.h
@@ -5,4 +5,8 @@ typedef struct {
     int x, y;
 } vec2i;
 
-#endif /* VEC2_H */
\ No newline at end of file
+typedef struct {
+    float x, y;
+} vec2f;
+
+#endif /* VEC2_H */