render: choose DMA-BUF texture target via eglQueryDmaBufModifiersEXT

EGL_EXT_image_dma_buf_import_modifiers tells us whether we should use
GL_TEXTURE_2D or GL_TEXTURE_EXTERNAL_OES. Using the right texture target
can fix some failures and/or improve performance on some drivers.

This does the same as a Weston commit [1].

[1]: https://gitlab.freedesktop.org/wayland/weston/commit/40c519a3e613

Closes: swaywm#2173

include/wlr/render/egl.h

@@ -65,6 +65,7 @@ struct wlr_egl {
 	struct wl_display *wl_display;
 
 	struct wlr_drm_format_set dmabuf_formats;
+	EGLBoolean **external_only_dmabuf_formats;
 };
 
 // TODO: Allocate and return a wlr_egl
@@ -106,7 +107,7 @@ EGLImageKHR wlr_egl_create_image_from_wl_drm(struct wlr_egl *egl,
  * of the dmabuf with wlr_egl_check_import_dmabuf once first.
  */
 EGLImageKHR wlr_egl_create_image_from_dmabuf(struct wlr_egl *egl,
-	struct wlr_dmabuf_attributes *attributes);
+	struct wlr_dmabuf_attributes *attributes, bool *external_only);
 
 /**
  * Get the available dmabuf formats

render/egl.c

@@ -87,7 +87,7 @@ static void load_egl_proc(void *proc_ptr, const char *name) {
 
 static int get_egl_dmabuf_formats(struct wlr_egl *egl, int **formats);
 static int get_egl_dmabuf_modifiers(struct wlr_egl *egl, int format,
-	uint64_t **modifiers);
+	uint64_t **modifiers, EGLBoolean **external_only);
 
 static void init_dmabuf_formats(struct wlr_egl *egl) {
 	int *formats;
@@ -96,11 +96,20 @@ static void init_dmabuf_formats(struct wlr_egl *egl) {
 		return;
 	}
 
+	egl->external_only_dmabuf_formats = calloc(formats_len, sizeof(EGLBoolean *));
+	if (egl->external_only_dmabuf_formats == NULL) {
+		wlr_log_errno(WLR_ERROR, "Allocation failed");
+		goto out;
+	}
+
+	size_t external_formats_len = 0;
 	for (int i = 0; i < formats_len; i++) {
 		uint32_t fmt = formats[i];
 
 		uint64_t *modifiers;
-		int modifiers_len = get_egl_dmabuf_modifiers(egl, fmt, &modifiers);
+		EGLBoolean *external_only;
+		int modifiers_len =
+			get_egl_dmabuf_modifiers(egl, fmt, &modifiers, &external_only);
 		if (modifiers_len < 0) {
 			continue;
 		}
@@ -114,6 +123,9 @@ static void init_dmabuf_formats(struct wlr_egl *egl) {
 		}
 
 		free(modifiers);
+
+		egl->external_only_dmabuf_formats[external_formats_len] = external_only;
+		external_formats_len++;
 	}
 
 	char *str_formats = malloc(formats_len * 5 + 1);
@@ -479,8 +491,29 @@ EGLImageKHR wlr_egl_create_image_from_wl_drm(struct wlr_egl *egl,
 		EGL_WAYLAND_BUFFER_WL, data, attribs);
 }
 
+static bool dmabuf_format_is_external_only(struct wlr_egl *egl,
+		uint32_t format, uint64_t modifier) {
+	for (size_t i = 0; i < egl->dmabuf_formats.len; i++) {
+		struct wlr_drm_format *fmt = egl->dmabuf_formats.formats[i];
+		if (fmt->format == format) {
+			if (egl->external_only_dmabuf_formats[i] == NULL) {
+				break;
+			}
+			for (size_t j = 0; j < fmt->len; j++) {
+				if (fmt->modifiers[j] == modifier) {
+					return egl->external_only_dmabuf_formats[i][j];
+				}
+			}
+			break;
+		}
+	}
+	// No info, we're doomed. Choose GL_TEXTURE_EXTERNAL_OES and hope for the
+	// best.
+	return true;
+}
+
 EGLImageKHR wlr_egl_create_image_from_dmabuf(struct wlr_egl *egl,
-		struct wlr_dmabuf_attributes *attributes) {
+		struct wlr_dmabuf_attributes *attributes, bool *external_only) {
 	if (!egl->exts.image_base_khr || !egl->exts.image_dmabuf_import_ext) {
 		wlr_log(WLR_ERROR, "dmabuf import extension not present");
 		return NULL;
@@ -561,8 +594,16 @@ EGLImageKHR wlr_egl_create_image_from_dmabuf(struct wlr_egl *egl,
 	attribs[atti++] = EGL_NONE;
 	assert(atti < sizeof(attribs)/sizeof(attribs[0]));
 
-	return egl->procs.eglCreateImageKHR(egl->display, EGL_NO_CONTEXT,
+	EGLImageKHR image = egl->procs.eglCreateImageKHR(egl->display, EGL_NO_CONTEXT,
 		EGL_LINUX_DMA_BUF_EXT, NULL, attribs);
+	if (image == EGL_NO_IMAGE_KHR) {
+		wlr_log(WLR_ERROR, "eglCreateImageKHR failed");
+		return EGL_NO_IMAGE_KHR;
+	}
+
+	*external_only = dmabuf_format_is_external_only(egl,
+		attributes->format, attributes->modifier);
+	return image;
 }
 
 static int get_egl_dmabuf_formats(struct wlr_egl *egl, int **formats) {
@@ -615,13 +656,15 @@ static int get_egl_dmabuf_formats(struct wlr_egl *egl, int **formats) {
 }
 
 static int get_egl_dmabuf_modifiers(struct wlr_egl *egl, int format,
-		uint64_t **modifiers) {
+		uint64_t **modifiers, EGLBoolean **external_only) {
+	*modifiers = NULL;
+	*external_only = NULL;
+
 	if (!egl->exts.image_dmabuf_import_ext) {
 		wlr_log(WLR_DEBUG, "DMA-BUF extension not present");
 		return -1;
 	}
 	if (!egl->exts.image_dmabuf_import_modifiers_ext) {
-		*modifiers = NULL;
 		return 0;
 	}
 
@@ -632,18 +675,24 @@ static int get_egl_dmabuf_modifiers(struct wlr_egl *egl, int format,
 		return -1;
 	}
 	if (num == 0) {
-		*modifiers = NULL;
 		return 0;
 	}
 
 	*modifiers = calloc(num, sizeof(uint64_t));
 	if (*modifiers == NULL) {
-		wlr_log(WLR_ERROR, "Allocation failed: %s", strerror(errno));
+		wlr_log_errno(WLR_ERROR, "Allocation failed");
+		return -1;
+	}
+	*external_only = calloc(num, sizeof(EGLBoolean));
+	if (*external_only == NULL) {
+		wlr_log_errno(WLR_ERROR, "Allocation failed");
+		free(*modifiers);
+		*modifiers = NULL;
 		return -1;
 	}
 
 	if (!egl->procs.eglQueryDmaBufModifiersEXT(egl->display, format, num,
-		*modifiers, NULL, &num)) {
+			*modifiers, *external_only, &num)) {
 		wlr_log(WLR_ERROR, "Failed to query dmabuf modifiers");
 		free(*modifiers);
 		return -1;

render/gles2/texture.c

@@ -273,19 +273,22 @@ struct wlr_texture *wlr_gles2_texture_from_dmabuf(struct wlr_egl *egl,
 	wlr_texture_init(&texture->wlr_texture, &texture_impl,
 		attribs->width, attribs->height);
 	texture->egl = egl;
-	texture->target = GL_TEXTURE_EXTERNAL_OES;
 	texture->has_alpha = true;
 	texture->wl_format = 0xFFFFFFFF; // texture can't be written anyways
 	texture->inverted_y =
 		(attribs->flags & WLR_DMABUF_ATTRIBUTES_FLAGS_Y_INVERT) != 0;
 
-	texture->image = wlr_egl_create_image_from_dmabuf(egl, attribs);
+	bool external_only;
+	texture->image =
+		wlr_egl_create_image_from_dmabuf(egl, attribs, &external_only);
 	if (texture->image == EGL_NO_IMAGE_KHR) {
 		wlr_log(WLR_ERROR, "Failed to create EGL image from DMA-BUF");
 		free(texture);
 		return NULL;
 	}
 
+	texture->target = external_only ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
+
 	PUSH_GLES2_DEBUG;
 
 	glGenTextures(1, &texture->tex);