Hold client buffers, version 2

src/drm.cpp

@@ -13,9 +13,14 @@
 #include <sys/select.h>
 #include <signal.h>
 
+extern "C" {
+#include <wlr/types/wlr_buffer.h>
+}
+
 #include "drm.hpp"
 #include "main.hpp"
 #include "vblankmanager.hpp"
+#include "wlserver.hpp"
 
 #include "gpuvis_trace_utils.h"
 
@@ -631,7 +636,7 @@ int drm_atomic_commit(struct drm_t *drm, struct Composite_t *pComposite, struct
 	return ret;
 }
 
-uint32_t drm_fbid_from_dmabuf( struct drm_t *drm, struct wlr_dmabuf_attributes *dma_buf )
+uint32_t drm_fbid_from_dmabuf( struct drm_t *drm, struct wlr_buffer *buf, struct wlr_dmabuf_attributes *dma_buf )
 {
 	assert( dma_buf->n_planes == 1);
 
@@ -655,7 +660,15 @@ uint32_t drm_fbid_from_dmabuf( struct drm_t *drm, struct wlr_dmabuf_attributes *
 	}
 	assert( drm->map_fbid_inflightflips[ fb_id ].held == false );
 
+	if ( buf != nullptr )
+	{
+		wlserver_lock();
+		buf = wlr_buffer_lock( buf );
+		wlserver_unlock();
+	}
+
 	drm->map_fbid_inflightflips[ fb_id ].id = fb_id;
+	drm->map_fbid_inflightflips[ fb_id ].buf = buf;
 	drm->map_fbid_inflightflips[ fb_id ].held = true;
 	drm->map_fbid_inflightflips[ fb_id ].n_refs = 0;
 
@@ -672,6 +685,13 @@ static void drm_free_fb( struct drm_t *drm, struct fb *fb )
 		perror( "drmModeRmFB failed" );
 	}
 
+	if ( fb->buf != nullptr )
+	{
+		wlserver_lock();
+		wlr_buffer_unlock( fb->buf );
+		wlserver_unlock();
+	}
+
 	fb = {};
 }
 

src/drm.hpp

@@ -40,6 +40,8 @@ struct connector {
 
 struct fb {
 	uint32_t id;
+	/* Client buffer, if any */
+	struct wlr_buffer *buf;
 	/* A FB is held if it's being used by steamcompmgr */
 	bool held;
 	/* Number of page-flips using the FB */
@@ -97,6 +99,6 @@ extern bool g_bDebugLayers;
 
 int init_drm(struct drm_t *drm, const char *device, const char *mode_str, unsigned int vrefresh);
 int drm_atomic_commit(struct drm_t *drm, struct Composite_t *pComposite, struct VulkanPipeline_t *pPipeline );
-uint32_t drm_fbid_from_dmabuf( struct drm_t *drm, struct wlr_dmabuf_attributes *dma_buf );
+uint32_t drm_fbid_from_dmabuf( struct drm_t *drm, struct wlr_buffer *buf, struct wlr_dmabuf_attributes *dma_buf );
 void drm_drop_fbid( struct drm_t *drm, uint32_t fbid );
 bool drm_can_avoid_composite( struct drm_t *drm, struct Composite_t *pComposite, struct VulkanPipeline_t *pPipeline );

src/main.cpp

@@ -171,14 +171,14 @@ int initOutput(void)
 	}
 }
 
-void wayland_commit(struct wlr_surface *surf, struct wlr_dmabuf_attributes *attribs)
+void wayland_commit(struct wlr_surface *surf, struct wlr_buffer *buf)
 {
 	{
 		std::lock_guard<std::mutex> lock( wayland_commit_lock );
 
 		ResListEntry_t newEntry = {
 			.surf = surf,
-			.attribs = *attribs
+			.buf = buf,
 		};
 		wayland_commit_queue.push_back( newEntry );
 	}

src/main.hpp

@@ -9,7 +9,7 @@ void startSteamCompMgr(void);
 
 void register_signal(void);
 
-void wayland_commit(struct wlr_surface *surf, struct wlr_dmabuf_attributes *attribs);
+void wayland_commit(struct wlr_surface *surf, struct wlr_buffer *buf);
 
 extern int g_nNestedWidth;
 extern int g_nNestedHeight;

src/rendervulkan.cpp

@@ -347,6 +347,7 @@ bool CVulkanTexture::BInit( uint32_t width, uint32_t height, VkFormat format, bo
 		// We assume we own the memory when doing this right now.
 		// We could support the import scenario as well if needed
 // 		assert( bTextureable == false );
+		assert( pDMA == nullptr );
 
 		m_DMA.modifier = DRM_FORMAT_MOD_INVALID;
 		m_DMA.n_planes = 1;
@@ -376,7 +377,7 @@ bool CVulkanTexture::BInit( uint32_t width, uint32_t height, VkFormat format, bo
 
 		m_DMA.stride[0] = image_layout.rowPitch;
 
-		m_FBID = drm_fbid_from_dmabuf( &g_DRM, &m_DMA );
+		m_FBID = drm_fbid_from_dmabuf( &g_DRM, nullptr, &m_DMA );
 
 		if ( m_FBID == 0 )
 			return false;

src/steamcompmgr.cpp

@@ -81,6 +81,7 @@ uint64_t maxCommmitID;
 
 struct commit_t
 {
+	struct wlr_buffer *buf;
 	uint32_t fb_id;
 	VulkanTexture_t vulkanTex;
 	uint64_t commitID;
@@ -515,14 +516,20 @@ release_commit ( commit_t &commit )
 		vulkan_free_texture( commit.vulkanTex );
 		commit.vulkanTex = 0;
 	}
+
+	wlserver_lock();
+	wlr_buffer_unlock( commit.buf );
+	wlserver_unlock();
 }
 
 static bool
-import_commit ( struct wlr_dmabuf_attributes *dmabuf, commit_t &commit )
+import_commit ( struct wlr_buffer *buf, struct wlr_dmabuf_attributes *dmabuf, commit_t &commit )
 {
+	commit.buf = buf;
+
 	if ( BIsNested() == False )
 	{
-		commit.fb_id = drm_fbid_from_dmabuf( &g_DRM, dmabuf );
+		commit.fb_id = drm_fbid_from_dmabuf( &g_DRM, buf, dmabuf );
 		assert( commit.fb_id != 0 );
 	}
 
@@ -2067,25 +2074,48 @@ void handle_done_commits( void )
 
 void check_new_wayland_res( void )
 {
-	std::lock_guard<std::mutex> lock( wayland_commit_lock );
+	// When importing buffer, we'll potentially need to perform operations with
+	// a wlserver lock (e.g. wlr_buffer_lock). We can't do this with a
+	// wayland_commit_queue lock because that causes deadlocks.
+	std::vector<ResListEntry_t> tmp_queue;
+	{
+		std::lock_guard<std::mutex> lock( wayland_commit_lock );
+		tmp_queue = wayland_commit_queue;
+		wayland_commit_queue.clear();
+	}
 
-	for ( uint32_t i = 0; i < wayland_commit_queue.size(); i++ )
+	for ( uint32_t i = 0; i < tmp_queue.size(); i++ )
 	{
-		win	*w = find_win( wayland_commit_queue[ i ].surf );
+		struct wlr_buffer *buf = tmp_queue[ i ].buf;
 
-		assert( wayland_commit_queue[ i ].attribs.fd[0] != -1 );
+		win	*w = find_win( tmp_queue[ i ].surf );
 
 		if ( w == nullptr )
 		{
-			wlr_dmabuf_attributes_finish( &wayland_commit_queue[ i ].attribs );
+			wlserver_lock();
+			wlr_buffer_unlock( buf );
+			wlserver_unlock();
 			fprintf (stderr, "waylandres but no win\n");
 			continue;
 		}
 
-		commit_t newCommit = {};
+		struct wlr_dmabuf_attributes dmabuf = {};
+		bool result = False;
+		if ( wlr_buffer_get_dmabuf( buf, &dmabuf ) ) {
+			result = true;
+			for ( int i = 0; i < dmabuf.n_planes; i++ ) {
+				dmabuf.fd[i] = dup( dmabuf.fd[i] );
+				assert( dmabuf.fd[i] >= 0 );
+			}
+		} else {
+			struct wlr_client_buffer *client_buf = (struct wlr_client_buffer *) buf;
+			result = wlr_texture_to_dmabuf( client_buf->texture, &dmabuf );
+		}
+		assert( result == true );
 
-		bool bSuccess = import_commit( &wayland_commit_queue[ i ].attribs, newCommit );
-		wlr_dmabuf_attributes_finish( &wayland_commit_queue[ i ].attribs );
+		commit_t newCommit = {};
+		bool bSuccess = import_commit( buf, &dmabuf, newCommit );
+		wlr_dmabuf_attributes_finish( &dmabuf );
 
 		if ( bSuccess == true )
 		{
@@ -2105,8 +2135,6 @@ void check_new_wayland_res( void )
 		// Wake up commit wait thread if chilling
 		waitListSem.signal();
 	}
-
-	wayland_commit_queue.clear();
 }
 
 int

src/steamcompmgr.hpp

@@ -1,5 +1,10 @@
 #include <stdint.h>
 
+extern "C" {
+#include <wlr/types/wlr_buffer.h>
+#include <wlr/render/wlr_texture.h>
+}
+
 extern uint32_t currentOutputWidth;
 extern uint32_t currentOutputHeight;
 
@@ -18,7 +23,7 @@ int steamcompmgr_main(int argc, char **argv);
 
 struct ResListEntry_t {
 	struct wlr_surface *surf;
-	struct wlr_dmabuf_attributes attribs;
+	struct wlr_buffer *buf;
 };
 
 struct _XDisplay;

src/wlserver.cpp

@@ -76,30 +76,17 @@ extern const struct wlr_surface_role xwayland_surface_role;
 
 void xwayland_surface_role_commit(struct wlr_surface *wlr_surface) {
 	assert(wlr_surface->role == &xwayland_surface_role);
-
-	struct wlr_texture *tex = wlr_surface_get_texture( wlr_surface );
 
-	if ( tex == NULL )
+	if ( wlr_surface->buffer == NULL )
 	{
 		return;
 	}
-
-	struct wlr_dmabuf_attributes dmabuf_attribs = {};
-	bool result = False;
-	if ( wlr_buffer_get_dmabuf( &wlr_surface->buffer->base, &dmabuf_attribs ) ) {
-		result = true;
-		for ( int i = 0; i < dmabuf_attribs.n_planes; i++ ) {
-			dmabuf_attribs.fd[i] = dup( dmabuf_attribs.fd[i] );
-			assert( dmabuf_attribs.fd[i] >= 0 );
-		}
-	} else {
-		result = wlr_texture_to_dmabuf( tex, &dmabuf_attribs );
-	}
-	assert( result == true );
-
+
+	struct wlr_buffer *buf = wlr_buffer_lock( &wlr_surface->buffer->base );
+
 	gpuvis_trace_printf( "xwayland_surface_role_commit wlr_surface %p\n", wlr_surface );
 
-	wayland_commit( wlr_surface, &dmabuf_attribs );
+	wayland_commit( wlr_surface, buf );
 }
 
 static void xwayland_surface_role_precommit(struct wlr_surface *wlr_surface) {