QmlStreamer: Use asynchronous rendering and image streaming; reuse QuickRenderer from Tide

deflect/qt/CMakeLists.txt

@@ -10,13 +10,16 @@ set(DEFLECTQT_HEADERS
 )
 set(DEFLECTQT_PUBLIC_HEADERS
   QmlStreamer.h
+  QuickRenderer.h
   TouchInjector.h
 )
 set(DEFLECTQT_SOURCES
   EventReceiver.cpp
   QmlStreamer.cpp
   QmlStreamerImpl.cpp
+  QuickRenderer.cpp
   TouchInjector.cpp
+
 )
 set(DEFLECTQT_LINK_LIBRARIES
   PUBLIC Deflect Qt5::Quick PRIVATE Qt5::Qml)

deflect/qt/EventReceiver.cpp

@@ -83,7 +83,7 @@ void EventReceiver::_onEvent( int socket )
         {
         case Event::EVT_CLOSE:
             _notifier->setEnabled( false );
-            QCoreApplication::quit();
+            emit closed();
             break;
         case Event::EVT_PRESS:
             emit pressed( _pos( deflectEvent ));

deflect/qt/EventReceiver.h

@@ -68,6 +68,7 @@ class EventReceiver : public QObject
     void moved( QPointF position );
 
     void resized( QSize newSize );
+    void closed();
 
     void keyPress( int key, int modifiers, QString text );
     void keyRelease( int key, int modifiers, QString text );

deflect/qt/QmlStreamer.cpp

@@ -60,6 +60,11 @@ QmlStreamer::~QmlStreamer()
 {
 }
 
+void QmlStreamer::useAsyncSend( const bool async )
+{
+    _impl->useAsyncSend( async );
+}
+
 QQuickItem* QmlStreamer::getRootItem()
 {
     return _impl->getRootItem();

deflect/qt/QmlStreamer.h

@@ -93,6 +93,9 @@ class QmlStreamer : public QObject
 
     DEFLECTQT_API ~QmlStreamer();
 
+    /** Use asynchronous send of images via Deflect stream. Default off. */
+    DEFLECTQT_API void useAsyncSend( bool async );
+
     /** @return the QML root item, might be nullptr if not ready yet. */
     DEFLECTQT_API QQuickItem* getRootItem();
 

deflect/qt/QmlStreamerImpl.cpp

@@ -39,7 +39,9 @@
 /*********************************************************************/
 
 #include "QmlStreamerImpl.h"
+
 #include "EventReceiver.h"
+#include "QuickRenderer.h"
 #include "QmlGestures.h"
 #include "TouchInjector.h"
 
@@ -54,10 +56,7 @@
 #include <QQuickItem>
 #include <QQuickRenderControl>
 #include <QQuickWindow>
-
-#if DEFLECT_USE_QT5GUI
-#include <QtGui/private/qopenglcontext_p.h>
-#endif
+#include <QThread>
 
 namespace
 {
@@ -66,6 +65,13 @@ const QString GESTURES_CONTEXT_PROPERTY( "deflectgestures" );
 const QString WEBENGINEVIEW_OBJECT_NAME( "webengineview" );
 const int TOUCH_TAPANDHOLD_DIST_PX = 20;
 const int TOUCH_TAPANDHOLD_TIMEOUT_MS = 200;
+
+deflect::Stream::Future make_ready_future( const bool value )
+{
+    std::promise< bool > promise;
+    promise.set_value( value );
+    return promise.get_future();
+}
 }
 
 class RenderControl : public QQuickRenderControl
@@ -94,29 +100,20 @@ namespace qt
 QmlStreamer::Impl::Impl( const QString& qmlFile, const std::string& streamHost,
                          const std::string& streamId )
     : QWindow()
-    , _context( new QOpenGLContext )
-    , _offscreenSurface( new QOffscreenSurface )
     , _renderControl( new RenderControl( this ))
     // Create a QQuickWindow that is associated with out render control. Note
     // that this window never gets created or shown, meaning that it will never
     // get an underlying native (platform) window.
     , _quickWindow( new QQuickWindow( _renderControl ))
     , _qmlEngine( new QQmlEngine )
     , _qmlComponent( new QQmlComponent( _qmlEngine, QUrl( qmlFile )))
-    , _rootItem( nullptr )
-    , _fbo( nullptr )
-    , _renderTimer( 0 )
-    , _stopRenderingDelayTimer( 0 )
-    , _stream( nullptr )
-    , _eventHandler( nullptr )
     , _qmlGestures( new QmlGestures )
     , _touchInjector( new TouchInjector( *_quickWindow,
                                          std::bind( &Impl::_mapToScene, this,
                                                     std::placeholders::_1 )))
-    , _streaming( false )
     , _streamHost( streamHost )
     , _streamId( streamId )
-    , _mouseMode( false )
+    , _sendFuture( make_ready_future( true ))
 {
     _mouseModeTimer.setSingleShot( true );
     _mouseModeTimer.setInterval( TOUCH_TAPANDHOLD_TIMEOUT_MS );
@@ -127,46 +124,17 @@ QmlStreamer::Impl::Impl( const QString& qmlFile, const std::string& streamHost,
 
     setSurfaceType( QSurface::OpenGLSurface );
 
-    // Qt Quick may need a depth and stencil buffer
-    QSurfaceFormat format_;
-    format_.setDepthBufferSize( 16 );
-    format_.setStencilBufferSize( 8 );
-    setFormat( format_ );
-
-    _context->setFormat( format_ );
-    _context->create();
-
-    // Test if user has setup shared GL contexts (QtWebEngine::initialize).
-    // If so, setup global share context needed by the Qml WebEngineView.
-    if( QCoreApplication::testAttribute( Qt::AA_ShareOpenGLContexts ))
-#if DEFLECT_USE_QT5GUI
-        qt_gl_set_global_share_context( _context );
-#else
-        qWarning() << "DeflectQt was not compiled with WebEngineView support";
-#endif
-    connect( &_mouseModeTimer, &QTimer::timeout, [this]() {
+    connect( &_mouseModeTimer, &QTimer::timeout, [&] {
         if( _touchIsTapAndHold( ))
             _switchFromTouchToMouseMode();
     });
 
-    // Pass _context->format(), not format_. Format does not specify and color
-    // buffer sizes, while the context, that has just been created, reports a
-    // format that has these values filled in. Pass this to the offscreen
-    // surface to make sure it will be compatible with the context's
-    // configuration.
-    _offscreenSurface->setFormat( _context->format( ));
-    _offscreenSurface->create();
-
     if( !_qmlEngine->incubationController( ))
         _qmlEngine->setIncubationController( _quickWindow->incubationController( ));
 
     // Now hook up the signals. For simplicy we don't differentiate between
     // renderRequested (only render is needed, no sync) and sceneChanged (polish
     // and sync is needed too).
-    connect( _quickWindow, &QQuickWindow::sceneGraphInitialized,
-             this, &QmlStreamer::Impl::_createFbo );
-    connect( _quickWindow, &QQuickWindow::sceneGraphInvalidated,
-             this, &QmlStreamer::Impl::_destroyFbo );
     connect( _renderControl, &QQuickRenderControl::renderRequested,
              this, &QmlStreamer::Impl::_requestRender );
     connect( _renderControl, &QQuickRenderControl::sceneChanged,
@@ -182,10 +150,9 @@ QmlStreamer::Impl::Impl( const QString& qmlFile, const std::string& streamHost,
 
 QmlStreamer::Impl::~Impl()
 {
-    delete _eventHandler;
-    delete _stream;
-
-    _context->makeCurrent( _offscreenSurface );
+    _quickRenderer->stop();
+    _quickRendererThread.quit();
+    _quickRendererThread.wait();
 
     // delete first to free scenegraph resources for following destructions
     delete _renderControl;
@@ -194,98 +161,114 @@ QmlStreamer::Impl::~Impl()
     delete _qmlComponent;
     delete _quickWindow;
     delete _qmlEngine;
-    delete _fbo;
 
-    _context->doneCurrent();
-
-    delete _offscreenSurface;
-    delete _context;
+    delete _quickRenderer;
 }
 
-void QmlStreamer::Impl::_createFbo()
+void QmlStreamer::Impl::_requestRender()
 {
-    _fbo =
-        new QOpenGLFramebufferObject( _quickWindow->size() * devicePixelRatio(),
-                               QOpenGLFramebufferObject::CombinedDepthStencil );
-    _quickWindow->setRenderTarget( _fbo );
+    killTimer( _stopRenderingDelayTimer );
+    _stopRenderingDelayTimer = 0;
+
+    if( _renderTimer == 0 )
+        _renderTimer = startTimer( 5, Qt::PreciseTimer );
 }
 
-void QmlStreamer::Impl::_destroyFbo()
+void QmlStreamer::Impl::_initRenderer()
 {
-    delete _fbo;
-    _fbo = 0;
+    _updateSizes( QSize( _rootItem->width(), _rootItem->height( )));
+
+    _quickRenderer = new QuickRenderer( *_quickWindow, *_renderControl,
+                                        true );
+
+#if QT_VERSION >= 0x050500
+    // Call required to make QtGraphicalEffects work in the initial scene.
+    _renderControl->prepareThread( &_quickRendererThread );
+#else
+    qDebug() << "missing QQuickRenderControl::prepareThread() on Qt < 5.5. "
+                "Expect some qWarnings and failing QtGraphicalEffects.";
+#endif
+
+    _quickRenderer->moveToThread( &_quickRendererThread );
+
+    _quickRendererThread.setObjectName( "Render" );
+    _quickRendererThread.start();
+
+    _quickRenderer->init();
+
+    connect( _quickRenderer, &QuickRenderer::afterRender,
+             this, &QmlStreamer::Impl::_afterRender, Qt::DirectConnection );
+    connect( _quickRenderer, &QuickRenderer::afterStop,
+             this, &QmlStreamer::Impl::_afterStop, Qt::DirectConnection );
 }
 
 void QmlStreamer::Impl::_render()
 {
-    if( !_context->makeCurrent( _offscreenSurface ))
-        return;
+    // Initialize the render control and our OpenGL resources. Do this as late
+    // as possible to use the proper size reported by the rootItem.
+    if( !_quickRendererThread.isRunning( ))
+        _initRenderer();
 
-    // Initialize the render control and our OpenGL resources. Do this as
-    // late as possible to use the proper size reported by the rootItem.
-    if( !_fbo )
-    {
-        _updateSizes( QSize( _rootItem->width(), _rootItem->height( )));
+    _renderControl->polishItems();
+    _quickRenderer->render();
 
-        _renderControl->initialize( _context );
+    if( _stopRenderingDelayTimer == 0 )
+        _stopRenderingDelayTimer = startTimer( 5000 /*ms*/ );
+}
 
+void QmlStreamer::Impl::_afterRender()
+{
+    if( _stream )
+        _streaming = _sendFuture.get();
+    else
+    {
         if( !_setupDeflectStream( ))
         {
             qWarning() << "Could not setup Deflect stream";
-            emit streamClosed();
+            _streaming = false;
+            return;
         }
+        _streaming = true;
     }
 
     if( !_streaming )
         return;
 
-    // Polish, synchronize and render the next frame (into our fbo). In this
-    // example everything happens on the same thread and therefore all three
-    // steps are performed in succession from here. In a threaded setup the
-    // render() call would happen on a separate thread.
-    _renderControl->polishItems();
-    _renderControl->sync();
-    _renderControl->render();
-
-    _quickWindow->resetOpenGLState();
-    QOpenGLFramebufferObject::bindDefault();
-
-    _context->functions()->glFlush();
-
-    const QImage image = _fbo->toImage();
-    if( image.isNull( ))
+    _quickRenderer->context()->functions()->glFlush();
+    _image = _quickRenderer->fbo()->toImage();
+    if( _image.isNull( ))
     {
         qDebug() << "Empty image not streamed";
         return;
     }
 
-    ImageWrapper imageWrapper( image.constBits(), image.width(), image.height(),
-                               BGRA, 0, 0 );
+    ImageWrapper imageWrapper( _image.constBits(), _image.width(),
+                               _image.height(), BGRA );
     imageWrapper.compressionPolicy = COMPRESSION_ON;
-    imageWrapper.compressionQuality = 100;
-    _streaming = _stream->send( imageWrapper ) && _stream->finishFrame();
+    imageWrapper.compressionQuality = 80;
 
-    if( !_streaming )
-    {
-        killTimer( _renderTimer );
-        _renderTimer = 0;
-        killTimer( _stopRenderingDelayTimer );
-        _stopRenderingDelayTimer = 0;
-        emit streamClosed();
-        return;
-    }
+    if( _asyncSend )
+        _sendFuture = _stream->asyncSend( imageWrapper );
+    else
+        _sendFuture = make_ready_future( _stream->send( imageWrapper ) &&
+                                         _stream->finishFrame( ));
+}
 
-    if( _stopRenderingDelayTimer == 0 )
-        _stopRenderingDelayTimer = startTimer( 5000 /*ms*/ );
+void QmlStreamer::Impl::_afterStop()
+{
+    if( _sendFuture.valid() && _asyncSend )
+        _sendFuture.wait();
+    delete _eventHandler;
+    delete _stream;
 }
 
-void QmlStreamer::Impl::_requestRender()
+void QmlStreamer::Impl::_onStreamClosed()
 {
+    killTimer( _renderTimer );
+    _renderTimer = 0;
     killTimer( _stopRenderingDelayTimer );
     _stopRenderingDelayTimer = 0;
-
-    if( _renderTimer == 0 )
-        _renderTimer = startTimer( 5, Qt::PreciseTimer );
+    emit streamClosed();
 }
 
 void QmlStreamer::Impl::_onPressed( const QPointF pos )
@@ -436,8 +419,6 @@ bool QmlStreamer::Impl::_setupDeflectStream()
     if( !_stream->isConnected( ))
         return false;
 
-    _stream->setDisconnectedCallback( [this](){ emit streamClosed(); } );
-
     if( !_stream->registerForEvents( ))
         return false;
 
@@ -478,6 +459,9 @@ bool QmlStreamer::Impl::_setupDeflectStream()
     connect( _eventHandler, &EventReceiver::swipeRight,
              _qmlGestures, &QmlGestures::swipeRight );
 
+    connect( _eventHandler, &EventReceiver::closed,
+             this, &QmlStreamer::Impl::_onStreamClosed );
+
     return true;
 }
 
@@ -552,14 +536,6 @@ void QmlStreamer::Impl::_sendMouseEvent( const QEvent::Type eventType,
 void QmlStreamer::Impl::resizeEvent( QResizeEvent* e )
 {
     _updateSizes( e->size( ));
-
-    if( _fbo && _fbo->size() != e->size() * devicePixelRatio() &&
-        _context->makeCurrent( _offscreenSurface ))
-    {
-        _destroyFbo();
-        _createFbo();
-        _context->doneCurrent();
-    }
 }
 
 void QmlStreamer::Impl::mousePressEvent( QMouseEvent* e )