Skip to content

Commit

Permalink
simplify netwotk content fetcher registry
Browse files Browse the repository at this point in the history 
kudos @wonder-sk
  • Loading branch information
@3nids
3nids committed May 9, 2018
1 parent 30b7fd1 commit 8e20996
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 130 deletions.
19 changes: 2 additions & 17 deletions python/core/qgsnetworkcontentfetcherregistry.sip.in
View file
Expand Up @@ -35,7 +35,7 @@ FetchedContent holds useful information about a network content being fetched
Failed Failed
}; };


explicit QgsFetchedContent( QTemporaryFile *file = 0, ContentStatus status = NotStarted ); explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = 0, ContentStatus status = NotStarted );
%Docstring %Docstring
Constructs a FetchedContent with pointer to the downloaded file and status of the download Constructs a FetchedContent with pointer to the downloaded file and status of the download
%End %End
Expand Down Expand Up @@ -76,21 +76,6 @@ Return the potential error of the download
void fetched(); void fetched();
%Docstring %Docstring
Emitted when the file is fetched and accessible Emitted when the file is fetched and accessible
%End

void downloadStarted( const bool redownload );
%Docstring
Emitted when the download actually starts
%End

void cancelTriggered();
%Docstring
Emitted when download is canceled.
%End

void taskCompleted();
%Docstring
Emitted when the download is finished (although file not accessible yet)
%End %End


}; };
Expand Down Expand Up @@ -125,7 +110,7 @@ Create the registry for temporary downloaded files


~QgsNetworkContentFetcherRegistry(); ~QgsNetworkContentFetcherRegistry();


const QgsFetchedContent *fetch( const QUrl &url, const FetchingMode fetchingMode = DownloadLater ); const QgsFetchedContent *fetch( const QString &url, const FetchingMode fetchingMode = DownloadLater );
%Docstring %Docstring
Initialize a download for the given URL Initialize a download for the given URL


Expand Down
168 changes: 82 additions & 86 deletions src/core/qgsnetworkcontentfetcherregistry.cpp
View file
Expand Up @@ -27,104 +27,30 @@ QgsNetworkContentFetcherRegistry::QgsNetworkContentFetcherRegistry()


QgsNetworkContentFetcherRegistry::~QgsNetworkContentFetcherRegistry() QgsNetworkContentFetcherRegistry::~QgsNetworkContentFetcherRegistry()
{ {
QMap<QUrl, QgsFetchedContent *>::const_iterator it = mFileRegistry.constBegin(); QMap<QString, QgsFetchedContent *>::const_iterator it = mFileRegistry.constBegin();
for ( ; it != mFileRegistry.constEnd(); ++it ) for ( ; it != mFileRegistry.constEnd(); ++it )
{ {
delete it.value(); delete it.value();
} }
mFileRegistry.clear(); mFileRegistry.clear();
} }


const QgsFetchedContent *QgsNetworkContentFetcherRegistry::fetch( const QUrl &url, const FetchingMode fetchingMode ) const QgsFetchedContent *QgsNetworkContentFetcherRegistry::fetch( const QString &url, const FetchingMode fetchingMode )
{ {
QMutexLocker locker( &mMutex );
if ( mFileRegistry.contains( url ) ) if ( mFileRegistry.contains( url ) )
{ {
return mFileRegistry.value( url ); return mFileRegistry.value( url );
} }


QgsFetchedContent *content = new QgsFetchedContent( nullptr, QgsFetchedContent::NotStarted ); QgsFetchedContent *content = new QgsFetchedContent( url, nullptr, QgsFetchedContent::NotStarted );

// start
QObject::connect( content, &QgsFetchedContent::downloadStarted, this, [ = ]( const bool redownload )
{
if ( redownload && content->status() == QgsFetchedContent::Downloading )
{
{
QMutexLocker locker( &mMutex );
if ( content->mFetchingTask )
disconnect( content->mFetchingTask, &QgsNetworkContentFetcherTask::fetched, content, &QgsFetchedContent::taskCompleted );
}
// no locker when calling cancel!
content->cancel();
}
QMutexLocker locker( &mMutex );
if ( redownload ||
content->status() == QgsFetchedContent::NotStarted ||
content->status() == QgsFetchedContent::Failed )
{
content->mFetchingTask = new QgsNetworkContentFetcherTask( url );
connect( content->mFetchingTask, &QgsNetworkContentFetcherTask::fetched, content, &QgsFetchedContent::taskCompleted );
QgsApplication::instance()->taskManager()->addTask( content->mFetchingTask );
content->mStatus = QgsFetchedContent::Downloading;
}
} );

// cancel
QObject::connect( content, &QgsFetchedContent::cancelTriggered, this, [ = ]()
{
QMutexLocker locker( &mMutex );
if ( content->mFetchingTask && content->mFetchingTask->canCancel() )
{
content->mFetchingTask->cancel();
}
if ( content->mFile )
{
content->mFile->deleteLater();
content->mFilePath = QString();
}
} );

// finished
connect( content, &QgsFetchedContent::taskCompleted, this, [ = ]()
{
QMutexLocker locker( &mMutex );
if ( !content->mFetchingTask || !content->mFetchingTask->reply() )
{
// if no reply, it has been canceled
content->mStatus = QgsFetchedContent::Failed;
content->mError = QNetworkReply::OperationCanceledError;
content->mFilePath = QString();
}
else
{
QNetworkReply *reply = content->mFetchingTask->reply();
if ( reply->error() == QNetworkReply::NoError )
{
QTemporaryFile *tf = new QTemporaryFile( QStringLiteral( "XXXXXX" ) );
content->mFile = tf;
tf->open();
content->mFile->write( reply->readAll() );
// Qt docs notes that on some system if fileName is not called before close, file might get deleted
content->mFilePath = tf->fileName();
tf->close();
content->mStatus = QgsFetchedContent::Finished;
}
else
{
content->mStatus = QgsFetchedContent::Failed;
content->mError = reply->error();
content->mFilePath = QString();
}
}
content->emitFetched();
} );


mFileRegistry.insert( url, content ); mFileRegistry.insert( url, content );


if ( fetchingMode == DownloadImmediately ) if ( fetchingMode == DownloadImmediately )
content->download(); content->download();



return content; return content;
} }


Expand All @@ -135,11 +61,10 @@ QFile *QgsNetworkContentFetcherRegistry::localFile( const QString &filePathOrUrl


if ( !QUrl::fromUserInput( filePathOrUrl ).isLocalFile() ) if ( !QUrl::fromUserInput( filePathOrUrl ).isLocalFile() )
{ {
QMutexLocker locker( &mMutex ); if ( mFileRegistry.contains( path ) )
if ( mFileRegistry.contains( QUrl( path ) ) )
{ {
const QgsFetchedContent *content = mFileRegistry.value( QUrl( path ) ); const QgsFetchedContent *content = mFileRegistry.value( path );
if ( content->status() == QgsFetchedContent::Finished && !content->file() ) if ( content && content->status() == QgsFetchedContent::Finished && content->file() )
{ {
file = content->file(); file = content->file();
} }
Expand All @@ -166,10 +91,9 @@ QString QgsNetworkContentFetcherRegistry::localPath( const QString &filePathOrUr


if ( !QUrl::fromUserInput( filePathOrUrl ).isLocalFile() ) if ( !QUrl::fromUserInput( filePathOrUrl ).isLocalFile() )
{ {
QMutexLocker locker( &mMutex ); if ( mFileRegistry.contains( path ) )
if ( mFileRegistry.contains( QUrl( path ) ) )
{ {
const QgsFetchedContent *content = mFileRegistry.value( QUrl( path ) ); const QgsFetchedContent *content = mFileRegistry.value( path );
if ( content->status() == QgsFetchedContent::Finished && !content->filePath().isEmpty() ) if ( content->status() == QgsFetchedContent::Finished && !content->filePath().isEmpty() )
{ {
path = content->filePath(); path = content->filePath();
Expand All @@ -190,3 +114,75 @@ QString QgsNetworkContentFetcherRegistry::localPath( const QString &filePathOrUr







void QgsFetchedContent::download( bool redownload )
{

if ( redownload && status() == QgsFetchedContent::Downloading )
{
{
if ( mFetchingTask )
disconnect( mFetchingTask, &QgsNetworkContentFetcherTask::taskCompleted, this, &QgsFetchedContent::taskCompleted );
}
cancel();
}
if ( redownload ||
status() == QgsFetchedContent::NotStarted ||
status() == QgsFetchedContent::Failed )
{
mFetchingTask = new QgsNetworkContentFetcherTask( mUrl );
// use taskCompleted which is main thread rather than fetched signal in worker thread
connect( mFetchingTask, &QgsNetworkContentFetcherTask::taskCompleted, this, &QgsFetchedContent::taskCompleted );
QgsApplication::instance()->taskManager()->addTask( mFetchingTask );
mStatus = QgsFetchedContent::Downloading;
}

}

void QgsFetchedContent::cancel()
{
if ( mFetchingTask && mFetchingTask->canCancel() )
{
mFetchingTask->cancel();
}
if ( mFile )
{
mFile->deleteLater();
mFilePath = QString();
}
}


void QgsFetchedContent::taskCompleted()
{
if ( !mFetchingTask || !mFetchingTask->reply() )
{
// if no reply, it has been canceled
mStatus = QgsFetchedContent::Failed;
mError = QNetworkReply::OperationCanceledError;
mFilePath = QString();
}
else
{
QNetworkReply *reply = mFetchingTask->reply();
if ( reply->error() == QNetworkReply::NoError )
{
QTemporaryFile *tf = new QTemporaryFile( QStringLiteral( "XXXXXX" ) );
mFile = tf;
tf->open();
mFile->write( reply->readAll() );
// Qt docs notes that on some system if fileName is not called before close, file might get deleted
mFilePath = tf->fileName();
tf->close();
mStatus = QgsFetchedContent::Finished;
}
else
{
mStatus = QgsFetchedContent::Failed;
mError = reply->error();
mFilePath = QString();
}
}

emit fetched();
}
28 changes: 8 additions & 20 deletions src/core/qgsnetworkcontentfetcherregistry.h
View file
Expand Up @@ -54,8 +54,8 @@ class CORE_EXPORT QgsFetchedContent : public QObject
}; };


//! Constructs a FetchedContent with pointer to the downloaded file and status of the download //! Constructs a FetchedContent with pointer to the downloaded file and status of the download
explicit QgsFetchedContent( QTemporaryFile *file = nullptr, ContentStatus status = NotStarted ) explicit QgsFetchedContent( const QString &url, QTemporaryFile *file = nullptr, ContentStatus status = NotStarted )
: QObject(), mFile( file ), mStatus( status ) {} : QObject(), mUrl( url ), mFile( file ), mStatus( status ) {}


~QgsFetchedContent() ~QgsFetchedContent()
{ {
Expand Down Expand Up @@ -84,36 +84,27 @@ class CORE_EXPORT QgsFetchedContent : public QObject
* \brief Start the download * \brief Start the download
* \param redownload if set to true, it will restart any achieved or pending download. * \param redownload if set to true, it will restart any achieved or pending download.
*/ */
void download( bool redownload = false ) {emit downloadStarted( redownload );} void download( bool redownload = false );


/** /**
* @brief Cancel the download operation * @brief Cancel the download operation
*/ */
void cancel() {emit cancelTriggered();} void cancel();


signals: signals:
//! Emitted when the file is fetched and accessible //! Emitted when the file is fetched and accessible
void fetched(); void fetched();


//! Emitted when the download actually starts private slots:
void downloadStarted( const bool redownload );

//! Emitted when download is canceled.
void cancelTriggered();

//! Emitted when the download is finished (although file not accessible yet)
void taskCompleted(); void taskCompleted();


private: private:
void emitFetched() {emit fetched();} QString mUrl;
QTemporaryFile *mFile = nullptr; QTemporaryFile *mFile = nullptr;
QString mFilePath; QString mFilePath;
QgsNetworkContentFetcherTask *mFetchingTask = nullptr; QgsNetworkContentFetcherTask *mFetchingTask = nullptr;
ContentStatus mStatus = NotStarted; ContentStatus mStatus = NotStarted;
QNetworkReply::NetworkError mError = QNetworkReply::NoError; QNetworkReply::NetworkError mError = QNetworkReply::NoError;

// allow modification of task and file from main class
friend class QgsNetworkContentFetcherRegistry;
}; };


/** /**
Expand Down Expand Up @@ -151,7 +142,7 @@ class CORE_EXPORT QgsNetworkContentFetcherRegistry : public QObject
* \param fetchingMode defines if the download will start immediately or shall be manually triggered * \param fetchingMode defines if the download will start immediately or shall be manually triggered
* \note If the download starts immediately, it will not redownload any already fetched or currently fetching file. * \note If the download starts immediately, it will not redownload any already fetched or currently fetching file.
*/ */
const QgsFetchedContent *fetch( const QUrl &url, const FetchingMode fetchingMode = DownloadLater ); const QgsFetchedContent *fetch( const QString &url, const FetchingMode fetchingMode = DownloadLater );


#ifndef SIP_RUN #ifndef SIP_RUN


Expand All @@ -169,10 +160,7 @@ class CORE_EXPORT QgsNetworkContentFetcherRegistry : public QObject
QString localPath( const QString &filePathOrUrl ); QString localPath( const QString &filePathOrUrl );


private: private:
QMap<QUrl, QgsFetchedContent *> mFileRegistry; QMap<QString, QgsFetchedContent *> mFileRegistry;

//! Mutex to prevent concurrent access to the class from multiple threads at once (may corrupt the entries otherwise).
mutable QMutex mMutex;


}; };


Expand Down
2 changes: 1 addition & 1 deletion src/gui/qgsattributeform.cpp
View file
Expand Up @@ -1121,7 +1121,7 @@ void QgsAttributeForm::init()
QgsDebugMsg( QString( "loading form: %1" ).arg( mLayer->editFormConfig().uiForm() ) ); QgsDebugMsg( QString( "loading form: %1" ).arg( mLayer->editFormConfig().uiForm() ) );
const QString path = mLayer->editFormConfig().uiForm(); const QString path = mLayer->editFormConfig().uiForm();
QFile *file = QgsApplication::instance()->networkContentFetcherRegistry()->localFile( path ); QFile *file = QgsApplication::instance()->networkContentFetcherRegistry()->localFile( path );
if ( file->isReadable() && file->open( QFile::ReadOnly ) ) if ( file && file->open( QFile::ReadOnly ) )
{ {
QUiLoader loader; QUiLoader loader;


Expand Down
11 changes: 5 additions & 6 deletions tests/src/python/test_qgsnetworkcontentfetcherregistry.py
View file
Expand Up @@ -22,7 +22,6 @@
from qgis.testing import unittest, start_app from qgis.testing import unittest, start_app
from qgis.core import QgsNetworkContentFetcherRegistry, QgsFetchedContent, QgsApplication from qgis.core import QgsNetworkContentFetcherRegistry, QgsFetchedContent, QgsApplication
from utilities import unitTestDataPath from utilities import unitTestDataPath
from qgis.PyQt.QtCore import QUrl
from qgis.PyQt.QtNetwork import QNetworkReply, QNetworkRequest from qgis.PyQt.QtNetwork import QNetworkReply, QNetworkRequest
import socketserver import socketserver
import threading import threading
Expand Down Expand Up @@ -55,7 +54,7 @@ def __init__(self, methodName):


def testFetchBadUrl(self): def testFetchBadUrl(self):
registry = QgsApplication.networkContentFetcherRegistry() registry = QgsApplication.networkContentFetcherRegistry()
content = registry.fetch(QUrl('http://x')) content = registry.fetch('http://x')
self.loaded = False self.loaded = False


def check_reply(): def check_reply():
Expand All @@ -72,7 +71,7 @@ def check_reply():
def testFetchGoodUrl(self): def testFetchGoodUrl(self):
url = 'http://localhost:' + str(self.port) + '/qgis_local_server/index.html' url = 'http://localhost:' + str(self.port) + '/qgis_local_server/index.html'
registry = QgsApplication.networkContentFetcherRegistry() registry = QgsApplication.networkContentFetcherRegistry()
content = registry.fetch(QUrl(url)) content = registry.fetch(url)
self.loaded = False self.loaded = False


def check_reply(): def check_reply():
Expand All @@ -89,7 +88,7 @@ def check_reply():
self.assertEqual(registry.localPath(url), content.filePath()) self.assertEqual(registry.localPath(url), content.filePath())


# create new content with same URL # create new content with same URL
contentV2 = registry.fetch(QUrl(url)) contentV2 = registry.fetch(url)
self.assertEqual(contentV2.status(), QgsFetchedContent.Finished) self.assertEqual(contentV2.status(), QgsFetchedContent.Finished)


def testFetchReloadUrl(self): def testFetchReloadUrl(self):
Expand All @@ -99,7 +98,7 @@ def writeSimpleFile(content):
self.file_content = content self.file_content = content


registry = QgsApplication.networkContentFetcherRegistry() registry = QgsApplication.networkContentFetcherRegistry()
content = registry.fetch(QUrl('http://localhost:' + str(self.port) + '/qgis_local_server/simple_content.txt')) content = registry.fetch('http://localhost:' + str(self.port) + '/qgis_local_server/simple_content.txt')
self.loaded = False self.loaded = False
writeSimpleFile('my initial content') writeSimpleFile('my initial content')


Expand Down Expand Up @@ -136,7 +135,7 @@ def testLocalPath(self):
self.assertEqual(registry.localPath('xxxx'), 'xxxx') self.assertEqual(registry.localPath('xxxx'), 'xxxx')


# an existent but unfinished download should return an empty path # an existent but unfinished download should return an empty path
content = registry.fetch(QUrl('xxxx')) content = registry.fetch('xxxx')
self.assertEqual(registry.localPath('xxxx'), '') self.assertEqual(registry.localPath('xxxx'), '')




Expand Down

0 comments on commit 8e20996

Please sign in to comment.