Add a simplified class for fetching HTTP network content

nyalldawson · nyalldawson · commit f93e536a00e2 · 2014-08-19T20:49:10.000+10:00
diff --git a/python/core/core.sip b/python/core/core.sip
@@ -65,6 +65,7 @@
 %Include qgsmessageoutput.sip
 %Include qgsmimedatautils.sip
 %Include qgsnetworkaccessmanager.sip
+%Include qgsnetworkcontentfetcher.sip
 %Include qgsofflineediting.sip
 %Include qgsogcutils.sip
 %Include qgsowsconnection.sip
diff --git a/python/core/qgsnetworkcontentfetcher.sip b/python/core/qgsnetworkcontentfetcher.sip
@@ -0,0 +1,44 @@
+/**
+  \class QgsNetworkContentFetcher
+  \ingroup core
+  \brief HTTP network content fetcher. A simple method for fetching remote HTTP content
+  and converting the content to standard formats. Url redirects are automatically
+  handled.
+  \since 2.5
+*/
+
+class QgsNetworkContentFetcher : QObject
+{
+%TypeHeaderCode
+#include <qgsnetworkcontentfetcher.h>
+%End
+
+  public:
+    QgsNetworkContentFetcher();
+
+    virtual ~QgsNetworkContentFetcher();
+
+    /**Fetches content from a remote URL and handles redirects. The finished()
+     * signal will be emitted when content has been fetched.
+     * @param url URL to fetch
+    */
+    void fetchContent( const QUrl url );
+    
+    /**Returns a reference to the network reply
+     * @returns QNetworkReply for fetched URL content
+    */
+    QNetworkReply* reply();
+     
+     
+    /**Returns the fetched content as a string
+     * @returns string containing network content
+    */
+    QString contentAsString() const;
+    
+  signals:
+
+    /**Emitted when content has loaded
+    */
+    void finished();
+
+};
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
@@ -115,6 +115,7 @@ SET(QGIS_CORE_SRCS
   qgsmessagelog.cpp
   qgsnetworkaccessmanager.cpp
   qgsnetworkreplyparser.cpp
+  qgsnetworkcontentfetcher.cpp
   qgsobjectcustomproperties.cpp
   qgsofflineediting.cpp
   qgsogcutils.cpp
@@ -353,6 +354,7 @@ SET(QGIS_CORE_MOC_HDRS
   qgsmessageoutput.h
   qgsmessagelog.h
   qgsnetworkreplyparser.h
+  qgsnetworkcontentfetcher.h
   qgsofflineediting.h
   qgscredentials.h
   qgspluginlayer.h
@@ -495,6 +497,7 @@ SET(QGIS_CORE_HDRS
   qgsmessageoutput.h
   qgsmimedatautils.h
   qgsnetworkreplyparser.h
+  qgsnetworkcontentfetcher.h
   qgsobjectcustomproperties.h
   qgsofflineediting.h
   qgsogcutils.h
diff --git a/src/core/composer/qgscomposerhtml.cpp b/src/core/composer/qgscomposerhtml.cpp
@@ -21,6 +21,7 @@
 #include "qgsmessagelog.h"
 #include "qgsexpression.h"
 #include "qgslogger.h"
+#include "qgsnetworkcontentfetcher.h"
 
 #include <QCoreApplication>
 #include <QPainter>
@@ -108,58 +109,6 @@ void QgsComposerHtml::setEvaluateExpressions( bool evaluateExpressions )
   loadHtml();
 }
 
-QString QgsComposerHtml::fetchHtml( QUrl url )
-{
-  QUrl nextUrlToFetch = url;
-  QNetworkReply* reply = 0;
-
-  //loop until fetched valid html
-  while ( 1 )
-  {
-    //set contents
-    QNetworkRequest request( nextUrlToFetch );
-    reply = QgsNetworkAccessManager::instance()->get( request );
-    connect( reply, SIGNAL( finished() ), this, SLOT( frameLoaded() ) );
-    //pause until HTML fetch
-    mLoaded = false;
-    while ( !mLoaded )
-    {
-      qApp->processEvents();
-    }
-
-    if ( reply->error() != QNetworkReply::NoError )
-    {
-      QgsMessageLog::logMessage( tr( "HTML fetch %1 failed with error %2" ).arg( reply->url().toString() ).arg( reply->errorString() ) );
-      reply->deleteLater();
-      return QString();
-    }
-
-    QVariant redirect = reply->attribute( QNetworkRequest::RedirectionTargetAttribute );
-    if ( redirect.isNull() )
-    {
-      //no error or redirect, got target
-      break;
-    }
-
-    //redirect, so fetch redirect target
-    nextUrlToFetch = redirect.toUrl();
-    reply->deleteLater();
-  }
-
-  QVariant status = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute );
-  if ( !status.isNull() && status.toInt() >= 400 )
-  {
-    QgsMessageLog::logMessage( tr( "HTML fetch %1 failed with error %2" ).arg( reply->url().toString() ).arg( status.toString() ) );
-    reply->deleteLater();
-    return QString();
-  }
-
-  QByteArray array = reply->readAll();
-  reply->deleteLater();
-  mFetchedHtml = QString( array );
-  return mFetchedHtml;
-}
-
 void QgsComposerHtml::loadHtml()
 {
   if ( !mWebPage )
@@ -195,6 +144,7 @@ void QgsComposerHtml::loadHtml()
       {
         loadedHtml = mFetchedHtml;
       }
+
       break;
     }
     case QgsComposerHtml::ManualHtml:
@@ -210,7 +160,7 @@ void QgsComposerHtml::loadHtml()
 
   mLoaded = false;
   //set html, using the specified url as base if in Url mode
-  mWebPage->mainFrame()->setHtml( loadedHtml, mContentMode == QgsComposerHtml::Url ? QUrl( mLastFetchedUrl ) : QUrl() );
+  mWebPage->mainFrame()->setHtml( loadedHtml, mContentMode == QgsComposerHtml::Url ? QUrl( mActualFetchedUrl ) : QUrl() );
 
   //set user stylesheet
   QWebSettings* settings = mWebPage->settings();
@@ -279,6 +229,24 @@ void QgsComposerHtml::renderCachedImage()
   painter.end();
 }
 
+QString QgsComposerHtml::fetchHtml( QUrl url )
+{
+  QgsNetworkContentFetcher fetcher;
+  //pause until HTML fetch
+  mLoaded = false;
+  fetcher.fetchContent( url );
+  connect( &fetcher, SIGNAL( finished() ), this, SLOT( frameLoaded() ) );
+
+  while ( !mLoaded )
+  {
+    qApp->processEvents();
+  }
+
+  mFetchedHtml = fetcher.contentAsString();
+  mActualFetchedUrl = fetcher.reply()->url().toString();
+  return mFetchedHtml;
+}
+
 QSizeF QgsComposerHtml::totalSize() const
 {
   return mSize;
diff --git a/src/core/composer/qgscomposerhtml.h b/src/core/composer/qgscomposerhtml.h
@@ -229,6 +229,7 @@ class CORE_EXPORT QgsComposerHtml: public QgsComposerMultiFrame
     QString mHtml;
     QString mFetchedHtml;
     QString mLastFetchedUrl;
+    QString mActualFetchedUrl; //may be different if page was redirected
     bool mLoaded;
     QSizeF mSize; //total size in mm
     double mHtmlUnitsToMM;
diff --git a/src/core/qgsnetworkcontentfetcher.cpp b/src/core/qgsnetworkcontentfetcher.cpp
@@ -0,0 +1,103 @@
+/***************************************************************************
+                       qgsnetworkcontentfetcher.cpp
+                             -------------------
+    begin                : July, 2014
+    copyright            : (C) 2014 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsnetworkcontentfetcher.h"
+#include "qgsnetworkaccessmanager.h"
+#include "qgsmessagelog.h"
+#include "qgsapplication.h"
+#include <QNetworkReply>
+
+QgsNetworkContentFetcher::QgsNetworkContentFetcher()
+    : mReply( 0 )
+    , mContentLoaded( false )
+{
+
+}
+
+QgsNetworkContentFetcher::~QgsNetworkContentFetcher()
+{
+  delete mReply;
+}
+
+void QgsNetworkContentFetcher::fetchContent( const QUrl url )
+{
+  QUrl nextUrlToFetch = url;
+  mContentLoaded = false;
+
+  //get contents
+  QNetworkRequest request( nextUrlToFetch );
+
+  mReply = QgsNetworkAccessManager::instance()->get( request );
+  connect( mReply, SIGNAL( finished() ), this, SLOT( contentLoaded() ) );
+}
+
+QNetworkReply *QgsNetworkContentFetcher::reply()
+{
+  if ( !mContentLoaded )
+  {
+    return 0;
+  }
+
+  return mReply;
+}
+
+QString QgsNetworkContentFetcher::contentAsString() const
+{
+  if ( !mContentLoaded || !mReply )
+  {
+    return QString();
+  }
+
+  QByteArray array = mReply->readAll();
+  return QString( array );
+}
+
+void QgsNetworkContentFetcher::contentLoaded( bool ok )
+{
+  Q_UNUSED( ok );
+
+  if ( mReply->error() != QNetworkReply::NoError )
+  {
+    QgsMessageLog::logMessage( tr( "HTTP fetch %1 failed with error %2" ).arg( mReply->url().toString() ).arg( mReply->errorString() ) );
+    mContentLoaded = true;
+    emit finished();
+    return;
+  }
+
+  QVariant redirect = mReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
+  if ( redirect.isNull() )
+  {
+    //no error or redirect, got target
+    QVariant status = mReply->attribute( QNetworkRequest::HttpStatusCodeAttribute );
+    if ( !status.isNull() && status.toInt() >= 400 )
+    {
+      QgsMessageLog::logMessage( tr( "HTTP fetch %1 failed with error %2" ).arg( mReply->url().toString() ).arg( status.toString() ) );
+    }
+    mContentLoaded = true;
+    emit finished();
+    return;
+  }
+
+  //redirect, so fetch redirect target
+  mReply->deleteLater();
+  fetchContent( redirect.toUrl() );
+}
+
+
+
+
diff --git a/src/core/qgsnetworkcontentfetcher.h b/src/core/qgsnetworkcontentfetcher.h
@@ -0,0 +1,82 @@
+/***************************************************************************
+                       qgsnetworkcontentfetcher.h
+                             -------------------
+    begin                : July, 2014
+    copyright            : (C) 2014 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+
+#ifndef QGSNETWORKCONTENTFETCHER_H
+#define QGSNETWORKCONTENTFETCHER_H
+
+#include <QNetworkReply>
+#include <QUrl>
+
+/**
+  \class QgsNetworkContentFetcher
+  \ingroup core
+  \brief HTTP network content fetcher. A simple method for fetching remote HTTP content
+  and converting the content to standard formats. Url redirects are automatically
+  handled.
+  \since 2.5
+*/
+
+class CORE_EXPORT QgsNetworkContentFetcher : public QObject
+{
+    Q_OBJECT
+
+  public:
+
+    QgsNetworkContentFetcher();
+
+    virtual ~QgsNetworkContentFetcher();
+
+    /**Fetches content from a remote URL and handles redirects. The finished()
+     * signal will be emitted when content has been fetched.
+     * @param url URL to fetch
+    */
+    void fetchContent( const QUrl url );
+
+    /**Returns a reference to the network reply
+     * @returns QNetworkReply for fetched URL content
+    */
+    QNetworkReply* reply();
+
+    /**Returns the fetched content as a string
+     * @returns string containing network content
+    */
+    QString contentAsString() const;
+
+  signals:
+
+    /**Emitted when content has loaded
+    */
+    void finished();
+
+  private:
+
+    QNetworkReply* mReply;
+
+    bool mContentLoaded;
+
+  private slots:
+
+    /**Called when fetchUrlContent has finished loading a url. If
+     * result is a redirect then the redirect is fetched automatically.
+    */
+    void contentLoaded( bool ok = true );
+
+};
+
+#endif
diff --git a/tests/src/core/CMakeLists.txt b/tests/src/core/CMakeLists.txt
@@ -128,3 +128,4 @@ ADD_QGIS_TEST(shapebursttest testqgsshapeburst.cpp )
 ADD_QGIS_TEST(invertedpolygontest testqgsinvertedpolygonrenderer.cpp )
 ADD_QGIS_TEST(colorschemeregistry testqgscolorschemeregistry.cpp)
 ADD_QGIS_TEST(colorscheme testqgscolorscheme.cpp)
+ADD_QGIS_TEST(networkcontentfetcher testqgsnetworkcontentfetcher.cpp )
diff --git a/tests/src/core/testqgsnetworkcontentfetcher.cpp b/tests/src/core/testqgsnetworkcontentfetcher.cpp
