Skip to content
Permalink
Browse files

[FEATURE] Download a file in Processing

  • Loading branch information
Gustry authored and nyalldawson committed Oct 23, 2017
1 parent 977ce42 commit 2f44de0e4f13a2e3d33a05f90ed9682cc3ad0da9
@@ -37,6 +37,7 @@ SET(QGIS_ANALYSIS_SRCS
processing/qgsalgorithmextractbyexpression.cpp
processing/qgsalgorithmextractbyextent.cpp
processing/qgsalgorithmextractbylocation.cpp
processing/qgsalgorithmfiledownloader.cpp
processing/qgsalgorithmfixgeometries.cpp
processing/qgsalgorithmjoinbyattribute.cpp
processing/qgsalgorithmjoinwithlines.cpp
@@ -66,7 +66,7 @@ void QgsExtractByAttributeAlgorithm::initAlgorithm( const QVariantMap & )

QString QgsExtractByAttributeAlgorithm::shortHelpString() const
{
return QObject::tr( " This algorithm creates a new vector layer that only contains matching features from an input layer. "
return QObject::tr( "This algorithm creates a new vector layer that only contains matching features from an input layer. "
"The criteria for adding features to the resulting layer is defined based on the values "
"of an attribute from the input layer." );
}
@@ -0,0 +1,132 @@
/***************************************************************************
qgsalgorithmfiledownloader.cpp
---------------------
begin : October 2017
copyright : (C) 2017 by Etienne Trimaille
email : etienne at kartoza dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgsalgorithmfiledownloader.h"
#include "qgsfiledownloader.h"
#include <QEventLoop>
#include <QFileInfo>
#include <QTimer>
#include <QUrl>

///@cond PRIVATE

QString QgsFileDownloaderAlgorithm::name() const
{
return QStringLiteral( "filedownloader" );
}

QString QgsFileDownloaderAlgorithm::displayName() const
{
return tr( "File downloader" );
}

QStringList QgsFileDownloaderAlgorithm::tags() const
{
return tr( "file,downloader,internet,url" ).split( ',' );
}

QString QgsFileDownloaderAlgorithm::group() const
{
return tr( "File tool" );
}

QString QgsFileDownloaderAlgorithm::shortHelpString() const
{
return tr( "This algorithm downloads a URL on the file system." );
}

QgsFileDownloaderAlgorithm *QgsFileDownloaderAlgorithm::createInstance() const
{
return new QgsFileDownloaderAlgorithm();
}

void QgsFileDownloaderAlgorithm::initAlgorithm( const QVariantMap & )
{
addParameter( new QgsProcessingParameterString( QStringLiteral( "URL" ), tr( "URL" ), QVariant(), false, false ) );
addParameter( new QgsProcessingParameterFileDestination( QStringLiteral( "OUTPUT" ),
tr( "File destination" ), QObject::tr( "*.*" ), QVariant(), true ) );
addOutput( new QgsProcessingOutputFile( QStringLiteral( "OUTPUT" ), tr( "File destination" ) ) );
}

QVariantMap QgsFileDownloaderAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
{
mFeedback = feedback;
QString url = parameterAsString( parameters, QStringLiteral( "URL" ), context );
QString outputFile = parameterAsFileOutput( parameters, QStringLiteral( "OUTPUT" ), context );

QEventLoop loop;
QTimer timer;
QgsFileDownloader *downloader = new QgsFileDownloader( QUrl( url ), outputFile, QString(), true );
connect( mFeedback, &QgsFeedback::canceled, downloader, &QgsFileDownloader::cancelDownload );
connect( downloader, &QgsFileDownloader::downloadError, this, &QgsFileDownloaderAlgorithm::reportErrors );
connect( downloader, &QgsFileDownloader::downloadProgress, this, &QgsFileDownloaderAlgorithm::receiveProgressFromDownloader );
connect( downloader, &QgsFileDownloader::downloadExited, &loop, &QEventLoop::quit );
connect( &timer, &QTimer::timeout, this, &QgsFileDownloaderAlgorithm::sendProgressFeedback );
downloader->startDownload();
timer.start( 1000 );

loop.exec();

timer.stop();
if ( ! QFileInfo( outputFile ).exists() )
throw QgsProcessingException( tr( "Output file doesn't exist." ) );

QVariantMap outputs;
outputs.insert( QStringLiteral( "OUTPUT" ), outputFile );
return outputs;
}

void QgsFileDownloaderAlgorithm::reportErrors( QStringList errors )
{
throw QgsProcessingException( errors.join( '\n' ) );
}

void QgsFileDownloaderAlgorithm::sendProgressFeedback()
{
if ( ! mReceived.isEmpty() && ! mTotal.isEmpty() )
mFeedback->pushInfo( tr( "%1 of %2 downloaded." ).arg( mReceived ).arg( mTotal ) );
}

void QgsFileDownloaderAlgorithm::receiveProgressFromDownloader( qint64 bytesReceived, qint64 bytesTotal )
{
if ( bytesTotal != 0 )
{
if ( mTotal.isEmpty() )
mTotal = humanSize( bytesTotal );

mReceived = humanSize( bytesReceived );
mFeedback->setProgress( ( bytesReceived * 100 ) / bytesTotal );
}
}

QString QgsFileDownloaderAlgorithm::humanSize( qint64 &bytes )
{
QStringList list;
list << "KB" << "MB" << "GB" << "TB";

QStringListIterator i( list );
QString unit( "bytes" );

while ( bytes >= 1024.0 && i.hasNext() )
{
unit = i.next();
bytes /= 1024.0;
}
return QString( "%1 %2" ).arg( QString::number( bytes ) ).arg( unit );
}

///@endcond
@@ -0,0 +1,62 @@
/***************************************************************************
qgsalgorithmfiledownloader.h
---------------------
begin : October 2017
copyright : (C) 2017 by Etienne Trimaille
email : etienne at kartoza dot com
***************************************************************************/

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/

#ifndef QGSALGORITHMFILEDOWNLOADER_H
#define QGSALGORITHMFILEDOWNLOADER_H

#define SIP_NO_FILE

#include "qgis.h"
#include "qgsprocessingalgorithm.h"

class QgsProcessingFeedback;

///@cond PRIVATE

/**
* Native file downloader algorithm.
*/
class QgsFileDownloaderAlgorithm : public QgsProcessingAlgorithm, public QObject
{
public:
QgsFileDownloaderAlgorithm() = default;
void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
QString name() const override;
QString displayName() const override;
virtual QStringList tags() const override;
QString group() const override;
QString shortHelpString() const override;
QgsFileDownloaderAlgorithm *createInstance() const override SIP_FACTORY;

protected:

virtual QVariantMap processAlgorithm( const QVariantMap &parameters,
QgsProcessingContext &context, QgsProcessingFeedback * ) override;

private:
QString mTotal;
QString mReceived;
QgsProcessingFeedback *mFeedback = nullptr;
void reportErrors( QStringList errors );
void receiveProgressFromDownloader( qint64 bytesReceived, qint64 bytesTotal );
static QString humanSize( qint64 &bytes );
void sendProgressFeedback();
};

///@endcond PRIVATE

#endif // QGSALGORITHMFILEDOWNLOADER_H
@@ -32,6 +32,7 @@
#include "qgsalgorithmextractbyexpression.h"
#include "qgsalgorithmextractbyextent.h"
#include "qgsalgorithmextractbylocation.h"
#include "qgsalgorithmfiledownloader.h"
#include "qgsalgorithmfixgeometries.h"
#include "qgsalgorithmjoinbyattribute.h"
#include "qgsalgorithmjoinwithlines.h"
@@ -92,6 +93,7 @@ void QgsNativeAlgorithms::loadAlgorithms()
addAlgorithm( new QgsCollectAlgorithm() );
addAlgorithm( new QgsExtractByAttributeAlgorithm() );
addAlgorithm( new QgsExtractByExpressionAlgorithm() );
addAlgorithm( new QgsFileDownloaderAlgorithm() );
addAlgorithm( new QgsMultipartToSinglepartAlgorithm() );
addAlgorithm( new QgsSubdivideAlgorithm() );
addAlgorithm( new QgsTransformAlgorithm() );

0 comments on commit 2f44de0

Please sign in to comment.
You can’t perform that action at this time.