Skip to content

Commit

Permalink
[GRASS] show import progress in browser properties widget
Browse files Browse the repository at this point in the history
  • Loading branch information
@blazek
blazek committed Sep 25, 2015
1 parent 2ef38ff commit a72cec0
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 41 deletions.
11 changes: 11 additions & 0 deletions src/providers/grass/qgis.v.in.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,18 +318,24 @@ int main( int argc, char **argv )
if ( isPolygon )
{
double snapTreshold = 0;
G_message( "Building partial topology" );
Vect_build_partial( map, GV_BUILD_BASE );

if ( snapTreshold > 0 )
{
Vect_snap_lines( map, GV_BOUNDARY, snapTreshold, NULL );
}
G_message( "Breaking polygons" );
Vect_break_polygons( map, GV_BOUNDARY, NULL );
G_message( "Removing duplicates" );
Vect_remove_duplicates( map, GV_BOUNDARY | GV_CENTROID, NULL );
while ( true )
{
G_message( "Breaking lines" );
Vect_break_lines( map, GV_BOUNDARY, NULL );
G_message( "Removing duplicates" );
Vect_remove_duplicates( map, GV_BOUNDARY, NULL );
G_message( "Cleaning small dangles at nodes" );
if ( Vect_clean_small_angles_at_nodes( map, GV_BOUNDARY, NULL ) == 0 )
{
break;
Expand Down Expand Up @@ -358,15 +364,19 @@ int main( int argc, char **argv )
}
}

G_message( "Merging lines" );
Vect_merge_lines( map, GV_BOUNDARY, NULL, NULL );
G_message( "Removing bridges" );
#if GRASS_VERSION_MAJOR < 7
Vect_remove_bridges( map, NULL );
#else
int linesRemoved, bridgesRemoved;
Vect_remove_bridges( map, NULL, &linesRemoved, &bridgesRemoved );
#endif
G_message( "Attaching islands" );
Vect_build_partial( map, GV_BUILD_ATTACH_ISLES );

G_message( "Creating centroids" );
QMap<QgsFeatureId, QgsFeature> centroids;
QgsSpatialIndex spatialIndex;
int nAreas = Vect_get_num_areas( map );
Expand All @@ -385,6 +395,7 @@ int main( int argc, char **argv )
centroids.insert( area, feature );
spatialIndex.insertFeature( feature );
}
G_message( "Attaching input polygons to cleaned areas" );
// read once more to assign centroids to polygons
while ( true )
{
Expand Down
124 changes: 84 additions & 40 deletions src/providers/grass/qgsgrassimport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ QgsGrassImport::QgsGrassImport( QgsGrassObject grassObject )
: QObject()
, mGrassObject( grassObject )
, mCanceled( false )
, mProcess( 0 )
, mFutureWatcher( 0 )
, mProgressMin( 0 )
, mProgressMax( 0 )
, mProgressValue( 0 )
{
// QMovie used by QgsAnimatedIcon is using QTimer which cannot be start from another thread
// (it works on Linux however) so we cannot start it connecting from QgsGrassImportItem and
Expand Down Expand Up @@ -121,6 +125,28 @@ void QgsGrassImport::cancel()
mCanceled = true;
}

void QgsGrassImport::emitProgressChanged()
{
emit progressChanged( mProgressHtml + mProgressTmpHtml, mProgressMin, mProgressMax, mProgressValue );

}

void QgsGrassImport::onReadyReadStandardError()
{
if ( mProcess )
{
// TODO: should be locked? Lock does not help anyway.
// TODO: parse better progress output
mProgressHtml += QString( mProcess->readAllStandardError() ).replace( "\n", "<br>" );
emitProgressChanged();
}
}

void QgsGrassImport::addProgressRow( QString html )
{
mProgressHtml += html + "<br>";
}

//------------------------------ QgsGrassRasterImport ------------------------------------
QgsGrassRasterImport::QgsGrassRasterImport( QgsRasterPipe* pipe, const QgsGrassObject& grassObject,
const QgsRectangle &extent, int xSize, int ySize )
Expand Down Expand Up @@ -170,6 +196,8 @@ bool QgsGrassRasterImport::import()
for ( int band = 1; band <= provider->bandCount(); band++ )
{
QgsDebugMsg( QString( "band = %1" ).arg( band ) );
addProgressRow( tr( "Writing band %1/%2" ).arg( band ).arg( provider->bandCount() ) );
emitProgressChanged();
int colorInterpretation = provider->colorInterpretation( band );
if ( colorInterpretation == QgsRaster::RedBand )
{
Expand Down Expand Up @@ -228,18 +256,17 @@ bool QgsGrassRasterImport::import()
}
arguments.append( "output=" + name ); // get list of all output names
QTemporaryFile gisrcFile;
QProcess* process = 0;
try
{
process = QgsGrass::startModule( mGrassObject.gisdbase(), mGrassObject.location(), mGrassObject.mapset(), module, arguments, gisrcFile );
mProcess = QgsGrass::startModule( mGrassObject.gisdbase(), mGrassObject.location(), mGrassObject.mapset(), module, arguments, gisrcFile );
}
catch ( QgsGrass::Exception &e )
{
setError( e.what() );
return false;
}

QDataStream outStream( process );
QDataStream outStream( mProcess );

outStream << mExtent << ( qint32 )mXSize << ( qint32 )mYSize;
outStream << ( qint32 )qgis_out_type;
Expand All @@ -264,11 +291,14 @@ bool QgsGrassRasterImport::import()
int iterCols = 0;
int iterRows = 0;
QgsRasterBlock* block = 0;
process->setReadChannel( QProcess::StandardOutput );
mProcess->setReadChannel( QProcess::StandardOutput );
mProgressMax = mYSize;
while ( iter.readNextRasterPart( band, iterCols, iterRows, &block, iterLeft, iterTop ) )
{
for ( int row = 0; row < iterRows; row++ )
{
mProgressValue = iterTop + row;
emitProgressChanged();
if ( !block->convert( qgis_out_type ) )
{
setError( tr( "Cannot convert block (%1) to data type %2" ).arg( block->toString() ).arg( qgis_out_type ) );
Expand Down Expand Up @@ -320,11 +350,11 @@ bool QgsGrassRasterImport::import()
outStream << byteArray;

// Without waitForBytesWritten() it does not finish ok on Windows (process timeout)
process->waitForBytesWritten( -1 );
mProcess->waitForBytesWritten( -1 );

#ifndef Q_OS_WIN
// wait until the row is written to allow quick cancel (don't send data to buffer)
process->waitForReadyRead();
mProcess->waitForReadyRead();
bool result;
outStream >> result;
#endif
Expand All @@ -339,34 +369,37 @@ bool QgsGrassRasterImport::import()

// TODO: send something back from module and read it here to close map correctly in module

process->closeWriteChannel();
mProcess->closeWriteChannel();
// TODO: best timeout?
process->waitForFinished( 30000 );
mProcess->waitForFinished( 30000 );

QString stdoutString = process->readAllStandardOutput().data();
QString stderrString = process->readAllStandardError().data();
QString stdoutString = mProcess->readAllStandardOutput().data();
QString stderrString = mProcess->readAllStandardError().data();

QString processResult = QString( "exitStatus=%1, exitCode=%2, error=%3, errorString=%4 stdout=%5, stderr=%6" )
.arg( process->exitStatus() ).arg( process->exitCode() )
.arg( process->error() ).arg( process->errorString() )
.arg( mProcess->exitStatus() ).arg( mProcess->exitCode() )
.arg( mProcess->error() ).arg( mProcess->errorString() )
.arg( stdoutString.replace( "\n", ", " ) ).arg( stderrString.replace( "\n", ", " ) );
QgsDebugMsg( "processResult: " + processResult );

if ( process->exitStatus() != QProcess::NormalExit )
if ( mProcess->exitStatus() != QProcess::NormalExit )
{
setError( process->errorString() );
delete process;
setError( mProcess->errorString() );
delete mProcess;
mProcess = 0;
return false;
}

if ( process->exitCode() != 0 )
if ( mProcess->exitCode() != 0 )
{
setError( stderrString );
delete process;
delete mProcess;
mProcess = 0;
return false;
}

delete process;
delete mProcess;
mProcess = 0;
}
QgsDebugMsg( QString( "redBand = %1 greenBand = %2 blueBand = %3" ).arg( redBand ).arg( greenBand ).arg( blueBand ) );
if ( redBand > 0 && greenBand > 0 && blueBand > 0 )
Expand Down Expand Up @@ -486,19 +519,20 @@ bool QgsGrassVectorImport::import()
QString name = mGrassObject.name();
arguments.append( "output=" + name );
QTemporaryFile gisrcFile;
QProcess* process = 0;
try
{
process = QgsGrass::startModule( mGrassObject.gisdbase(), mGrassObject.location(), mGrassObject.mapset(), module, arguments, gisrcFile );
mProcess = QgsGrass::startModule( mGrassObject.gisdbase(), mGrassObject.location(), mGrassObject.mapset(), module, arguments, gisrcFile );
}
catch ( QgsGrass::Exception &e )
{
setError( e.what() );
return false;
}
// TODO: connecting readyReadStandardError() is causing hangs or crashes
//connect(mProcess, SIGNAL(readyReadStandardError()), this, SLOT(onReadyReadStandardError()));

QDataStream outStream( process );
process->setReadChannel( QProcess::StandardOutput );
QDataStream outStream( mProcess );
mProcess->setReadChannel( QProcess::StandardOutput );

QGis::WkbType wkbType = mProvider->geometryType();
bool isPolygon = QGis::singleType( QGis::flatType( wkbType ) ) == QGis::WKBPolygon;
Expand All @@ -508,9 +542,11 @@ bool QgsGrassVectorImport::import()

QgsFeatureIterator iterator = mProvider->getFeatures();
QgsFeature feature;
mProgressMax = mProvider->featureCount();
for ( int i = 0; i < ( isPolygon ? 2 : 1 ); i++ ) // two cycles with polygons
{
if ( i > 0 )
addProgressRow( tr( "Writing features" ) );
if ( i > 0 ) // second run for polygons
{
//iterator.rewind(); // rewind does not work
iterator = mProvider->getFeatures();
Expand All @@ -519,6 +555,9 @@ bool QgsGrassVectorImport::import()
int count = 0;
while ( iterator.nextFeature( feature ) )
{
mProgressTmpHtml = tr( "Feature %1/%2" ).arg( count + 1 ).arg( mProgressMax );
mProgressValue = count + 1;
emitProgressChanged();
if ( !feature.isValid() )
{
continue;
Expand All @@ -536,11 +575,11 @@ bool QgsGrassVectorImport::import()
outStream << feature;

// Without waitForBytesWritten() it does not finish ok on Windows (data lost)
process->waitForBytesWritten( -1 );
mProcess->waitForBytesWritten( -1 );

#ifndef Q_OS_WIN
// wait until the feature is written to allow quick cancel (don't send data to buffer)
process->waitForReadyRead();
mProcess->waitForReadyRead();
bool result;
outStream >> result;
#endif
Expand All @@ -551,56 +590,61 @@ bool QgsGrassVectorImport::import()
QgsDebugMsg( QString( "%1 features written" ).arg( count ) );
}
}

feature = QgsFeature(); // indicate end by invalid feature
outStream << false; // not canceled
outStream << feature;

process->waitForBytesWritten( -1 );
mProcess->waitForBytesWritten( -1 );
QgsDebugMsg( "features sent" );
#ifndef Q_OS_WIN
process->waitForReadyRead();
mProcess->waitForReadyRead();
bool result;
outStream >> result;
#endif
}

iterator.close();

// Close write channel before waiting for response to avoid stdin buffer problem on Windows
process->closeWriteChannel();
mProcess->closeWriteChannel();

QgsDebugMsg( "waitForReadyRead" );
bool result;
process->waitForReadyRead();
mProcess->waitForReadyRead();
outStream >> result;
QgsDebugMsg( QString( "result = %1" ).arg( result ) );

QgsDebugMsg( "waitForFinished" );
process->waitForFinished( 30000 );
mProcess->waitForFinished( 30000 );

QString stdoutString = process->readAllStandardOutput().data();
QString stderrString = process->readAllStandardError().data();
QString stdoutString = mProcess->readAllStandardOutput().data();
QString stderrString = mProcess->readAllStandardError().data();

QString processResult = QString( "exitStatus=%1, exitCode=%2, error=%3, errorString=%4 stdout=%5, stderr=%6" )
.arg( process->exitStatus() ).arg( process->exitCode() )
.arg( process->error() ).arg( process->errorString() )
.arg( mProcess->exitStatus() ).arg( mProcess->exitCode() )
.arg( mProcess->error() ).arg( mProcess->errorString() )
.arg( stdoutString.replace( "\n", ", " ) ).arg( stderrString.replace( "\n", ", " ) );
QgsDebugMsg( "processResult: " + processResult );

if ( process->exitStatus() != QProcess::NormalExit )
if ( mProcess->exitStatus() != QProcess::NormalExit )
{
setError( process->errorString() );
delete process;
setError( mProcess->errorString() );
delete mProcess;
mProcess = 0;
return false;
}

if ( process->exitCode() != 0 )
if ( mProcess->exitCode() != 0 )
{
setError( stderrString );
delete process;
delete mProcess;
mProcess = 0;
return false;
}

delete process;
delete mProcess;
mProcess = 0;
return true;
}

Expand Down
15 changes: 14 additions & 1 deletion src/providers/grass/qgsgrassimport.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ class GRASS_LIB_EXPORT QgsGrassImport : public QObject
QString error();
virtual QStringList names() const;
bool isCanceled() const;
void emitProgressChanged();
public slots:
void onFinished();
// TODO: this is not completely kosher, because QgsGrassImport exist on the main thread
Expand All @@ -58,19 +59,31 @@ class GRASS_LIB_EXPORT QgsGrassImport : public QObject
// and thus recieves the signal.
// Most probably however, it will work correctly, even if read/write the bool wasn't atomic
void cancel();
void frameChanged() {};
void frameChanged() {}

void onReadyReadStandardError();

signals:
// sent when process finished
void finished( QgsGrassImport *import );

void progressChanged( QString html, int min, int max, int value );

protected:
static bool run( QgsGrassImport *imp );
void setError( QString error );
void addProgressRow( QString html );
QgsGrassObject mGrassObject;
QString mError;
bool mCanceled;
QProcess* mProcess;
QFutureWatcher<bool>* mFutureWatcher;
QString mProgressHtml;
// temporary part of progress, e.g. number of features written.
QString mProgressTmpHtml;
int mProgressMin;
int mProgressMax;
int mProgressValue;
};

class GRASS_LIB_EXPORT QgsGrassRasterImport : public QgsGrassImport
Expand Down
Loading

0 comments on commit a72cec0

Please sign in to comment.