Merge pull request #3121 from rouault/pyramids

Various improvements and fixes related to raster pyramids

Author: rouault

src/app/qgsrasterlayerproperties.cpp

@@ -183,6 +183,16 @@ QgsRasterLayerProperties::QgsRasterLayerProperties( QgsMapLayer* lyr, QgsMapCanv
     {
       cboResamplingMethod->addItem( method.second, method.first );
     }
+
+    // keep it in sync with qgsrasterpyramidsoptionwidget.cpp
+    QString prefix = provider->name() + "/driverOptions/_pyramids/";
+    QSettings mySettings;
+    QString defaultMethod = mySettings.value( prefix + "resampling", "AVERAGE" ).toString();
+    int idx = cboResamplingMethod->findData( defaultMethod );
+    if ( idx >= 0 )
+      cboResamplingMethod->setCurrentIndex( idx );
+
+
     // build pyramid list
     QList< QgsRasterPyramid > myPyramidList = provider->buildPyramidList();
     QList< QgsRasterPyramid >::iterator myRasterPyramidIterator;
@@ -988,6 +998,13 @@ void QgsRasterLayerProperties::on_buttonBuildPyramids_clicked()
     //mark to be pyramided
     myPyramidList[myCounterInt].build = myItem->isSelected() || myPyramidList[myCounterInt].exists;
   }
+
+  // keep it in sync with qgsrasterpyramidsoptionwidget.cpp
+  QString prefix = provider->name() + "/driverOptions/_pyramids/";
+  QSettings mySettings;
+  QString resamplingMethod( cboResamplingMethod->itemData( cboResamplingMethod->currentIndex() ).toString() );
+  mySettings.setValue( prefix + "resampling", resamplingMethod );
+
   //
   // Ask raster layer to build the pyramids
   //
@@ -996,7 +1013,7 @@ void QgsRasterLayerProperties::on_buttonBuildPyramids_clicked()
   QApplication::setOverrideCursor( Qt::WaitCursor );
   QString res = provider->buildPyramids(
                   myPyramidList,
-                  cboResamplingMethod->itemData( cboResamplingMethod->currentIndex() ).toString(),
+                  resamplingMethod,
                   ( QgsRaster::RasterPyramidsFormat ) cbxPyramidsFormat->currentIndex() );
   QApplication::restoreOverrideCursor();
   mPyramidProgress->setValue( 0 );

src/gui/qgsrasterformatsaveoptionswidget.cpp

@@ -31,6 +31,9 @@
 
 QMap< QString, QStringList > QgsRasterFormatSaveOptionsWidget::mBuiltinProfiles;
 
+static const QString PYRAMID_JPEG_YCBCR_COMPRESSION( "JPEG_QUALITY_OVERVIEW=75 COMPRESS_OVERVIEW=JPEG PHOTOMETRIC_OVERVIEW=YCBCR INTERLEAVE_OVERVIEW=PIXEL" );
+static const QString PYRAMID_JPEG_COMPRESSION( "JPEG_QUALITY_OVERVIEW=75 COMPRESS_OVERVIEW=JPEG INTERLEAVE_OVERVIEW=PIXEL" );
+
 QgsRasterFormatSaveOptionsWidget::QgsRasterFormatSaveOptionsWidget( QWidget* parent, const QString& format,
     QgsRasterFormatSaveOptionsWidget::Type type, const QString& provider )
     : QWidget( parent )
@@ -81,7 +84,7 @@ QgsRasterFormatSaveOptionsWidget::QgsRasterFormatSaveOptionsWidget( QWidget* par
         << "COMPRESS_OVERVIEW=DEFLATE PREDICTOR_OVERVIEW=2 ZLEVEL=9" ); // how to set zlevel?
     mBuiltinProfiles[ "z__pyramids_gtiff_4jpeg" ] =
       ( QStringList() << "_pyramids" << tr( "JPEG compression" )
-        << "JPEG_QUALITY_OVERVIEW=75 COMPRESS_OVERVIEW=JPEG PHOTOMETRIC_OVERVIEW=YCBCR INTERLEAVE_OVERVIEW=PIXEL" );
+        << PYRAMID_JPEG_YCBCR_COMPRESSION );
   }
 
   connect( mProfileComboBox, SIGNAL( currentIndexChanged( const QString & ) ),
@@ -151,10 +154,15 @@ void QgsRasterFormatSaveOptionsWidget::setType( QgsRasterFormatSaveOptionsWidget
   }
 }
 
+QString QgsRasterFormatSaveOptionsWidget::pseudoFormat() const
+{
+  return mPyramids ? "_pyramids" : mFormat;
+}
+
 void QgsRasterFormatSaveOptionsWidget::updateProfiles()
 {
   // build profiles list = user + builtin(last)
-  QString format = mPyramids ? "_pyramids" : mFormat;
+  QString format = pseudoFormat();
   QStringList profileKeys = profiles();
   QMapIterator<QString, QStringList> it( mBuiltinProfiles );
   while ( it.hasNext() )
@@ -209,6 +217,14 @@ void QgsRasterFormatSaveOptionsWidget::updateOptions()
   QString myOptions = mOptionsMap.value( currentProfileKey() );
   QStringList myOptionsList = myOptions.trimmed().split( ' ', QString::SkipEmptyParts );
 
+  // If the default JPEG compression profile was selected, remove PHOTOMETRIC_OVERVIEW=YCBCR
+  // if the raster is not RGB. Otherwise this is bound to fail afterwards.
+  if ( mRasterLayer && mRasterLayer->bandCount() != 3 &&
+       myOptions == PYRAMID_JPEG_YCBCR_COMPRESSION )
+  {
+    myOptions = PYRAMID_JPEG_COMPRESSION;
+  }
+
   if ( mOptionsStackedWidget->currentIndex() == 0 )
   {
     mOptionsTable->setRowCount( 0 );
@@ -487,7 +503,7 @@ QString QgsRasterFormatSaveOptionsWidget::settingsKey( QString profileName ) con
     profileName = "/profile_" + profileName;
   else
     profileName = "/profile_default" + profileName;
-  return mProvider + "/driverOptions/" + mFormat.toLower() + profileName + "/create";
+  return mProvider + "/driverOptions/" + pseudoFormat().toLower() + profileName + "/create";
 }
 
 QString QgsRasterFormatSaveOptionsWidget::currentProfileKey() const
@@ -523,9 +539,9 @@ void QgsRasterFormatSaveOptionsWidget::setCreateOptions()
     myProfiles += i.key() + QLatin1String( " " );
     ++i;
   }
-  mySettings.setValue( mProvider + "/driverOptions/" + mFormat.toLower() + "/profiles",
+  mySettings.setValue( mProvider + "/driverOptions/" + pseudoFormat().toLower() + "/profiles",
                        myProfiles.trimmed() );
-  mySettings.setValue( mProvider + "/driverOptions/" + mFormat.toLower() + "/defaultProfile",
+  mySettings.setValue( mProvider + "/driverOptions/" + pseudoFormat().toLower() + "/defaultProfile",
                        currentProfileKey().trimmed() );
 }
 
@@ -543,7 +559,7 @@ void QgsRasterFormatSaveOptionsWidget::setCreateOptions( const QString& profileN
 QStringList QgsRasterFormatSaveOptionsWidget::profiles() const
 {
   QSettings mySettings;
-  return mySettings.value( mProvider + "/driverOptions/" + mFormat.toLower() + "/profiles", "" ).toString().trimmed().split( ' ', QString::SkipEmptyParts );
+  return mySettings.value( mProvider + "/driverOptions/" + pseudoFormat().toLower() + "/profiles", "" ).toString().trimmed().split( ' ', QString::SkipEmptyParts );
 }
 
 void QgsRasterFormatSaveOptionsWidget::swapOptionsUI( int newIndex )

src/gui/qgsrasterformatsaveoptionswidget.h

@@ -104,6 +104,7 @@ class GUI_EXPORT QgsRasterFormatSaveOptionsWidget: public QWidget,
     void setCreateOptions( const QString& profile, const QStringList& list );
     QStringList profiles() const;
     bool eventFilter( QObject *obj, QEvent *event ) override;
+    QString pseudoFormat() const;
 
 };
 

src/gui/qgsrasterlayersaveasdialog.cpp

@@ -21,6 +21,8 @@
 #include "qgsrasterformatsaveoptionswidget.h"
 #include "qgsgenericprojectionselector.h"
 
+#include "gdal.h"
+
 #include <QFileDialog>
 #include <QMessageBox>
 #include <QSettings>

src/gui/qgsrasterpyramidsoptionswidget.cpp

@@ -27,7 +27,6 @@
 #include <QMenu>
 #include <QCheckBox>
 
-
 QgsRasterPyramidsOptionsWidget::QgsRasterPyramidsOptionsWidget( QWidget* parent, const QString& provider )
     : QWidget( parent )
     , mProvider( provider )
@@ -52,14 +51,14 @@ void QgsRasterPyramidsOptionsWidget::updateUi()
   QString prefix = mProvider + "/driverOptions/_pyramids/";
   QString tmpStr;
 
-  // cbxPyramidsInternal->setChecked( mySettings.value( prefix + "internal", false ).toBool() );
-  tmpStr = mySettings.value( prefix + "format", "gtiff" ).toString();
+  // keep it in sync with qgsrasterlayerproperties.cpp
+  tmpStr = mySettings.value( prefix + "format", "external" ).toString();
   if ( tmpStr == "internal" )
-    cbxPyramidsFormat->setCurrentIndex( 1 );
+    cbxPyramidsFormat->setCurrentIndex( Format::INTERNAL );
   else if ( tmpStr == "external_erdas" )
-    cbxPyramidsFormat->setCurrentIndex( 2 );
+    cbxPyramidsFormat->setCurrentIndex( Format::ERDAS );
   else
-    cbxPyramidsFormat->setCurrentIndex( 0 );
+    cbxPyramidsFormat->setCurrentIndex( Format::GTIFF );
 
   // initialize resampling methods
   cboResamplingMethod->clear();
@@ -68,8 +67,9 @@ void QgsRasterPyramidsOptionsWidget::updateUi()
   {
     cboResamplingMethod->addItem( method.second, method.first );
   }
-  cboResamplingMethod->setCurrentIndex( cboResamplingMethod->findData(
-                                          mySettings.value( prefix + "resampling", "AVERAGE" ).toString() ) );
+  QString defaultMethod = mySettings.value( prefix + "resampling", "AVERAGE" ).toString();
+  int idx = cboResamplingMethod->findData( defaultMethod );
+  cboResamplingMethod->setCurrentIndex( idx );
 
   // validate string, only space-separated positive integers are allowed
   lePyramidsLevels->setEnabled( cbxPyramidsLevelsCustom->isChecked() );
@@ -126,9 +126,9 @@ void QgsRasterPyramidsOptionsWidget::apply()
   QString tmpStr;
 
   // mySettings.setValue( prefix + "internal", cbxPyramidsInternal->isChecked() );
-  if ( cbxPyramidsFormat->currentIndex() == 1 )
+  if ( cbxPyramidsFormat->currentIndex() == Format::INTERNAL )
     tmpStr = "internal";
-  else if ( cbxPyramidsFormat->currentIndex() == 2 )
+  else if ( cbxPyramidsFormat->currentIndex() == Format::ERDAS )
     tmpStr = "external_erdas";
   else
     tmpStr = "external";
@@ -165,8 +165,25 @@ void QgsRasterPyramidsOptionsWidget::on_cbxPyramidsLevelsCustom_toggled( bool to
 
 void QgsRasterPyramidsOptionsWidget::on_cbxPyramidsFormat_currentIndexChanged( int index )
 {
-  mSaveOptionsWidget->setEnabled( index != 2 );
-  mSaveOptionsWidget->setPyramidsFormat(( QgsRaster::RasterPyramidsFormat ) index );
+  mSaveOptionsWidget->setEnabled( index != Format::ERDAS );
+  QgsRaster::RasterPyramidsFormat format;
+  switch ( index )
+  {
+    case Format::GTIFF:
+      format = QgsRaster::PyramidsGTiff;
+      break;
+    case Format::INTERNAL:
+      format = QgsRaster::PyramidsInternal;
+      break;
+    case Format::ERDAS:
+      format = QgsRaster::PyramidsErdas;
+      break;
+    default:
+      QgsDebugMsg( "Should not happen !" );
+      format = QgsRaster::PyramidsGTiff;
+      break;
+  }
+  mSaveOptionsWidget->setPyramidsFormat( format );
 }
 
 void QgsRasterPyramidsOptionsWidget::setOverviewList()

src/gui/qgsrasterpyramidsoptionswidget.h

@@ -63,6 +63,15 @@ class GUI_EXPORT QgsRasterPyramidsOptionsWidget: public QWidget,
 
   private:
 
+    // Must be in the same order as in the .ui file
+    typedef enum
+    {
+      GTIFF = 0,
+      INTERNAL = 1,
+      ERDAS = 2
+    } Format;
+
+
     QString mProvider;
     QList< int > mOverviewList;
     QMap< int, QCheckBox* > mOverviewCheckBoxes;

src/providers/gdal/qgsgdalprovider.cpp

@@ -1598,13 +1598,20 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> & theRaste
     Q_FOREACH ( const QString& option, theConfigOptions )
     {
       QStringList opt = option.split( '=' );
-      QByteArray key = opt[0].toLocal8Bit();
-      QByteArray value = opt[1].toLocal8Bit();
-      // save previous value
-      myConfigOptionsOld[ opt[0] ] = QString( CPLGetConfigOption( key.data(), nullptr ) );
-      // set temp. value
-      CPLSetConfigOption( key.data(), value.data() );
-      QgsDebugMsg( QString( "set option %1=%2" ).arg( key.data(), value.data() ) );
+      if ( opt.size() == 2 )
+      {
+        QByteArray key = opt[0].toLocal8Bit();
+        QByteArray value = opt[1].toLocal8Bit();
+        // save previous value
+        myConfigOptionsOld[ opt[0] ] = QString( CPLGetConfigOption( key.data(), nullptr ) );
+        // set temp. value
+        CPLSetConfigOption( key.data(), value.data() );
+        QgsDebugMsg( QString( "set option %1=%2" ).arg( key.data(), value.data() ) );
+      }
+      else
+      {
+        QgsDebugMsg( QString( "invalid pyramid option: %1" ).arg( option ) );
+      }
     }
   }
 
@@ -1634,7 +1641,8 @@ QString QgsGdalProvider::buildPyramids( const QList<QgsRasterPyramid> & theRaste
     }
   }
   /* From : http://www.gdal.org/classGDALDataset.html#a2aa6f88b3bbc840a5696236af11dde15
-   * pszResampling : one of "NEAREST", "GAUSS", "CUBIC", "AVERAGE", "MODE", "AVERAGE_MAGPHASE" or "NONE" controlling the downsampling method applied.
+   * pszResampling : one of "NEAREST", "GAUSS", "CUBIC", "CUBICSPLINE" (GDAL >= 2.0),
+   * "LANCZOS" ( GDAL >= 2.0), "AVERAGE", "MODE" or "NONE" controlling the downsampling method applied.
    * nOverviews : number of overviews to build.
    * panOverviewList : the list of overview decimation factors to build.
    * nListBands : number of bands to build overviews for in panBandList. Build for all bands if this is 0.
@@ -2962,7 +2970,7 @@ QString QgsGdalProvider::validateCreationOptions( const QStringList& createOptio
   return message;
 }
 
-QString QgsGdalProvider::validatePyramidsCreationOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
+QString QgsGdalProvider::validatePyramidsConfigOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
     const QStringList & theConfigOptions, const QString & fileFormat )
 {
   // Erdas Imagine format does not support config options
@@ -2977,21 +2985,19 @@ QString QgsGdalProvider::validatePyramidsCreationOptions( QgsRaster::RasterPyram
   else if ( pyramidsFormat == QgsRaster::PyramidsInternal )
   {
     QStringList supportedFormats;
-    supportedFormats << "gtiff" << "georaster" << "hfa" << "jp2kak" << "mrsid" << "nitf";
+    supportedFormats << "gtiff" << "georaster" << "hfa" << "gpkg" << "rasterlite" << "nitf";
     if ( ! supportedFormats.contains( fileFormat.toLower() ) )
-      return QString( "Internal pyramids format only supported for gtiff/georaster/hfa/jp2kak/mrsid/nitf files (using %1)" ).arg( fileFormat );
-    // TODO - check arguments for georaster hfa jp2kak mrsid nitf
-    // for now, only test gtiff
-    else if ( fileFormat.toLower() != "gtiff" )
-      return QString();
+      return QString( "Internal pyramids format only supported for gtiff/georaster/gpkg/rasterlite/nitf files (using %1)" ).arg( fileFormat );
   }
-
-  // for gtiff external or internal pyramids, validate gtiff-specific values
-  // PHOTOMETRIC_OVERVIEW=YCBCR requires a source raster with only 3 bands (RGB)
-  if ( theConfigOptions.contains( "PHOTOMETRIC_OVERVIEW=YCBCR" ) )
+  else
   {
-    if ( GDALGetRasterCount( mGdalDataset ) != 3 )
-      return "PHOTOMETRIC_OVERVIEW=YCBCR requires a source raster with only 3 bands (RGB)";
+    // for gtiff external pyramids, validate gtiff-specific values
+    // PHOTOMETRIC_OVERVIEW=YCBCR requires a source raster with only 3 bands (RGB)
+    if ( theConfigOptions.contains( "PHOTOMETRIC_OVERVIEW=YCBCR" ) )
+    {
+      if ( GDALGetRasterCount( mGdalDataset ) != 3 )
+        return "PHOTOMETRIC_OVERVIEW=YCBCR requires a source raster with only 3 bands (RGB)";
+    }
   }
 
   return QString();
@@ -3022,6 +3028,10 @@ QGISEXTERN QList<QPair<QString, QString> > *pyramidResamplingMethods()
     methods.append( QPair<QString, QString>( "AVERAGE", QObject::tr( "Average" ) ) );
     methods.append( QPair<QString, QString>( "GAUSS", QObject::tr( "Gauss" ) ) );
     methods.append( QPair<QString, QString>( "CUBIC", QObject::tr( "Cubic" ) ) );
+#if GDAL_VERSION_MAJOR >= 2
+    methods.append( QPair<QString, QString>( "CUBICSPLINE", QObject::tr( "Cubic Spline" ) ) );
+    methods.append( QPair<QString, QString>( "LANCZOS", QObject::tr( "Lanczos" ) ) );
+#endif
     methods.append( QPair<QString, QString>( "MODE", QObject::tr( "Mode" ) ) );
     methods.append( QPair<QString, QString>( "NONE", QObject::tr( "None" ) ) );
   }

src/providers/gdal/qgsgdalprovider.h

@@ -244,8 +244,8 @@ class QgsGdalProvider : public QgsRasterDataProvider, QgsGdalProviderBase
     bool remove() override;
 
     QString validateCreationOptions( const QStringList& createOptions, const QString& format ) override;
-    QString validatePyramidsCreationOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
-        const QStringList & theConfigOptions, const QString & fileFormat );
+    QString validatePyramidsConfigOptions( QgsRaster::RasterPyramidsFormat pyramidsFormat,
+                                           const QStringList & theConfigOptions, const QString & fileFormat ) override;
 
   private:
     // update mode

tests/src/core/testqgsrasterlayer.cpp

@@ -24,6 +24,7 @@
 #include <QDesktopServices>
 
 #include "cpl_conv.h"
+#include "gdal.h"
 
 //qgis includes...
 #include <qgsrasterlayer.h>
@@ -512,6 +513,7 @@ void TestQgsRasterLayer::buildExternalOverviews()
   QString myResult =
     mypLayer->dataProvider()->buildPyramids( myPyramidList, "NEAREST", myFormatFlag );
   qDebug( "%s", myResult.toLocal8Bit().constData() );
+  QVERIFY( myResult.isEmpty() );
   //
   // Lets verify we have pyramids now...
   //
@@ -526,8 +528,42 @@ void TestQgsRasterLayer::buildExternalOverviews()
   // And that they were indeed in an external file...
   //
   QVERIFY( QFile::exists( myTempPath + "landsat.tif.ovr" ) );
+
+  //cleanup
+  delete mypLayer;
+
+  QFile::remove( myTempPath + "landsat.tif.ovr" );
+  mypLayer = new QgsRasterLayer( myRasterFileInfo.filePath(),
+                                 myRasterFileInfo.completeBaseName() );
+  myPyramidList = mypLayer->dataProvider()->buildPyramidList();
+  for ( int myCounterInt = 0; myCounterInt < myPyramidList.count(); myCounterInt++ )
+  {
+    //mark to be pyramided
+    myPyramidList[myCounterInt].build = true;
+  }
+
+  // Test with options
+  QStringList optionList;
+  optionList << "COMPRESS_OVERVIEW=DEFLATE";
+  optionList << "invalid";
+
+  myResult =
+    mypLayer->dataProvider()->buildPyramids( myPyramidList, "NEAREST", myFormatFlag, optionList );
+  qDebug( "%s", myResult.toLocal8Bit().constData() );
+  QVERIFY( myResult.isEmpty() );
+  QVERIFY( QFile::exists( myTempPath + "landsat.tif.ovr" ) );
+
   //cleanup
   delete mypLayer;
+
+  // Check that the overview is Deflate compressed
+  QString ovrFilename( myTempPath + "landsat.tif.ovr" );
+  GDALDatasetH hDS = GDALOpen( ovrFilename.toLocal8Bit().constData(), GA_ReadOnly );
+  QVERIFY( hDS );
+  const char* pszCompression = GDALGetMetadataItem( hDS, "COMPRESSION", "IMAGE_STRUCTURE" );
+  QVERIFY( pszCompression && EQUAL( pszCompression, "DEFLATE" ) );
+  GDALClose( hDS );
+
   mReport += "<h2>Check Overviews</h2>\n";
   mReport += "<p>Passed</p>";
 }