Skip to content
Permalink
Browse files

Merge pull request #37457 from elpaso/bugfix-gh37449-raster-float64-q…

…uantile-histogram

Fix raster quantile with float
  • Loading branch information
elpaso committed Jun 29, 2020
2 parents b18b927 + 090e5da commit fb9df9f6debc80d6e89edf2c9593822822933419
Showing with 7 additions and 6 deletions.
  1. +7 −6 src/core/raster/qgsrasterinterface.cpp
@@ -306,7 +306,7 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
{
// Calc resolution from theSampleSize
double xRes, yRes;
xRes = yRes = std::sqrt( ( finalExtent.width() * finalExtent.height() ) / sampleSize );
xRes = yRes = std::sqrt( ( static_cast<double>( finalExtent.width( ) ) * finalExtent.height() ) / sampleSize );

// But limit by physical resolution
if ( capabilities() & Size )
@@ -336,7 +336,7 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
}
QgsDebugMsgLevel( QStringLiteral( "theHistogram.width = %1 histogram.height = %2" ).arg( histogram.width ).arg( histogram.height ), 4 );

int myBinCount = binCount;
qint64 myBinCount = binCount;
if ( myBinCount == 0 )
{
// TODO: this was OK when stats/histogram were calced in provider,
@@ -359,16 +359,17 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
mySrcDataType == Qgis::Int16 || mySrcDataType == Qgis::Int32 ||
mySrcDataType == Qgis::UInt16 || mySrcDataType == Qgis::UInt32 ) )
{
myBinCount = std::min( histogram.width * histogram.height, static_cast<int>( std::ceil( histogram.maximum - histogram.minimum + 1 ) ) );
myBinCount = std::min( static_cast<qint64>( histogram.width ) * histogram.height, static_cast<qint64>( std::ceil( histogram.maximum - histogram.minimum + 1 ) ) );
}
else
{
// This is for not integer types:
myBinCount = std::min( 2000, histogram.width * histogram.height );
// This is for not integer types
myBinCount = static_cast<qint64>( histogram.width ) * static_cast<qint64>( histogram.height );
}
}
}
histogram.binCount = myBinCount;
// Hard limit 10'000'000
histogram.binCount = static_cast<int>( std::min( 10000000LL, myBinCount ) );
QgsDebugMsgLevel( QStringLiteral( "theHistogram.binCount = %1" ).arg( histogram.binCount ), 4 );
}

0 comments on commit fb9df9f

Please sign in to comment.
You can’t perform that action at this time.