Skip to content

Commit

Permalink
Merge pull request #10233 from rouault/huge_block_count
Browse files Browse the repository at this point in the history 
Rasterband methods (histogram, statistics): make them compatible of a …
  • Loading branch information
@rouault
rouault committed Jun 18, 2024
2 parents bbb0e21 + 42adada commit 3c09cd1
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 42 deletions.
18 changes: 10 additions & 8 deletions gcore/gdal_misc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1180,8 +1180,10 @@ int CPL_STDCALL GDALGetRandomRasterSample(GDALRasterBandH hBand, int nSamples,
const int nBlocksPerColumn =
(poBand->GetYSize() + nBlockYSize - 1) / nBlockYSize;

const int nBlockPixels = nBlockXSize * nBlockYSize;
const int nBlockCount = nBlocksPerRow * nBlocksPerColumn;
const GIntBig nBlockPixels =
static_cast<GIntBig>(nBlockXSize) * nBlockYSize;
const GIntBig nBlockCount =
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;

if (nBlocksPerRow == 0 || nBlocksPerColumn == 0 || nBlockPixels == 0 ||
nBlockCount == 0)
Expand All @@ -1206,18 +1208,18 @@ int CPL_STDCALL GDALGetRandomRasterSample(GDALRasterBandH hBand, int nSamples,
int nBlockSampleRate = 1;

if ((nSamples / ((nBlockCount - 1) / nSampleRate + 1)) != 0)
nBlockSampleRate =
std::max(1, nBlockPixels /
(nSamples / ((nBlockCount - 1) / nSampleRate + 1)));
nBlockSampleRate = static_cast<int>(std::max<GIntBig>(
1,
nBlockPixels / (nSamples / ((nBlockCount - 1) / nSampleRate + 1))));

int nActualSamples = 0;

for (int iSampleBlock = 0; iSampleBlock < nBlockCount;
for (GIntBig iSampleBlock = 0; iSampleBlock < nBlockCount;
iSampleBlock += nSampleRate)
{

const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock = static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock = static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *const poBlock =
poBand->GetLockedBlockRef(iXBlock, iYBlock);
Expand Down
79 changes: 45 additions & 34 deletions gcore/gdalrasterband.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,10 @@ GDALRasterBand::~GDALRasterBand()
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn &&
nBand == 1 && poDS != nullptr)
{
CPLDebug("GDAL", "%d block reads on %d block band 1 of %s.",
nBlockReads, nBlocksPerRow * nBlocksPerColumn,
poDS->GetDescription());
CPLDebug(
"GDAL", "%d block reads on " CPL_FRMT_GIB " block band 1 of %s.",
nBlockReads, static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn,
poDS->GetDescription());
}

InvalidateMaskBand();
Expand Down Expand Up @@ -3752,21 +3753,22 @@ CPLErr GDALRasterBand::GetHistogram(double dfMin, double dfMax, int nBuckets,
/* Read the blocks, and add to histogram. */
/* --------------------------------------------------------------------
*/
for (int iSampleBlock = 0;
iSampleBlock < nBlocksPerRow * nBlocksPerColumn;
for (GIntBig iSampleBlock = 0;
iSampleBlock <
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;
iSampleBlock += nSampleRate)
{
if (!pfnProgress(
iSampleBlock /
static_cast<double>(iSampleBlock) /
(static_cast<double>(nBlocksPerRow) * nBlocksPerColumn),
"Compute Histogram", pProgressData))
{
CPLFree(pabyMaskData);
return CE_Failure;
}

const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock = static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock = static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *poBlock = GetLockedBlockRef(iXBlock, iYBlock);
if (poBlock == nullptr)
Expand Down Expand Up @@ -5951,12 +5953,15 @@ CPLErr GDALRasterBand::ComputeStatistics(int bApproxOK, double *pdfMin,
? static_cast<GUInt32>(dfNoDataValue + 1e-10)
: nMaxValueType + 1;

for (int iSampleBlock = 0;
iSampleBlock < nBlocksPerRow * nBlocksPerColumn;
for (GIntBig iSampleBlock = 0;
iSampleBlock <
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;
iSampleBlock += nSampleRate)
{
const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock =
static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock =
static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *const poBlock =
GetLockedBlockRef(iXBlock, iYBlock);
Expand Down Expand Up @@ -5989,9 +5994,9 @@ CPLErr GDALRasterBand::ComputeStatistics(int bApproxOK, double *pdfMin,

poBlock->DropLock();

if (!pfnProgress(iSampleBlock /
static_cast<double>(nBlocksPerRow *
nBlocksPerColumn),
if (!pfnProgress(static_cast<double>(iSampleBlock) /
static_cast<double>(nBlocksPerRow) *
nBlocksPerColumn,
"Compute Statistics", pProgressData))
{
ReportError(CE_Failure, CPLE_UserInterrupt,
Expand Down Expand Up @@ -6077,12 +6082,13 @@ CPLErr GDALRasterBand::ComputeStatistics(int bApproxOK, double *pdfMin,
}
}

for (int iSampleBlock = 0;
iSampleBlock < nBlocksPerRow * nBlocksPerColumn;
for (GIntBig iSampleBlock = 0;
iSampleBlock <
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;
iSampleBlock += nSampleRate)
{
const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock = static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock = static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *const poBlock =
GetLockedBlockRef(iXBlock, iYBlock);
Expand Down Expand Up @@ -6141,10 +6147,10 @@ CPLErr GDALRasterBand::ComputeStatistics(int bApproxOK, double *pdfMin,

poBlock->DropLock();

if (!pfnProgress(
iSampleBlock /
static_cast<double>(nBlocksPerRow * nBlocksPerColumn),
"Compute Statistics", pProgressData))
if (!pfnProgress(static_cast<double>(iSampleBlock) /
static_cast<double>(nBlocksPerRow) *
nBlocksPerColumn,
"Compute Statistics", pProgressData))
{
ReportError(CE_Failure, CPLE_UserInterrupt, "User terminated");
CPLFree(pabyMaskData);
Expand Down Expand Up @@ -6480,9 +6486,10 @@ static void ComputeMinMaxGeneric(const void *pData, GDALDataType eDataType,

static bool ComputeMinMaxGenericIterBlocks(
GDALRasterBand *poBand, GDALDataType eDataType, bool bSignedByte,
int nTotalBlocks, int nSampleRate, int nBlocksPerRow, bool bGotNoDataValue,
double dfNoDataValue, bool bGotFloatNoDataValue, float fNoDataValue,
GDALRasterBand *poMaskBand, double &dfMin, double &dfMax)
GIntBig nTotalBlocks, int nSampleRate, int nBlocksPerRow,
bool bGotNoDataValue, double dfNoDataValue, bool bGotFloatNoDataValue,
float fNoDataValue, GDALRasterBand *poMaskBand, double &dfMin,
double &dfMax)

{
GByte *pabyMaskData = nullptr;
Expand All @@ -6499,11 +6506,11 @@ static bool ComputeMinMaxGenericIterBlocks(
}
}

for (int iSampleBlock = 0; iSampleBlock < nTotalBlocks;
for (GIntBig iSampleBlock = 0; iSampleBlock < nTotalBlocks;
iSampleBlock += nSampleRate)
{
const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock = static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock = static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *poBlock = poBand->GetLockedBlockRef(iXBlock, iYBlock);
if (poBlock == nullptr)
Expand Down Expand Up @@ -6811,12 +6818,15 @@ CPLErr GDALRasterBand::ComputeRasterMinMax(int bApproxOK, double *adfMinMax)

if (bUseOptimizedPath)
{
for (int iSampleBlock = 0;
iSampleBlock < nBlocksPerRow * nBlocksPerColumn;
for (GIntBig iSampleBlock = 0;
iSampleBlock <
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;
iSampleBlock += nSampleRate)
{
const int iYBlock = iSampleBlock / nBlocksPerRow;
const int iXBlock = iSampleBlock - nBlocksPerRow * iYBlock;
const int iYBlock =
static_cast<int>(iSampleBlock / nBlocksPerRow);
const int iXBlock =
static_cast<int>(iSampleBlock % nBlocksPerRow);

GDALRasterBlock *poBlock = GetLockedBlockRef(iXBlock, iYBlock);
if (poBlock == nullptr)
Expand All @@ -6838,7 +6848,8 @@ CPLErr GDALRasterBand::ComputeRasterMinMax(int bApproxOK, double *adfMinMax)
}
else
{
const int nTotalBlocks = nBlocksPerRow * nBlocksPerColumn;
const GIntBig nTotalBlocks =
static_cast<GIntBig>(nBlocksPerRow) * nBlocksPerColumn;
if (!ComputeMinMaxGenericIterBlocks(
this, eDataType, bSignedByte, nTotalBlocks, nSampleRate,
nBlocksPerRow, CPL_TO_BOOL(bGotNoDataValue), dfNoDataValue,
Expand Down

0 comments on commit 3c09cd1

Please sign in to comment.