Skip to content

Commit

Permalink
VRTComplexSource: perf improvement: add specialization when only NODA…
Browse files Browse the repository at this point in the history 
…TA for Byte/UInt16/Int16 data types

Add a specialization when a VRTComplexSource has only a <NODATA> value
and no other processing (scaling, color table expansion, LUT), and that
the source data type is Byte, UInt16, Int16

On a 3-band 20791x22938 Byte VRT, "gdal_translate in.vrt out.tif" goes
from 9.8s to 4s
  • Loading branch information
@rouault
rouault committed Oct 14, 2023
1 parent 7c78481 commit cd67491
Show file tree
Hide file tree
Showing 4 changed files with 668 additions and 272 deletions.
155 changes: 155 additions & 0 deletions autotest/gcore/vrt_read.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2360,3 +2360,158 @@ def test_vrt_read_top_and_bottom_strips_average():
assert ds.GetRasterBand(1).ReadRaster(*args, **kwargs) == ref_data
assert ds.ReadRaster(*args, **kwargs) == ref_data
assert ref_data[0] == 128


###############################################################################


@pytest.mark.parametrize(
"input_datatype", [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_Int16]
)
@pytest.mark.parametrize("vrt_type", ["Byte", "UInt16", "Int16"])
@pytest.mark.parametrize("nodata", [0, 254])
@pytest.mark.parametrize(
"request_type", [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_Int16]
)
def test_vrt_read_complex_source_nodata(
tmp_vsimem, input_datatype, vrt_type, nodata, request_type
):

if input_datatype == gdal.GDT_Byte:
array_type = "B"
elif input_datatype == gdal.GDT_UInt16:
array_type = "H"
elif input_datatype == gdal.GDT_Int16:
array_type = "h"
else:
assert False
input_data = array.array(
array_type,
[
nodata,
1,
2,
3,
nodata, # EOL
4,
nodata,
5,
6,
7, # EOL
16,
17,
18,
19,
20, # EOL
8,
9,
nodata,
10,
11, # EOL
nodata,
nodata,
nodata,
nodata,
nodata, # EOL
12,
13,
14,
nodata,
15, # EOL
],
)
input_filename = str(tmp_vsimem / "source.tif")
ds = gdal.GetDriverByName("GTiff").Create(input_filename, 5, 6, 1, input_datatype)
ds.WriteRaster(0, 0, 5, 6, input_data, buf_type=input_datatype)
ds.Close()
complex_xml = f"""<VRTDataset rasterXSize="5" rasterYSize="6">
<VRTRasterBand dataType="{vrt_type}" band="1">
<NoDataValue>255</NoDataValue>
<ComplexSource>
<SourceFilename relativeToVRT="0">{input_filename}</SourceFilename>
<SourceBand>1</SourceBand>
<NODATA>{nodata}</NODATA>
</ComplexSource>
</VRTRasterBand>
</VRTDataset>
"""
vrt_ds = gdal.Open(complex_xml)
if request_type == gdal.GDT_Byte:
array_request_type = "B"
elif request_type == gdal.GDT_UInt16:
array_request_type = "H"
elif request_type == gdal.GDT_Int16:
array_request_type = "h"
else:
assert False
got_data = vrt_ds.ReadRaster(buf_type=request_type)
got_data = struct.unpack(array_request_type * (5 * 6), got_data)
assert got_data == (
255,
1,
2,
3,
255, # EOL
4,
255,
5,
6,
7, # EOL
16,
17,
18,
19,
20, # EOL
8,
9,
255,
10,
11, # EOL
255,
255,
255,
255,
255, # EOL
12,
13,
14,
255,
15, # EOL
)


###############################################################################


@pytest.mark.parametrize("data_type", [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_Int16])
def test_vrt_read_complex_source_nodata_out_of_range(tmp_vsimem, data_type):

if data_type == gdal.GDT_Byte:
array_type = "B"
elif data_type == gdal.GDT_UInt16:
array_type = "H"
elif data_type == gdal.GDT_Int16:
array_type = "h"
else:
assert False
input_data = array.array(array_type, [1])
input_filename = str(tmp_vsimem / "source.tif")
ds = gdal.GetDriverByName("GTiff").Create(input_filename, 1, 1, 1, data_type)
ds.WriteRaster(0, 0, 1, 1, input_data, buf_type=data_type)
ds.Close()
vrt_type = gdal.GetDataTypeName(data_type)
complex_xml = f"""<VRTDataset rasterXSize="1" rasterYSize="1">
<VRTRasterBand dataType="{vrt_type}" band="1">
<NoDataValue>255</NoDataValue>
<ComplexSource>
<SourceFilename relativeToVRT="0">{input_filename}</SourceFilename>
<SourceBand>1</SourceBand>
<NODATA>-100000000</NODATA>
</ComplexSource>
</VRTRasterBand>
</VRTDataset>
"""
vrt_ds = gdal.Open(complex_xml)
got_data = vrt_ds.ReadRaster(buf_type=data_type)
got_data = struct.unpack(array_type, got_data)
assert got_data == (1,)
84 changes: 58 additions & 26 deletions frmts/vrt/vrtdataset.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ class CPL_DLL VRTSource
public:
virtual ~VRTSource();

virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand Down Expand Up @@ -1063,7 +1063,7 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource
int *, int *, int *, int *, int *, int *,
bool &bErrorOut);

virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand Down Expand Up @@ -1096,7 +1096,7 @@ class CPL_DLL VRTSimpleSource CPL_NON_FINAL : public VRTSource
GDALRasterBand *GetRasterBand() const;
GDALRasterBand *GetMaskBandMainBand();
int IsSameExceptBandNumber(VRTSimpleSource *poOtherSource);
CPLErr DatasetRasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
CPLErr DatasetRasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType, int nBandCount,
int *panBandMap, GSpacing nPixelSpace,
Expand Down Expand Up @@ -1124,7 +1124,7 @@ class VRTAveragedSource final : public VRTSimpleSource

public:
VRTAveragedSource();
virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand All @@ -1151,30 +1151,53 @@ class VRTAveragedSource final : public VRTSimpleSource
/* VRTComplexSource */
/************************************************************************/

typedef enum
{
VRT_SCALING_NONE,
VRT_SCALING_LINEAR,
VRT_SCALING_EXPONENTIAL,
} VRTComplexSourceScaling;

class CPL_DLL VRTComplexSource CPL_NON_FINAL : public VRTSimpleSource
{
CPL_DISALLOW_COPY_ASSIGN(VRTComplexSource)

protected:
int m_bNoDataSet = false;
static constexpr int PROCESSING_FLAG_NODATA = 1 << 0;
static constexpr int PROCESSING_FLAG_USE_MASK_BAND =
1 << 1; // Mutually exclusive with NODATA
static constexpr int PROCESSING_FLAG_SCALING_LINEAR = 1 << 2;
static constexpr int PROCESSING_FLAG_SCALING_EXPONENTIAL =
1 << 3; // Mutually exclusive with SCALING_LINEAR
static constexpr int PROCESSING_FLAG_COLOR_TABLE_EXPANSION = 1 << 4;
static constexpr int PROCESSING_FLAG_LUT = 1 << 5;

int m_nProcessingFlags = 0;

// GByte whose initialization constructor does nothing
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Weffc++"
#endif
struct NoInitByte
{
GByte value;
// cppcheck-suppress uninitMemberVar
NoInitByte()
{
// do nothing
}
};
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif

std::vector<NoInitByte> m_abyWrkBuffer{};
std::vector<NoInitByte> m_abyWrkBufferMask{};

// adjusted value should be read with GetAdjustedNoDataValue()
double m_dfNoDataValue = VRT_NODATA_UNSET;
std::string
m_osNoDataValueOri{}; // string value read in XML deserialization

VRTComplexSourceScaling m_eScalingType = VRT_SCALING_NONE;
double m_dfScaleOff = 0; // For linear scaling.
double m_dfScaleRatio = 1; // For linear scaling.

// For non-linear scaling with a power function.
int m_bSrcMinMaxDefined = FALSE;
bool m_bSrcMinMaxDefined = false;
double m_dfSrcMin = 0;
double m_dfSrcMax = 0;
double m_dfDstMin = 0;
Expand All @@ -1183,29 +1206,35 @@ class CPL_DLL VRTComplexSource CPL_NON_FINAL : public VRTSimpleSource

int m_nColorTableComponent = 0;

bool m_bUseMaskBand = false;

double *m_padfLUTInputs = nullptr;
double *m_padfLUTOutputs = nullptr;
int m_nLUTItemCount = 0;
std::vector<double> m_adfLUTInputs{};
std::vector<double> m_adfLUTOutputs{};

double GetAdjustedNoDataValue() const;

template <class WorkingDT>
CPLErr
RasterIOInternal(GDALDataType eBandDataType, int nReqXOff, int nReqYOff,
RasterIOInternal(GDALRasterBand *poSourceBand,
GDALDataType eVRTBandDataType, int nReqXOff, int nReqYOff,
int nReqXSize, int nReqYSize, void *pData, int nOutXSize,
int nOutYSize, GDALDataType eBufType, GSpacing nPixelSpace,
GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg,
GDALDataType eWrkDataType);

template <class SourceDT, GDALDataType eSourceType>
CPLErr RasterIOProcessNoData(GDALRasterBand *poSourceBand,
GDALDataType eVRTBandDataType, int nReqXOff,
int nReqYOff, int nReqXSize, int nReqYSize,
void *pData, int nOutXSize, int nOutYSize,
GDALDataType eBufType, GSpacing nPixelSpace,
GSpacing nLineSpace,
GDALRasterIOExtraArg *psExtraArg);

public:
VRTComplexSource();
VRTComplexSource() = default;
VRTComplexSource(const VRTComplexSource *poSrcSource, double dfXDstRatio,
double dfYDstRatio);
virtual ~VRTComplexSource();

virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand Down Expand Up @@ -1235,7 +1264,10 @@ class CPL_DLL VRTComplexSource CPL_NON_FINAL : public VRTSimpleSource

void SetUseMaskBand(bool bUseMaskBand)
{
m_bUseMaskBand = bUseMaskBand;
if (bUseMaskBand)
m_nProcessingFlags |= PROCESSING_FLAG_USE_MASK_BAND;
else
m_nProcessingFlags &= ~PROCESSING_FLAG_USE_MASK_BAND;
}

void SetLinearScaling(double dfOffset, double dfScale);
Expand Down Expand Up @@ -1271,7 +1303,7 @@ class VRTFilteredSource CPL_NON_FINAL : public VRTComplexSource
virtual CPLErr FilterData(int nXSize, int nYSize, GDALDataType eType,
GByte *pabySrcData, GByte *pabyDstData) = 0;

virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand Down Expand Up @@ -1345,7 +1377,7 @@ class VRTFuncSource final : public VRTSource
}
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;

virtual CPLErr RasterIO(GDALDataType eBandDataType, int nXOff, int nYOff,
virtual CPLErr RasterIO(GDALDataType eVRTBandDataType, int nXOff, int nYOff,
int nXSize, int nYSize, void *pData, int nBufXSize,
int nBufYSize, GDALDataType eBufType,
GSpacing nPixelSpace, GSpacing nLineSpace,
Expand Down
7 changes: 4 additions & 3 deletions frmts/vrt/vrtfilters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ int VRTFilteredSource::IsTypeSupported(GDALDataType eTestType) const
/* RasterIO() */
/************************************************************************/

CPLErr VRTFilteredSource::RasterIO(GDALDataType eBandDataType, int nXOff,
CPLErr VRTFilteredSource::RasterIO(GDALDataType eVRTBandDataType, int nXOff,
int nYOff, int nXSize, int nYSize,
void *pData, int nBufXSize, int nBufYSize,
GDALDataType eBufType, GSpacing nPixelSpace,
Expand All @@ -139,7 +139,7 @@ CPLErr VRTFilteredSource::RasterIO(GDALDataType eBandDataType, int nXOff,
if (nBufXSize != nXSize || nBufYSize != nYSize)
{
return VRTComplexSource::RasterIO(
eBandDataType, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize,
eVRTBandDataType, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize,
nBufYSize, eBufType, nPixelSpace, nLineSpace, psExtraArg);
}

Expand Down Expand Up @@ -317,7 +317,8 @@ CPLErr VRTFilteredSource::RasterIO(GDALDataType eBandDataType, int nXOff,
const bool bIsComplex =
CPL_TO_BOOL(GDALDataTypeIsComplex(eOperDataType));
const CPLErr eErr = VRTComplexSource::RasterIOInternal<float>(
eBandDataType, nFileXOff, nFileYOff, nFileXSize, nFileYSize,
l_band, eVRTBandDataType, nFileXOff, nFileYOff, nFileXSize,
nFileYSize,
pabyWorkData + nLineOffset * nTopFill + nPixelOffset * nLeftFill,
nFileXSize, nFileYSize, eOperDataType, nPixelOffset, nLineOffset,
&sExtraArgs, bIsComplex ? GDT_CFloat32 : GDT_Float32);
Expand Down

0 comments on commit cd67491

Please sign in to comment.