From fe9f10d609f122c635741c440898f34d76222b19 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 22 Apr 2024 20:10:51 +0300 Subject: [PATCH 01/11] TileDB: migrate away from deprecated APIs --- frmts/tiledb/tiledbdense.cpp | 33 ++++-- frmts/tiledb/tiledbmultidimarray.cpp | 10 +- frmts/tiledb/tiledbsparse.cpp | 155 ++++++++++++++++----------- 3 files changed, 121 insertions(+), 77 deletions(-) diff --git a/frmts/tiledb/tiledbdense.cpp b/frmts/tiledb/tiledbdense.cpp index b213e3ca2ab5..cb159630c509 100644 --- a/frmts/tiledb/tiledbdense.cpp +++ b/frmts/tiledb/tiledbdense.cpp @@ -374,12 +374,16 @@ CPLErr TileDBRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, if (poGDS->m_array->schema().domain().ndim() == 3) { - poQuery->set_subarray(oaSubarray); + tiledb::Subarray subarray(*poGDS->m_roCtx, *poGDS->m_roArray); + subarray.set_subarray(oaSubarray); + poQuery->set_subarray(subarray); } else { - poQuery->set_subarray(std::vector(oaSubarray.cbegin() + 2, + tiledb::Subarray subarray(*poGDS->m_roCtx, *poGDS->m_roArray); + subarray.set_subarray(std::vector(oaSubarray.cbegin() + 2, oaSubarray.cend())); + poQuery->set_subarray(subarray); } SetBuffer(poQuery.get(), eDataType, osAttrName, pData, nXSize * nYSize); @@ -576,7 +580,9 @@ CPLErr TileDBRasterDataset::IRasterIO( if (poQuery != nullptr) { - poQuery->set_subarray(oaSubarray); + tiledb::Subarray subarray(*m_roCtx, *m_array); + subarray.set_subarray(oaSubarray); + poQuery->set_subarray(subarray); for (int b = 0; b < nBandCount; b++) { @@ -781,7 +787,8 @@ CPLErr TileDBRasterDataset::TrySaveXML() if (nTimestamp) { auto oMeta = std::unique_ptr(new tiledb::Array( - *m_ctx, m_array->uri(), TILEDB_WRITE, nTimestamp)); + *m_ctx, m_array->uri(), TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, nTimestamp))); oMeta->put_metadata(GDAL_ATTRIBUTE_NAME, TILEDB_UINT8, static_cast(strlen(pszTree)), pszTree); oMeta->close(); @@ -1177,12 +1184,14 @@ GDALDataset *TileDBRasterDataset::Open(GDALOpenInfo *poOpenInfo) if (eMode == TILEDB_READ) { poDS->m_array.reset(new tiledb::Array( - *poDS->m_ctx, osArrayPath, TILEDB_READ, poDS->nTimestamp)); + *poDS->m_ctx, osArrayPath, TILEDB_READ, + tiledb::TemporalPolicy(tiledb::TimeTravel, poDS->nTimestamp))); } else { poDS->m_array.reset(new tiledb::Array( - *poDS->m_ctx, osArrayPath, TILEDB_WRITE, poDS->nTimestamp)); + *poDS->m_ctx, osArrayPath, TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, poDS->nTimestamp))); } } else @@ -2081,9 +2090,10 @@ CPLErr TileDBRasterDataset::CopySubDatasets(GDALDataset *poSrcDS, if (poDstDS->nTimestamp) { - poDstDS->m_array.reset( - new tiledb::Array(*poDstDS->m_ctx, poDstDS->GetDescription(), - TILEDB_WRITE, poDstDS->nTimestamp)); + poDstDS->m_array.reset(new tiledb::Array( + *poDstDS->m_ctx, poDstDS->GetDescription(), TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, + poDstDS->nTimestamp))); } else poDstDS->m_array.reset(new tiledb::Array( @@ -2196,8 +2206,9 @@ GDALDataset *TileDBRasterDataset::Create(const char *pszFilename, int nXSize, tiledb::Array::create(osArrayPath, *poDS->m_schema); if (poDS->nTimestamp) - poDS->m_array.reset(new tiledb::Array(*poDS->m_ctx, osArrayPath, - TILEDB_WRITE, poDS->nTimestamp)); + poDS->m_array.reset(new tiledb::Array( + *poDS->m_ctx, osArrayPath, TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, poDS->nTimestamp))); else poDS->m_array.reset( new tiledb::Array(*poDS->m_ctx, osArrayPath, TILEDB_WRITE)); diff --git a/frmts/tiledb/tiledbmultidimarray.cpp b/frmts/tiledb/tiledbmultidimarray.cpp index dc5c57db1e08..09649188d551 100644 --- a/frmts/tiledb/tiledbmultidimarray.cpp +++ b/frmts/tiledb/tiledbmultidimarray.cpp @@ -924,7 +924,10 @@ bool TileDBArray::IRead(const GUInt64 *arrayStartIdx, const size_t *count, { tiledb::Query query(m_poSharedResource->GetCtx(), *(m_poTileDBArray.get())); - query.set_subarray(anSubArray); + tiledb::Subarray subarray(m_poSharedResource->GetCtx(), + *(m_poTileDBArray.get())); + subarray.set_subarray(anSubArray); + query.set_subarray(subarray); query.set_data_buffer(m_osAttrName, pDstBuffer, nBufferSize); if (m_bStats) @@ -995,7 +998,10 @@ bool TileDBArray::IWrite(const GUInt64 *arrayStartIdx, const size_t *count, { tiledb::Query query(m_poSharedResource->GetCtx(), *(m_poTileDBArray.get())); - query.set_subarray(anSubArray); + tiledb::Subarray subarray(m_poSharedResource->GetCtx(), + *(m_poTileDBArray.get())); + subarray.set_subarray(anSubArray); + query.set_subarray(subarray); query.set_data_buffer(m_osAttrName, const_cast(pSrcBuffer), nBufferSize); diff --git a/frmts/tiledb/tiledbsparse.cpp b/frmts/tiledb/tiledbsparse.cpp index eeea3f433b08..1d062d69d818 100644 --- a/frmts/tiledb/tiledbsparse.cpp +++ b/frmts/tiledb/tiledbsparse.cpp @@ -717,8 +717,9 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx, } if (m_nTimestamp) - m_array.reset( - new tiledb::Array(*m_ctx, m_osFilename, TILEDB_READ, m_nTimestamp)); + m_array.reset(new tiledb::Array( + *m_ctx, m_osFilename, TILEDB_READ, + tiledb::TemporalPolicy(tiledb::TimeTravel, m_nTimestamp))); else m_array.reset(new tiledb::Array(*m_ctx, m_osFilename, TILEDB_READ)); @@ -1238,7 +1239,7 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) m_anFIDs->resize(m_nBatchSize); if (!m_osFIDColumn.empty()) { - m_query->set_buffer(m_osFIDColumn, *(m_anFIDs)); + m_query->set_data_buffer(m_osFIDColumn, *(m_anFIDs)); } if (!m_poFeatureDefn->GetGeomFieldDefn(0)->IsIgnored()) @@ -1254,16 +1255,19 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) const auto colType = m_schema->attribute(pszGeomColName).type(); if (colType == TILEDB_UINT8) { - m_query->set_buffer(pszGeomColName, *m_anGeometryOffsets, - *m_abyGeometries); + m_query->set_data_buffer(pszGeomColName, *m_abyGeometries); + m_query->set_offsets_buffer(pszGeomColName, + *m_anGeometryOffsets); } else if (colType == TILEDB_BLOB) { - m_query->set_buffer( - pszGeomColName, m_anGeometryOffsets->data(), - m_anGeometryOffsets->size(), + m_query->set_data_buffer( + pszGeomColName, reinterpret_cast(m_abyGeometries->data()), m_abyGeometries->size()); + m_query->set_offsets_buffer(pszGeomColName, + m_anGeometryOffsets->data(), + m_anGeometryOffsets->size()); } else { @@ -1273,15 +1277,15 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) else { m_adfXs->resize(m_nBatchSize); - m_query->set_buffer(m_osXDim, *m_adfXs); + m_query->set_data_buffer(m_osXDim, *m_adfXs); m_adfYs->resize(m_nBatchSize); - m_query->set_buffer(m_osYDim, *m_adfYs); + m_query->set_data_buffer(m_osYDim, *m_adfYs); if (!m_osZDim.empty()) { m_adfZs->resize(m_nBatchSize); - m_query->set_buffer(m_osZDim, *m_adfZs); + m_query->set_data_buffer(m_osZDim, *m_adfZs); } } } @@ -1313,9 +1317,9 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) std::get>(fieldValues)); v.resize(m_nBatchSize); #ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T - m_query->set_buffer(pszFieldName, v.data(), v.size()); + m_query->set_data_buffer(pszFieldName, v.data(), v.size()); #else - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); #endif } else @@ -1325,21 +1329,21 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } else if (m_aeFieldTypes[i] == TILEDB_INT32) { auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } else if (m_aeFieldTypes[i] == TILEDB_UINT8) { auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } else if (m_aeFieldTypes[i] == TILEDB_UINT16) { @@ -1347,7 +1351,7 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } else { @@ -1390,7 +1394,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else if (m_aeFieldTypes[i] == TILEDB_INT32) { @@ -1400,7 +1405,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else if (m_aeFieldTypes[i] == TILEDB_UINT8) { @@ -1410,7 +1416,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else if (m_aeFieldTypes[i] == TILEDB_UINT16) { @@ -1421,7 +1428,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else { @@ -1438,7 +1446,7 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); break; } @@ -1456,7 +1464,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); break; } @@ -1467,14 +1476,14 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } else { auto &v = *(std::get>>( fieldValues)); v.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, v); + m_query->set_data_buffer(pszFieldName, v); } break; } @@ -1495,7 +1504,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) v.resize(GetValueSize( pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else { @@ -1504,7 +1514,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) v.resize(GetValueSize( pszFieldName, m_anFieldValuesCapacity[i], nMulFactor)); m_anFieldValuesCapacity[i] = v.capacity(); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } break; } @@ -1520,7 +1531,8 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) : 8)); m_anFieldValuesCapacity[i] = v.capacity(); anOffsets.resize(m_nBatchSize); - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); break; } @@ -1538,13 +1550,16 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays) anOffsets.resize(m_nBatchSize); if (eType == TILEDB_UINT8) { - m_query->set_buffer(pszFieldName, anOffsets, v); + m_query->set_data_buffer(pszFieldName, v); + m_query->set_offsets_buffer(pszFieldName, anOffsets); } else if (eType == TILEDB_BLOB) { - m_query->set_buffer( - pszFieldName, anOffsets.data(), anOffsets.size(), - reinterpret_cast(v.data()), v.size()); + m_query->set_data_buffer( + pszFieldName, reinterpret_cast(v.data()), + v.size()); + m_query->set_offsets_buffer(pszFieldName, anOffsets.data(), + anOffsets.size()); } else { @@ -2193,8 +2208,9 @@ void OGRTileDBLayer::SwitchToReadingMode() try { if (m_nTimestamp) - m_array.reset(new tiledb::Array(*m_ctx, m_osFilename, - TILEDB_READ, m_nTimestamp)); + m_array.reset(new tiledb::Array( + *m_ctx, m_osFilename, TILEDB_READ, + tiledb::TemporalPolicy(tiledb::TimeTravel, m_nTimestamp))); else m_array.reset( new tiledb::Array(*m_ctx, m_osFilename, TILEDB_READ)); @@ -3747,8 +3763,9 @@ void OGRTileDBLayer::InitializeSchemaAndArray() #endif if (m_nTimestamp) - m_array.reset(new tiledb::Array(*m_ctx, m_osFilename, TILEDB_WRITE, - m_nTimestamp)); + m_array.reset(new tiledb::Array( + *m_ctx, m_osFilename, TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, m_nTimestamp))); else m_array.reset( new tiledb::Array(*m_ctx, m_osFilename, TILEDB_WRITE)); @@ -3864,8 +3881,9 @@ void OGRTileDBLayer::SwitchToWritingMode() try { if (m_nTimestamp) - m_array.reset(new tiledb::Array(*m_ctx, m_osFilename, - TILEDB_WRITE, m_nTimestamp)); + m_array.reset(new tiledb::Array( + *m_ctx, m_osFilename, TILEDB_WRITE, + tiledb::TemporalPolicy(tiledb::TimeTravel, m_nTimestamp))); else m_array.reset( new tiledb::Array(*m_ctx, m_osFilename, TILEDB_WRITE)); @@ -4275,11 +4293,11 @@ void OGRTileDBLayer::FlushArrays() tiledb::Query query(*m_ctx, *m_array); query.set_layout(TILEDB_UNORDERED); if (!m_osFIDColumn.empty()) - query.set_buffer(m_osFIDColumn, *m_anFIDs); - query.set_buffer(m_osXDim, *m_adfXs); - query.set_buffer(m_osYDim, *m_adfYs); + query.set_data_buffer(m_osFIDColumn, *m_anFIDs); + query.set_data_buffer(m_osXDim, *m_adfXs); + query.set_data_buffer(m_osYDim, *m_adfYs); if (!m_osZDim.empty()) - query.set_buffer(m_osZDim, *m_adfZs); + query.set_data_buffer(m_osZDim, *m_adfZs); const char *pszGeomColName = GetDatabaseGeomColName(); if (pszGeomColName) @@ -4287,16 +4305,18 @@ void OGRTileDBLayer::FlushArrays() m_anGeometryOffsets->pop_back(); if (m_schema->attribute(pszGeomColName).type() == TILEDB_UINT8) { - query.set_buffer(pszGeomColName, *m_anGeometryOffsets, - *m_abyGeometries); + query.set_data_buffer(pszGeomColName, *m_abyGeometries); + query.set_offsets_buffer(pszGeomColName, *m_anGeometryOffsets); } else if (m_schema->attribute(pszGeomColName).type() == TILEDB_BLOB) { - query.set_buffer( - pszGeomColName, m_anGeometryOffsets->data(), - m_anGeometryOffsets->size(), + query.set_data_buffer( + pszGeomColName, reinterpret_cast(m_abyGeometries->data()), m_abyGeometries->size()); + query.set_offsets_buffer(pszGeomColName, + m_anGeometryOffsets->data(), + m_anGeometryOffsets->size()); } else { @@ -4332,9 +4352,9 @@ void OGRTileDBLayer::FlushArrays() auto &v = *(std::get>( fieldValues)); #ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T - query.set_buffer(pszFieldName, v.data(), v.size()); + query.set_data_buffer(pszFieldName, v.data(), v.size()); #else - query.set_buffer(pszFieldName, v); + query.set_data_buffer(pszFieldName, v); #endif } else @@ -4372,7 +4392,7 @@ void OGRTileDBLayer::FlushArrays() case OFTInteger64: { - query.set_buffer( + query.set_data_buffer( pszFieldName, *std::get>>( fieldValues)); @@ -4382,10 +4402,11 @@ void OGRTileDBLayer::FlushArrays() case OFTInteger64List: { anOffsets.pop_back(); - query.set_buffer( - pszFieldName, anOffsets, + query.set_data_buffer( + pszFieldName, *std::get>>( fieldValues)); + query.set_offsets_buffer(pszFieldName, anOffsets); break; } @@ -4393,14 +4414,14 @@ void OGRTileDBLayer::FlushArrays() { if (poFieldDefn->GetSubType() == OFSTFloat32) { - query.set_buffer( + query.set_data_buffer( pszFieldName, *std::get>>( fieldValues)); } else { - query.set_buffer( + query.set_data_buffer( pszFieldName, *std::get>>( fieldValues)); @@ -4413,17 +4434,19 @@ void OGRTileDBLayer::FlushArrays() anOffsets.pop_back(); if (poFieldDefn->GetSubType() == OFSTFloat32) { - query.set_buffer( - pszFieldName, anOffsets, + query.set_data_buffer( + pszFieldName, *std::get>>( fieldValues)); + query.set_offsets_buffer(pszFieldName, anOffsets); } else { - query.set_buffer( - pszFieldName, anOffsets, + query.set_data_buffer( + pszFieldName, *std::get>>( fieldValues)); + query.set_offsets_buffer(pszFieldName, anOffsets); } break; } @@ -4431,9 +4454,10 @@ void OGRTileDBLayer::FlushArrays() case OFTString: { anOffsets.pop_back(); - query.set_buffer( - pszFieldName, anOffsets, + query.set_data_buffer( + pszFieldName, *std::get>(fieldValues)); + query.set_offsets_buffer(pszFieldName, anOffsets); break; } @@ -4444,13 +4468,16 @@ void OGRTileDBLayer::FlushArrays() fieldValues)); if (m_aeFieldTypes[i] == TILEDB_UINT8) { - query.set_buffer(pszFieldName, anOffsets, v); + query.set_data_buffer(pszFieldName, v); + query.set_offsets_buffer(pszFieldName, anOffsets); } else if (m_aeFieldTypes[i] == TILEDB_BLOB) { - query.set_buffer( - pszFieldName, anOffsets.data(), anOffsets.size(), + query.set_data_buffer( + pszFieldName, reinterpret_cast(v.data()), v.size()); + query.set_offsets_buffer(pszFieldName, anOffsets.data(), + anOffsets.size()); } else { @@ -4463,7 +4490,7 @@ void OGRTileDBLayer::FlushArrays() case OFTDateTime: case OFTTime: { - query.set_buffer( + query.set_data_buffer( pszFieldName, *std::get>>( fieldValues)); From dad01e672a8956bc95c33d72762d4a63183ad6ec Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Apr 2024 21:54:09 +0200 Subject: [PATCH 02/11] TileDB: fix crash in previous commit --- frmts/tiledb/tiledbdense.cpp | 88 +++++++++++++++--------------------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/frmts/tiledb/tiledbdense.cpp b/frmts/tiledb/tiledbdense.cpp index cb159630c509..3090bc670e77 100644 --- a/frmts/tiledb/tiledbdense.cpp +++ b/frmts/tiledb/tiledbdense.cpp @@ -351,7 +351,6 @@ CPLErr TileDBRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, nBufferDTSize > 0 && (nPixelSpace % nBufferDTSize) == 0 && (nLineSpace % nBufferDTSize) == 0) { - std::unique_ptr poQuery; const uint64_t nBandIdx = poGDS->nBandStart + nBand - 1; std::vector oaSubarray = { nBandIdx, nBandIdx, @@ -361,30 +360,25 @@ CPLErr TileDBRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, std::rotate(oaSubarray.begin(), oaSubarray.begin() + 2, oaSubarray.end()); - if ((eRWFlag == GF_Read) && - ((eAccess == GA_Update) && (poGDS->m_roArray))) - { - poQuery.reset( - new tiledb::Query(*poGDS->m_roCtx, *poGDS->m_roArray)); - } - else - { - poQuery.reset(new tiledb::Query(*poGDS->m_ctx, *poGDS->m_array)); - } + const bool bUseReadOnlyObjs = + ((eRWFlag == GF_Read) && (eAccess == GA_Update) && + (poGDS->m_roArray)); + const auto &oCtxt = bUseReadOnlyObjs ? *poGDS->m_roCtx : *poGDS->m_ctx; + const auto &oArray = + bUseReadOnlyObjs ? *poGDS->m_roArray : *poGDS->m_array; + auto poQuery = std::make_unique(oCtxt, oArray); + tiledb::Subarray subarray(oCtxt, oArray); if (poGDS->m_array->schema().domain().ndim() == 3) { - tiledb::Subarray subarray(*poGDS->m_roCtx, *poGDS->m_roArray); subarray.set_subarray(oaSubarray); - poQuery->set_subarray(subarray); } else { - tiledb::Subarray subarray(*poGDS->m_roCtx, *poGDS->m_roArray); subarray.set_subarray(std::vector(oaSubarray.cbegin() + 2, oaSubarray.cend())); - poQuery->set_subarray(subarray); } + poQuery->set_subarray(subarray); SetBuffer(poQuery.get(), eDataType, osAttrName, pData, nXSize * nYSize); @@ -564,52 +558,44 @@ CPLErr TileDBRasterDataset::IRasterIO( nBufferDTSize > 0 && (nPixelSpace % nBufferDTSize) == 0 && (nLineSpace % nBufferDTSize) == 0) { - std::unique_ptr poQuery; std::vector oaSubarray = { (uint64_t)nYOff, (uint64_t)nYOff + nYSize - 1, (uint64_t)nXOff, (uint64_t)nXOff + nXSize - 1}; - if ((eRWFlag == GF_Read) && (eAccess == GA_Update && m_roArray)) - { - poQuery.reset(new tiledb::Query(*m_roCtx, *m_roArray)); - } - else - { - poQuery.reset(new tiledb::Query(*m_ctx, *m_array)); - } - - if (poQuery != nullptr) - { - tiledb::Subarray subarray(*m_roCtx, *m_array); - subarray.set_subarray(oaSubarray); - poQuery->set_subarray(subarray); + const bool bUseReadOnlyObjs = + ((eRWFlag == GF_Read) && (eAccess == GA_Update) && (m_roArray)); + const auto &oCtxt = bUseReadOnlyObjs ? *m_roCtx : *m_ctx; + const auto &oArray = bUseReadOnlyObjs ? *m_roArray : *m_array; - for (int b = 0; b < nBandCount; b++) - { - TileDBRasterBand *poBand = - (TileDBRasterBand *)GetRasterBand(panBandMap[b]); - int nRegionSize = nBufXSize * nBufYSize * nBufferDTSize; - SetBuffer(poQuery.get(), eDataType, poBand->osAttrName, - ((GByte *)pData) + b * nRegionSize, nRegionSize); - } + auto poQuery = std::make_unique(oCtxt, oArray); + tiledb::Subarray subarray(oCtxt, oArray); + subarray.set_subarray(oaSubarray); + poQuery->set_subarray(subarray); - if (bStats) - tiledb::Stats::enable(); + for (int b = 0; b < nBandCount; b++) + { + TileDBRasterBand *poBand = + (TileDBRasterBand *)GetRasterBand(panBandMap[b]); + int nRegionSize = nBufXSize * nBufYSize * nBufferDTSize; + SetBuffer(poQuery.get(), eDataType, poBand->osAttrName, + ((GByte *)pData) + b * nRegionSize, nRegionSize); + } - auto status = poQuery->submit(); + if (bStats) + tiledb::Stats::enable(); - if (bStats) - { - tiledb::Stats::dump(stdout); - tiledb::Stats::disable(); - } + auto status = poQuery->submit(); - if (status == tiledb::Query::Status::FAILED) - return CE_Failure; - else - return CE_None; + if (bStats) + { + tiledb::Stats::dump(stdout); + tiledb::Stats::disable(); } - return CE_Failure; + + if (status == tiledb::Query::Status::FAILED) + return CE_Failure; + else + return CE_None; } return GDALPamDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, From 130560b919ef774d204f702fde533fe896eb4e0f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Apr 2024 21:54:46 +0200 Subject: [PATCH 03/11] TileDB: remove TILEDB_DEPRECATED suppression --- frmts/tiledb/CMakeLists.txt | 1 - frmts/tiledb/include_tiledb.h | 9 --------- frmts/tiledb/tiledbdense.cpp | 10 ---------- 3 files changed, 20 deletions(-) diff --git a/frmts/tiledb/CMakeLists.txt b/frmts/tiledb/CMakeLists.txt index 2b68b39f7ef7..86fc626591c6 100644 --- a/frmts/tiledb/CMakeLists.txt +++ b/frmts/tiledb/CMakeLists.txt @@ -25,5 +25,4 @@ endif() gdal_standard_includes(gdal_TileDB) target_include_directories(gdal_TileDB PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/mem) -target_compile_definitions(gdal_TileDB PRIVATE -DTILEDB_DEPRECATED=) gdal_target_link_libraries(gdal_TileDB PRIVATE TileDB::tiledb_shared) diff --git a/frmts/tiledb/include_tiledb.h b/frmts/tiledb/include_tiledb.h index f721cb3c59cb..cec9844846ac 100644 --- a/frmts/tiledb/include_tiledb.h +++ b/frmts/tiledb/include_tiledb.h @@ -35,11 +35,6 @@ #pragma GCC system_header #endif -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4996) /* XXXX was deprecated */ -#endif - #ifdef INCLUDE_ONLY_TILEDB_VERSION #include "tiledb/tiledb_version.h" #else @@ -47,10 +42,6 @@ #include "tiledb/tiledb_experimental" #endif -#ifdef _MSC_VER -#pragma warning(pop) -#endif - #if TILEDB_VERSION_MAJOR > 2 || \ (TILEDB_VERSION_MAJOR == 2 && TILEDB_VERSION_MINOR >= 9) #define HAS_TILEDB_GROUP diff --git a/frmts/tiledb/tiledbdense.cpp b/frmts/tiledb/tiledbdense.cpp index 3090bc670e77..e431faf60947 100644 --- a/frmts/tiledb/tiledbdense.cpp +++ b/frmts/tiledb/tiledbdense.cpp @@ -31,12 +31,6 @@ #include "tiledbheaders.h" -#ifdef _MSC_VER -#pragma warning(push) -// 'tiledb::Array::Array': was declared deprecated -#pragma warning(disable : 4996) /* XXXX was deprecated */ -#endif - constexpr const char *RASTER_DATASET_TYPE = "raster"; /************************************************************************/ @@ -2412,7 +2406,3 @@ GDALDataset *TileDBRasterDataset::CreateCopy(const char *pszFilename, } return nullptr; } - -#ifdef _MSC_VER -#pragma warning(pop) -#endif From e440870001312795fa05b894195910e47adf3e51 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 23 Apr 2024 19:41:00 +0200 Subject: [PATCH 04/11] CMake: require TileDB >= 2.15 --- cmake/helpers/CheckDependentLibraries.cmake | 2 +- doc/source/development/building_from_source.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake index 6eeb5d880066..fd06585acf00 100644 --- a/cmake/helpers/CheckDependentLibraries.cmake +++ b/cmake/helpers/CheckDependentLibraries.cmake @@ -751,7 +751,7 @@ gdal_check_package(Crnlib "enable gdal_DDS driver" CAN_DISABLE) gdal_check_package(basisu "Enable BASISU driver" CONFIG CAN_DISABLE) gdal_check_package(IDB "enable ogr_IDB driver" CAN_DISABLE) gdal_check_package(rdb "enable RIEGL RDB library" CONFIG CAN_DISABLE) -gdal_check_package(TileDB "enable TileDB driver" CONFIG CAN_DISABLE VERSION "2.7") +gdal_check_package(TileDB "enable TileDB driver" CONFIG CAN_DISABLE VERSION "2.15") gdal_check_package(OpenEXR "OpenEXR >=2.2" CAN_DISABLE) gdal_check_package(MONGOCXX "Enable MongoDBV3 driver" CAN_DISABLE) diff --git a/doc/source/development/building_from_source.rst b/doc/source/development/building_from_source.rst index bed99ec09c82..af4eaa44fef6 100644 --- a/doc/source/development/building_from_source.rst +++ b/doc/source/development/building_from_source.rst @@ -1907,6 +1907,8 @@ TileDB The `TileDB ` library is required for the :ref:`raster.tiledb` driver. Specify install prefix in the ``CMAKE_PREFIX_PATH`` variable. +TileDB >= 2.15 is required since GDAL 3.9 + .. option:: GDAL_USE_TILEDB=ON/OFF Control whether to use TileDB. Defaults to ON when TileDB is found. From e4e71a89cae863a9d84b476679245680e71ecbbc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 23 Apr 2024 20:10:54 +0200 Subject: [PATCH 05/11] TileDB: cleanups related to requiring TileDB >= 2.7 --- frmts/tiledb/tiledbdense.cpp | 137 +---------------------------------- 1 file changed, 2 insertions(+), 135 deletions(-) diff --git a/frmts/tiledb/tiledbdense.cpp b/frmts/tiledb/tiledbdense.cpp index e431faf60947..40565fb2e812 100644 --- a/frmts/tiledb/tiledbdense.cpp +++ b/frmts/tiledb/tiledbdense.cpp @@ -107,8 +107,6 @@ static const char *index_type_name(TILEDB_INTERLEAVE_MODE eMode) /* SetBuffer() */ /************************************************************************/ -#if ((TILEDB_VERSION_MAJOR > 2) || \ - (TILEDB_VERSION_MAJOR == 2 && TILEDB_VERSION_MINOR >= 4)) static CPLErr SetBuffer(tiledb::Query *poQuery, GDALDataType eType, const CPLString &osAttrName, void *pImage, int nSize) { @@ -175,74 +173,6 @@ static CPLErr SetBuffer(tiledb::Query *poQuery, GDALDataType eType, } return CE_None; } -#else -static CPLErr SetBuffer(tiledb::Query *poQuery, GDALDataType eType, - const CPLString &osAttrName, void *pImage, int nSize) -{ - switch (eType) - { - case GDT_Byte: - poQuery->set_buffer( - osAttrName, reinterpret_cast(pImage), nSize); - break; - case GDT_Int8: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_UInt16: - poQuery->set_buffer( - osAttrName, reinterpret_cast(pImage), nSize); - break; - case GDT_UInt32: - poQuery->set_buffer( - osAttrName, reinterpret_cast(pImage), nSize); - break; - case GDT_UInt64: - poQuery->set_buffer(osAttrName, - reinterpret_cast(pImage), nSize); - break; - case GDT_Int16: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_Int32: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_Int64: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_Float32: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_Float64: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize); - break; - case GDT_CInt16: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize * 2); - break; - case GDT_CInt32: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize * 2); - break; - case GDT_CFloat32: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize * 2); - break; - case GDT_CFloat64: - poQuery->set_buffer(osAttrName, reinterpret_cast(pImage), - nSize * 2); - break; - default: - return CE_Failure; - } - return CE_None; -} -#endif /************************************************************************/ /* TileDBRasterBand() */ @@ -683,11 +613,7 @@ CPLErr TileDBRasterDataset::TrySaveXML() if (psTree == nullptr) { /* If we have unset all metadata, we have to delete the PAM file */ -#if TILEDB_VERSION_MAJOR == 1 && TILEDB_VERSION_MINOR < 7 - vfs.remove_file(psPam->pszPamFilename); -#else m_array->delete_metadata(GDAL_ATTRIBUTE_NAME); -#endif return CE_None; } @@ -757,11 +683,9 @@ CPLErr TileDBRasterDataset::TrySaveXML() /* Try saving the auxiliary metadata. */ /* -------------------------------------------------------------------- */ - bool bSaved = false; CPLErrorHandlerPusher oQuietError(CPLQuietErrorHandler); char *pszTree = CPLSerializeXMLTree(psTree); -#if TILEDB_VERSION_MAJOR > 1 || TILEDB_VERSION_MINOR >= 7 if (eAccess == GA_ReadOnly) { if (nTimestamp) @@ -791,64 +715,8 @@ CPLErr TileDBRasterDataset::TrySaveXML() static_cast(strlen(pszTree)), pszTree); } - bSaved = true; -#endif - - // cppcheck-suppress knownConditionTrueFalse - if (!bSaved) - { - vfs.touch(psPam->pszPamFilename); - tiledb::VFS::filebuf fbuf(vfs); - fbuf.open(psPam->pszPamFilename, std::ios::out); - std::ostream os(&fbuf); - - if (os.good()) - { - os.write(pszTree, strlen(pszTree)); - bSaved = true; - } - - fbuf.close(); - } - CPLFree(pszTree); - /* -------------------------------------------------------------------- - */ - /* If it fails, check if we have a proxy directory for auxiliary */ - /* metadata to be stored in, and try to save there. */ - /* -------------------------------------------------------------------- - */ - CPLErr eErr = CE_None; - - if (bSaved) - eErr = CE_None; - else - { - const char *pszBasename = GetDescription(); - - if (psPam->osPhysicalFilename.length() > 0) - pszBasename = psPam->osPhysicalFilename; - - const char *pszNewPam = nullptr; - if (PamGetProxy(pszBasename) == nullptr && - ((pszNewPam = PamAllocateProxy(pszBasename)) != nullptr)) - { - CPLErrorReset(); - CPLFree(psPam->pszPamFilename); - psPam->pszPamFilename = CPLStrdup(pszNewPam); - eErr = TrySaveXML(); - } - /* No way we can save into a /vsicurl resource */ - else if (!VSISupportsSequentialWrite(psPam->pszPamFilename, false)) - { - CPLError(CE_Warning, CPLE_AppDefined, - "Unable to save auxiliary information in %s.", - psPam->pszPamFilename); - eErr = CE_Warning; - } - } - /* -------------------------------------------------------------------- */ /* Cleanup */ @@ -857,7 +725,7 @@ CPLErr TileDBRasterDataset::TrySaveXML() if (psTree) CPLDestroyXMLNode(psTree); - return eErr; + return CE_None; } catch (const std::exception &e) { @@ -927,7 +795,6 @@ CPLErr TileDBRasterDataset::TryLoadCachedXML(char ** /*papszSiblingFiles*/, { CPLErrorHandlerPusher oQuietError(CPLQuietErrorHandler); -#if TILEDB_VERSION_MAJOR > 1 || TILEDB_VERSION_MINOR >= 7 if (bReload) { tiledb_datatype_t v_type = @@ -957,7 +824,7 @@ CPLErr TileDBRasterDataset::TryLoadCachedXML(char ** /*papszSiblingFiles*/, } psTree = CPLParseXMLString(osMetaDoc); } -#endif + if (bReload && psTree == nullptr && vfs.is_file(psPam->pszPamFilename)) { From 0a202ae4ca0cbdb68b9febb50a24e582c67da4d5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 23 Apr 2024 19:42:53 +0200 Subject: [PATCH 06/11] TileDB: cleanups related to requiring TileDB >= 2.9 --- autotest/ogr/ogr_tiledb.py | 5 ----- frmts/tiledb/include_tiledb.h | 5 ----- frmts/tiledb/tiledbcommon.cpp | 4 ---- frmts/tiledb/tiledbdrivercore.cpp | 2 -- frmts/tiledb/tiledbsparse.cpp | 14 -------------- 5 files changed, 30 deletions(-) diff --git a/autotest/ogr/ogr_tiledb.py b/autotest/ogr/ogr_tiledb.py index e9fcf1f2c2bd..2311a3f6f962 100644 --- a/autotest/ogr/ogr_tiledb.py +++ b/autotest/ogr/ogr_tiledb.py @@ -1029,11 +1029,6 @@ def test_ogr_tiledb_switch_between_read_and_write(): def test_ogr_tiledb_create_group(): - if "CREATE_GROUP" not in gdal.GetDriverByName("TileDB").GetMetadataItem( - gdal.DMD_CREATIONOPTIONLIST - ): - pytest.skip("CREATE_GROUP not supported in TileDB < 2.9") - if os.path.exists("tmp/test.tiledb"): shutil.rmtree("tmp/test.tiledb") diff --git a/frmts/tiledb/include_tiledb.h b/frmts/tiledb/include_tiledb.h index cec9844846ac..9eecd245d561 100644 --- a/frmts/tiledb/include_tiledb.h +++ b/frmts/tiledb/include_tiledb.h @@ -42,11 +42,6 @@ #include "tiledb/tiledb_experimental" #endif -#if TILEDB_VERSION_MAJOR > 2 || \ - (TILEDB_VERSION_MAJOR == 2 && TILEDB_VERSION_MINOR >= 9) -#define HAS_TILEDB_GROUP -#endif - #if TILEDB_VERSION_MAJOR > 2 || \ (TILEDB_VERSION_MAJOR == 2 && TILEDB_VERSION_MINOR >= 10) #define HAS_TILEDB_BOOL diff --git a/frmts/tiledb/tiledbcommon.cpp b/frmts/tiledb/tiledbcommon.cpp index 5679a6f5b9eb..eb38195c9424 100644 --- a/frmts/tiledb/tiledbcommon.cpp +++ b/frmts/tiledb/tiledbcommon.cpp @@ -150,14 +150,12 @@ int TileDBDataset::Identify(GDALOpenInfo *poOpenInfo) CPLString osArrayPath = TileDBDataset::VSI_to_tiledb_uri(poOpenInfo->pszFilename); const auto eType = tiledb::Object::object(ctx, osArrayPath).type(); -#ifdef HAS_TILEDB_GROUP if ((poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) != 0) { if (eType == tiledb::Object::Type::Array || eType == tiledb::Object::Type::Group) return true; } -#endif #ifdef HAS_TILEDB_MULTIDIM if ((poOpenInfo->nOpenFlags & GDAL_OF_MULTIDIM_RASTER) != 0) { @@ -259,7 +257,6 @@ GDALDataset *TileDBDataset::Open(GDALOpenInfo *poOpenInfo) TileDBDataset::VSI_to_tiledb_uri(poOpenInfo->pszFilename); const auto eType = tiledb::Object::object(oCtx, osPath).type(); -#ifdef HAS_TILEDB_GROUP if ((poOpenInfo->nOpenFlags & GDAL_OF_VECTOR) != 0 && eType == tiledb::Object::Type::Group) { @@ -314,7 +311,6 @@ GDALDataset *TileDBDataset::Open(GDALOpenInfo *poOpenInfo) } return nullptr; } -#endif #endif tiledb::ArraySchema schema(oCtx, osPath); diff --git a/frmts/tiledb/tiledbdrivercore.cpp b/frmts/tiledb/tiledbdrivercore.cpp index d9ae98c5da1d..b83b94e45370 100644 --- a/frmts/tiledb/tiledbdrivercore.cpp +++ b/frmts/tiledb/tiledbdrivercore.cpp @@ -149,11 +149,9 @@ void TileDBDriverSetCommonMetadata(GDALDriver *poDriver) "