Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TileDB: use GEOM_WKB type when creating geometry columns with TileDB >= 2.21 #9726

Merged
merged 1 commit into from
May 16, 2024
Merged

TileDB: use GEOM_WKB type when creating geometry columns with TileDB >= 2.21 #9726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions autotest/ogr/ogr_tiledb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,15 @@

pytestmark = pytest.mark.require_driver("TileDB")


def get_tiledb_version():
drv = gdal.GetDriverByName("TileDB")
if drv is None:
return (0, 0, 0)
x, y, z = [int(x) for x in drv.GetMetadataItem("TILEDB_VERSION").split(".")]
return (x, y, z)


###############################################################################


Expand Down Expand Up @@ -1655,3 +1664,49 @@ def test_ogr_tiledb_arrow_stream_numpy_detailed_spatial_filter():
ds = None

shutil.rmtree("tmp/test.tiledb")


###############################################################################


@pytest.mark.skipif(get_tiledb_version() < (2, 21, 0), reason="tiledb 2.21 required")
@pytest.mark.parametrize(
"TILEDB_WKB_GEOMETRY_TYPE, OGR_TILEDB_WRITE_GEOMETRY_ATTRIBUTE_NAME",
(("BLOB", True), ("GEOM_WKB", True), (None, False)),
)
def test_ogr_tiledb_tiledb_geometry_type(
tmp_path, TILEDB_WKB_GEOMETRY_TYPE, OGR_TILEDB_WRITE_GEOMETRY_ATTRIBUTE_NAME
):
with gdal.config_options(
{
"TILEDB_WKB_GEOMETRY_TYPE": TILEDB_WKB_GEOMETRY_TYPE,
"OGR_TILEDB_WRITE_GEOMETRY_ATTRIBUTE_NAME": "YES"
if OGR_TILEDB_WRITE_GEOMETRY_ATTRIBUTE_NAME
else "NO",
}
):
filename = str(tmp_path / "test.tiledb")
with ogr.GetDriverByName("TileDB").CreateDataSource(filename) as ds:
srs = osr.SpatialReference()
srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
srs.ImportFromEPSG(32631)
lyr = ds.CreateLayer("test", srs=srs)
f = ogr.Feature(lyr.GetLayerDefn())
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt("POINT (1 2)"))
lyr.CreateFeature(f)

with ogr.Open(filename) as ds:
lyr = ds.GetLayer(0)
tiledb_md = json.loads(lyr.GetMetadata_List("json:TILEDB")[0])
expected = {
"name": "wkb_geometry",
"type": TILEDB_WKB_GEOMETRY_TYPE
if TILEDB_WKB_GEOMETRY_TYPE
else "GEOM_WKB",
"cell_val_num": "variable",
"nullable": False,
"filter_list": [],
}
assert expected in tiledb_md["schema"]["attributes"]
f = lyr.GetNextFeature()
assert f.GetGeometryRef().ExportToWkt() == "POINT (1 2)"
5 changes: 5 additions & 0 deletions frmts/tiledb/tiledbdrivercore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,11 @@ void TileDBDriverSetCommonMetadata(GDALDriver *poDriver)
poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
poDriver->SetMetadataItem(GDAL_DCAP_CREATECOPY, "YES");

poDriver->SetMetadataItem(
"TILEDB_VERSION",
STRINGIFY(TILEDB_VERSION_MAJOR) "." STRINGIFY(
TILEDB_VERSION_MINOR) "." STRINGIFY(TILEDB_VERSION_PATCH));
}

/************************************************************************/
Expand Down
107 changes: 97 additions & 10 deletions frmts/tiledb/tiledbsparse.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,23 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx,
}
break;
}

#ifdef HAS_TILEDB_GEOM_WKB_WKT
case TILEDB_GEOM_WKT:
{
eType = OFTString;
fieldValues.push_back(std::make_shared<std::string>());
break;
}

case TILEDB_GEOM_WKB:
{
eType = OFTBinary;
fieldValues.push_back(std::make_shared<std::vector<uint8_t>>());
break;
}
#endif

case TILEDB_CHAR:
case TILEDB_INT8:
case TILEDB_UINT32:
Expand Down Expand Up @@ -844,10 +861,6 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx,
case TILEDB_TIME_FS:
case TILEDB_TIME_AS:
case TILEDB_ANY:
#ifdef HAS_TILEDB_GEOM_WKB_WKT
case TILEDB_GEOM_WKB: // TODO: take that into account
case TILEDB_GEOM_WKT:
#endif
{
// TODO ?
const char *pszTypeName = "";
Expand Down Expand Up @@ -981,6 +994,24 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx,
{
osGeomColumn.assign(static_cast<const char *>(v_r), v_num);
}
#ifdef HAS_TILEDB_GEOM_WKB_WKT
else
{
// If GEOMETRY_ATTRIBUTE_NAME isn't defined, identify the first attribute
// of type TILEDB_GEOM_WKB
osGeomColumn.clear();
for (unsigned i = 0; i < m_schema->attribute_num(); ++i)
{
auto attr = m_schema->attribute(i);
if (attr.type() == TILEDB_GEOM_WKB &&
attr.cell_val_num() == TILEDB_VAR_NUM)
{
if (osGeomColumn.empty())
osGeomColumn = attr.name();
}
}
}
#endif

bool bFoundWkbGeometry = false;
CPLJSONArray oAttributes;
Expand Down Expand Up @@ -1021,12 +1052,29 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx,
continue;
}
if (attr.name() == osGeomColumn &&
(attr.type() == TILEDB_UINT8 || attr.type() == TILEDB_BLOB) &&
(attr.type() == TILEDB_UINT8 || attr.type() == TILEDB_BLOB
#ifdef HAS_TILEDB_GEOM_WKB_WKT
|| attr.type() == TILEDB_GEOM_WKB
#endif
) &&
attr.cell_val_num() == TILEDB_VAR_NUM)
{
bFoundWkbGeometry = true;
continue;
}
#ifdef HAS_TILEDB_GEOM_WKB_WKT
else if (attr.type() == TILEDB_GEOM_WKB &&
attr.cell_val_num() == TILEDB_VAR_NUM)
{
CPLError(CE_Warning, CPLE_AppDefined,
"Attribute %s has type GeomWKB, but another one (%s) is "
"already used as the OGR geometry column. Dealing with %s "
"has a Binary field",
attr.name().c_str(), osGeomColumn.c_str(),
attr.name().c_str());
}
#endif

const bool bIsSingle = attr.cell_val_num() == 1;
if (attr.cell_val_num() > 1 && attr.cell_val_num() != TILEDB_VAR_NUM)
{
Expand Down Expand Up @@ -1265,6 +1313,19 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays)
reinterpret_cast<std::byte *>(m_abyGeometries->data()),
m_abyGeometries->size());
}
#ifdef HAS_TILEDB_GEOM_WKB_WKT
else if (colType == TILEDB_GEOM_WKB)
{
m_query->set_offsets_buffer(pszGeomColName,
*m_anGeometryOffsets);
// We could use API expected std::byte, but this requires
// TileDB 2.22 because of https://github.com/TileDB-Inc/TileDB/pull/4826
m_query->set_data_buffer(
pszGeomColName,
static_cast<void *>(m_abyGeometries->data()),
m_abyGeometries->size());
}
#endif
else
{
CPLAssert(false);
Expand Down Expand Up @@ -3545,10 +3606,21 @@ void OGRTileDBLayer::InitializeSchemaAndArray()
if (pszGeomColName)
{
const char *pszWkbBlobType =
CPLGetConfigOption("TILEDB_WKB_GEOMETRY_TYPE", "BLOB");
CPLGetConfigOption("TILEDB_WKB_GEOMETRY_TYPE",
#ifdef HAS_TILEDB_GEOM_WKB_WKT
"GEOM_WKB"
#else
"BLOB"
#endif
);
auto wkbGeometryAttr = tiledb::Attribute::create(
*m_ctx, pszGeomColName,
EQUAL(pszWkbBlobType, "UINT8") ? TILEDB_UINT8 : TILEDB_BLOB);
#ifdef HAS_TILEDB_GEOM_WKB_WKT
EQUAL(pszWkbBlobType, "GEOM_WKB") ? TILEDB_GEOM_WKB :
#endif
EQUAL(pszWkbBlobType, "UINT8") ? TILEDB_UINT8
: TILEDB_BLOB);

wkbGeometryAttr.set_filter_list(*m_filterList);
wkbGeometryAttr.set_cell_val_num(TILEDB_VAR_NUM);
m_schema->add_attribute(wkbGeometryAttr);
Expand Down Expand Up @@ -3760,7 +3832,9 @@ void OGRTileDBLayer::InitializeSchemaAndArray()
m_osFIDColumn.c_str());
}

if (pszGeomColName)
if (pszGeomColName &&
CPLTestBool(CPLGetConfigOption(
"OGR_TILEDB_WRITE_GEOMETRY_ATTRIBUTE_NAME", "YES")))
{
m_array->put_metadata("GEOMETRY_ATTRIBUTE_NAME", TILEDB_STRING_UTF8,
static_cast<int>(strlen(pszGeomColName)),
Expand Down Expand Up @@ -4285,19 +4359,32 @@ void OGRTileDBLayer::FlushArrays()
if (pszGeomColName)
{
m_anGeometryOffsets->pop_back();
if (m_schema->attribute(pszGeomColName).type() == TILEDB_UINT8)
const auto eTileDBType = m_schema->attribute(pszGeomColName).type();
if (eTileDBType == TILEDB_UINT8)
{
query.set_buffer(pszGeomColName, *m_anGeometryOffsets,
*m_abyGeometries);
}
else if (m_schema->attribute(pszGeomColName).type() == TILEDB_BLOB)
else if (eTileDBType == TILEDB_BLOB)
{
query.set_buffer(
pszGeomColName, m_anGeometryOffsets->data(),
m_anGeometryOffsets->size(),
reinterpret_cast<std::byte *>(m_abyGeometries->data()),
m_abyGeometries->size());
}
#ifdef HAS_TILEDB_GEOM_WKB_WKT
else if (eTileDBType == TILEDB_GEOM_WKB)
{
query.set_offsets_buffer(pszGeomColName, *m_anGeometryOffsets);
// We could use API expected std::byte, but this requires
// TileDB 2.22 because of https://github.com/TileDB-Inc/TileDB/pull/4826
query.set_data_buffer(
pszGeomColName,
static_cast<void *>(m_abyGeometries->data()),
m_abyGeometries->size());
}
#endif
else
{
CPLAssert(false);
Expand Down
Loading