Skip to content

Commit

Permalink
Fix feature access (#6564)
Browse files Browse the repository at this point in the history 
Co-authored-by: Björn Harrtell <bjorn@wololo.org>
  • Loading branch information
@github-actions @bjornharrtell
github-actions[bot] and bjornharrtell committed Jul 7, 2022
1 parent b4a762f commit 61bbcfd
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 51 deletions.
60 changes: 31 additions & 29 deletions flatgeobuf/flatgeobuf_c.cpp
View file
Expand Up @@ -15,15 +15,11 @@ uint8_t FLATGEOBUF_MAGICBYTES_SIZE = sizeof(flatgeobuf_magicbytes);
uint32_t INIT_BUFFER_SIZE = 1024 * 4;

template <typename T>
char *to_string(uint8_t *data) {
void parse_value(uint8_t *data, char **values, uint16_t i, uint32_t &offset, bool found)
{
using std::to_string;
return msStrdup(to_string(*((T*)data)).c_str());
}

template <typename T>
void parse_value(uint8_t *data, char **values, uint16_t i, uint32_t &offset, bool found) {
if (found)
values[i] = to_string<int8_t>(data + offset);
values[i] = msStrdup(to_string(*((T*) (data + offset))).c_str());
offset += sizeof(T);
}

Expand Down Expand Up @@ -73,18 +69,21 @@ void flatgeobuf_ensure_point(ctx *ctx, uint32_t len)
}
}

void flatgeobuf_ensure_buf(ctx *ctx, uint32_t size)
int flatgeobuf_ensure_buf(ctx *ctx, uint32_t size)
{
if (size > 100 * 1024 * 1024)
return -1;
if (!ctx->buf) {
ctx->buf_size = INIT_BUFFER_SIZE;
ctx->buf = (uint8_t *) malloc(ctx->buf_size);
return;
return 0;
}
if (ctx->buf_size < size) {
ctx->buf_size = ctx->buf_size * 2;
ctx->buf = (uint8_t *) realloc(ctx->buf, ctx->buf_size);
flatgeobuf_ensure_buf(ctx, size);
}
return 0;
}

int flatgeobuf_decode_feature(ctx *ctx, layerObj *layer, shapeObj *shape)
Expand All @@ -101,7 +100,10 @@ int flatgeobuf_decode_feature(ctx *ctx, layerObj *layer, shapeObj *shape)
}

ctx->offset += sizeof(uoffset_t);
flatgeobuf_ensure_buf(ctx, featureSize);
if (flatgeobuf_ensure_buf(ctx, featureSize) != 0) {
msSetError(MS_FGBERR, "Failed to allocate buffer for feature", "flatgeobuf_decode_feature");
return -1;
}

if (VSIFReadL(ctx->buf, 1, featureSize, ctx->file) != featureSize) {
msSetError(MS_FGBERR, "Failed to read feature", "flatgeobuf_decode_feature");
Expand Down Expand Up @@ -168,37 +170,37 @@ int flatgeobuf_decode_properties(ctx *ctx, layerObj *layer, shapeObj *shape)
type = column.type;
switch (type) {
case flatgeobuf_column_type_bool:
parse_value<uint8_t>(data + offset, values, ii, offset, found);
parse_value<uint8_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_byte:
parse_value<int8_t>(data + offset, values, ii, offset, found);
parse_value<int8_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_ubyte:
parse_value<uint8_t>(data + offset, values, ii, offset, found);
parse_value<uint8_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_short:
parse_value<int16_t>(data + offset, values, ii, offset, found);
parse_value<int16_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_ushort:
parse_value<uint16_t>(data + offset, values, ii, offset, found);
parse_value<uint16_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_int:
parse_value<int32_t>(data + offset, values, ii, offset, found);
break;
parse_value<int32_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_uint:
parse_value<uint32_t>(data + offset, values, ii, offset, found);
parse_value<uint32_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_long:
parse_value<int64_t>(data + offset, values, ii, offset, found);
break;
parse_value<int64_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_ulong:
parse_value<uint64_t>(data + offset, values, ii, offset, found);
parse_value<uint64_t>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_float:
parse_value<float>(data + offset, values, ii, offset, found);
parse_value<float>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_double:
parse_value<double>(data + offset, values, ii, offset, found);
parse_value<double>(data, values, ii, offset, found);
break;
case flatgeobuf_column_type_datetime:
case flatgeobuf_column_type_string: {
Expand Down Expand Up @@ -230,7 +232,7 @@ int flatgeobuf_check_magicbytes(ctx *ctx)
msSetError(MS_FGBERR, "Unexpected offset", "flatgeobuf_check_magicbytes");
return -1;
}
ctx->buf = (uint8_t *) malloc(FLATGEOBUF_MAGICBYTES_SIZE);
flatgeobuf_ensure_buf(ctx, FLATGEOBUF_MAGICBYTES_SIZE);
if (VSIFReadL(ctx->buf, 8, 1, ctx->file) != 1) {
msSetError(MS_FGBERR, "Failed to read magicbytes", "flatgeobuf_check_magicbytes");
return -1;
Expand Down Expand Up @@ -262,7 +264,9 @@ int flatgeobuf_decode_header(ctx *ctx)
return -1;
}
ctx->offset += sizeof(uoffset_t);
flatgeobuf_ensure_buf(ctx, headerSize);
if (flatgeobuf_ensure_buf(ctx, headerSize) != -1) {
msSetError(MS_FGBERR, "Failed to allocate buffer for header", "flatgeobuf_decode_header");
}
if (VSIFReadL(ctx->buf, 1, headerSize, ctx->file) != headerSize) {
msSetError(MS_FGBERR, "Failed to read header", "flatgeobuf_decode_header");
return -1;
Expand Down Expand Up @@ -349,11 +353,9 @@ int flatgeobuf_index_skip(ctx *ctx)

int flatgeobuf_read_feature_offset(ctx *ctx, uint64_t index, uint64_t *featureOffset)
{
try
{
const auto treeSize = PackedRTree::size(ctx->features_count, ctx->index_node_size);
try {
const auto levelBounds = PackedRTree::generateLevelBounds(ctx->features_count, ctx->index_node_size);
const auto bottomLevelOffset = ctx->index_offset - treeSize + (levelBounds.front().first * sizeof(NodeItem));
const auto bottomLevelOffset = ctx->index_offset + (levelBounds.front().first * sizeof(NodeItem));
const auto nodeItemOffset = bottomLevelOffset + (index * sizeof(NodeItem));
const auto featureOffsetOffset = nodeItemOffset + (sizeof(double) * 4);
if (VSIFSeekL(ctx->file, featureOffsetOffset, SEEK_SET) == -1) {
Expand Down
3 changes: 2 additions & 1 deletion flatgeobuf/flatgeobuf_c.h
View file
Expand Up @@ -106,6 +106,7 @@ typedef struct flatgeobuf_ctx
uint32_t point_len;

bool is_null_geom;
uint64_t feature_index;
int ms_type;
uint8_t *properties;
uint32_t properties_size;
Expand All @@ -114,7 +115,7 @@ typedef struct flatgeobuf_ctx
flatgeobuf_ctx *flatgeobuf_init_ctx();
void flatgeobuf_free_ctx(flatgeobuf_ctx *ctx);

void flatgeobuf_ensure_buf(flatgeobuf_ctx *ctx, uint32_t size);
int flatgeobuf_ensure_buf(flatgeobuf_ctx *ctx, uint32_t size);
void flatgeobuf_ensure_line(flatgeobuf_ctx *ctx, uint32_t len);
void flatgeobuf_ensure_point(flatgeobuf_ctx *ctx, uint32_t len);

Expand Down
46 changes: 27 additions & 19 deletions mapflatgeobuf.c
View file
Expand Up @@ -42,12 +42,12 @@
static void msFGBPassThroughFieldDefinitions(layerObj *layer, flatgeobuf_ctx *ctx)
{
for (int i = 0; i < ctx->columns_len; i++) {
char item[16];
//int nWidth=0, nPrecision=0;
char item[255];
char gml_width[32], gml_precision[32];
const char *gml_type = NULL;

flatgeobuf_column column = ctx->columns[i];
strncpy(item, column.name, 255 - 1);

gml_width[0] = '\0';
gml_precision[0] = '\0';
Expand All @@ -61,25 +61,25 @@ static void msFGBPassThroughFieldDefinitions(layerObj *layer, flatgeobuf_ctx *ct
case flatgeobuf_column_type_int:
case flatgeobuf_column_type_uint:
gml_type = "Integer";
//sprintf( gml_width, "%d", nWidth );
sprintf( gml_width, "%d", 4 );
break;
case flatgeobuf_column_type_long:
case flatgeobuf_column_type_ulong:
gml_type = "Long";
//sprintf( gml_width, "%d", nWidth );
sprintf( gml_width, "%d", 8 );
break;
case flatgeobuf_column_type_float:
case flatgeobuf_column_type_double:
gml_type = "Real";
//sprintf( gml_width, "%d", nWidth );
//sprintf( gml_precision, "%d", nPrecision );
sprintf( gml_width, "%d", 8 );
sprintf( gml_precision, "%d", 15 );
break;
case flatgeobuf_column_type_string:
case flatgeobuf_column_type_json:
case flatgeobuf_column_type_datetime:
default:
gml_type = "Character";
//sprintf( gml_width, "%d", nWidth );
sprintf( gml_width, "%d", 4096 );
break;
}

Expand Down Expand Up @@ -212,22 +212,25 @@ int msFlatGeobufLayerNextShape(layerObj *layer, shapeObj *shape)
if (!ctx)
return MS_FAILURE;

if (ctx->search_result) {
if (ctx->search_index >= ctx->search_result_len - 1)
return MS_DONE;
flatgeobuf_search_item item = ctx->search_result[ctx->search_index];
if (VSIFSeekL(ctx->file, ctx->feature_offset + item.offset, SEEK_SET) == -1) {
msSetError(MS_FGBERR, "Unable to seek in file", "msFlatGeobufLayerNextShape");
return MS_FAILURE;
}
ctx->offset = ctx->feature_offset + item.offset;
ctx->search_index++;
}

do {
if (ctx->search_result) {
if (ctx->search_index >= ctx->search_result_len - 1)
return MS_DONE;
flatgeobuf_search_item item = ctx->search_result[ctx->search_index];
if (VSIFSeekL(ctx->file, ctx->feature_offset + item.offset, SEEK_SET) == -1) {
msSetError(MS_FGBERR, "Unable to seek in file", "msFlatGeobufLayerNextShape");
return MS_FAILURE;
}
ctx->offset = ctx->feature_offset + item.offset;
ctx->search_index++;
ctx->feature_index = item.index;
}
int ret = flatgeobuf_decode_feature(ctx, layer, shape);
if (ret == -1)
return MS_FAILURE;
shape->index = ctx->feature_index;
if (!ctx->search_result)
ctx->feature_index++;
if (ctx->done)
return MS_DONE;
} while(ctx->is_null_geom);
Expand All @@ -237,13 +240,18 @@ int msFlatGeobufLayerNextShape(layerObj *layer, shapeObj *shape)

int msFlatGeobufLayerGetShape(layerObj *layer, shapeObj *shape, resultObj *record)
{

(void)shape;
(void)record;
flatgeobuf_ctx *ctx;
ctx = layer->layerinfo;
if (!ctx)
return MS_FAILURE;
long i = record->shapeindex;
if (i < 0 || (uint64_t) i >= ctx->features_count) {
msSetError(MS_MISCERR, "Invalid feature id", "msFlatGeobufLayerGetShape");
return MS_FAILURE;
}
uint64_t offset;
flatgeobuf_read_feature_offset(ctx, i, &offset);
if (VSIFSeekL(ctx->file, ctx->feature_offset + offset, SEEK_SET) == -1) {
Expand Down
Binary file modified msautotest/misc/data/africa.fgb
View file
Binary file not shown.
42 changes: 40 additions & 2 deletions msautotest/misc/flatgeobuf.map
View file
Expand Up @@ -15,9 +15,32 @@ MAP
IMAGECOLOR 255 255 255
IMAGETYPE png

WEB
METADATA
"wfs_title" "Test FlatGeobuf WFS output"
"wfs_abstract" "Longer text describing the FlatGeobuf WFS service."
"wfs_onlineresource" "http://localhost/path/to/flatgeobuf?"
"wfs_srs" "EPSG:4326 EPSG:4269 EPSG:3978 EPSG:3857"
"wfs_enable_request" "*"
END
END

PROJECTION
"init=epsg:4326"
END

/* Africa Continent */
LAYER
NAME "africa-continent"
METADATA
"wfs_title" "Africa Continent"
"wfs_srs" "EPSG:4326"
"gml_include_items" "all"
"gml_featureid" "id"
"gml_types" "auto"
"wfs_enable_request" "*"
"wfs_use_default_extent_for_getfeature" "false"
END
TYPE POLYGON
STATUS ON
CONNECTIONTYPE flatgeobuf
Expand All @@ -28,12 +51,24 @@ MAP
COLOR 50 50 50
OUTLINECOLOR 120 120 120
END #style
END #class
END #class
PROJECTION
"init=epsg:4326"
END
END # layer

/* Africa classes */
LAYER
NAME "africa-classes"
METADATA
"wfs_title" "Africa Classes"
"wfs_srs" "EPSG:4326"
"gml_include_items" "all"
"gml_featureid" "id"
"gml_types" "auto"
"wfs_enable_request" "*"
"wfs_use_default_extent_for_getfeature" "false"
END
TYPE POLYGON
STATUS ON
CONNECTIONTYPE flatgeobuf
Expand Down Expand Up @@ -64,7 +99,10 @@ MAP
OUTLINECOLOR 120 120 120
END #style
END #class
TEMPLATE "ttt.html"
TEMPLATE "ttt.html"
PROJECTION
"init=epsg:4326"
END
END # layer

END # map

0 comments on commit 61bbcfd

Please sign in to comment.