Skip to content

Commit

Permalink
MVT: Improve performance and memory usage
Browse files Browse the repository at this point in the history
- Reduce memory usage (raises protobuf-c requirement to 1.1).
- Improves performance when merging tiles (from parallel queries).
- Uses protobuf values to reduce the number of transformations and
  memory usage.
- Uses a temporal context when transforming data (reading rows) to
  have a fast way of cleaning up memory before the parallel process
  starts.
- Processes texts and cstrings with each known reading functions
  instead of the `OidOutputFunctionCall` fallback.
- Avoids multiple hash function calls when a value is not found in
  the hash table (once for the lookup and once for the insertion).

Closes #576
Closes #4737
  • Loading branch information
Algunenano committed Aug 17, 2020
1 parent 9ef9130 commit 99c50d4
Show file tree
Hide file tree
Showing 6 changed files with 235 additions and 278 deletions.
5 changes: 5 additions & 0 deletions NEWS
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ PostGIS 3.1.0beta1
2020/xx/xx
Only tickets not included in 3.1.0alpha2

* Breaking changes *
- #4737, Bump minimum protobuf-c requirement to 1.1.0 (Raúl Marín)

* New features *
- #4698, Add a precision parameter to ST_AsEWKT (Raúl Marín)

Expand All @@ -23,6 +26,8 @@ Only tickets not included in 3.1.0alpha2
ST_BoundingDiagonal.
- #4741, Don't use ST_PointInsideCircle if you need indexes, use ST_DWithin instead.
Documentation adjusted (Darafei Praliaskouski)
- #4737, Improve performance and reduce memory usage in ST_AsMVT, especially in
queries involving parallelism (Raúl Marín)

* Bug fixes *
- #4691, Fix segfault during gist index creation with empty geometries (Raúl Marín)
Expand Down
8 changes: 3 additions & 5 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -998,7 +998,7 @@ if test "$CHECK_PROTOBUF" != "no"; then
dnl Try pkgconfig first
if test -n "$PKG_CONFIG"; then
dnl Ensure libprotobuf-c is of minimum required version
PKG_CHECK_MODULES([PROTOBUFC], [libprotobuf-c >= 1.0.0], [
PKG_CHECK_MODULES([PROTOBUFC], [libprotobuf-c >= 1.1.0], [
PROTOBUF_CPPFLAGS="$PROTOBUFC_CFLAGS";
PROTOBUF_LDFLAGS="$PROTOBUFC_LIBS";
], [
Expand Down Expand Up @@ -1069,10 +1069,8 @@ if test "$CHECK_PROTOBUF" != "no"; then
if test "$HAVE_PROTOBUF" = "yes"; then
AC_DEFINE([HAVE_LIBPROTOBUF], [1], [Define to 1 if libprotobuf-c is present])
AC_DEFINE_UNQUOTED([LIBPROTOBUF_VERSION], [$PROTOC_VERSION], [Numeric version number for libprotobuf-c])
if test $PROTOC_VERSION -ge 1001000; then
AC_DEFINE([HAVE_GEOBUF], [1], [Define to 1 if libprotobuf is >= 1.1])
HAVE_GEOBUF=yes
fi
AC_DEFINE([HAVE_GEOBUF], [1], [Define to 1 if geobuf is present])
HAVE_GEOBUF=yes
fi

AC_SUBST([HAVE_PROTOBUF])
Expand Down
19 changes: 16 additions & 3 deletions postgis/lwgeom_out_mvt.c
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,14 @@ Datum pgis_asmvt_transfn(PG_FUNCTION_ARGS)
elog(ERROR, "Missing libprotobuf-c");
PG_RETURN_NULL();
#else
MemoryContext aggcontext;
MemoryContext aggcontext, old_context;
mvt_agg_context *ctx;

if (!AggCheckCallContext(fcinfo, &aggcontext))
elog(ERROR, "%s called in non-aggregate context", __func__);
MemoryContextSwitchTo(aggcontext);

if (PG_ARGISNULL(0)) {
old_context = MemoryContextSwitchTo(aggcontext);
ctx = palloc(sizeof(*ctx));
ctx->name = "default";
if (PG_NARGS() > 2 && !PG_ARGISNULL(2))
Expand All @@ -151,7 +151,12 @@ Datum pgis_asmvt_transfn(PG_FUNCTION_ARGS)
ctx->id_name = text_to_cstring(PG_GETARG_TEXT_P(5));
else
ctx->id_name = NULL;

ctx->trans_context = AllocSetContextCreate(aggcontext, "MVT transfn", ALLOCSET_DEFAULT_SIZES);

MemoryContextSwitchTo(ctx->trans_context);
mvt_agg_init_context(ctx);
MemoryContextSwitchTo(old_context);
} else {
ctx = (mvt_agg_context *) PG_GETARG_POINTER(0);
}
Expand All @@ -160,7 +165,10 @@ Datum pgis_asmvt_transfn(PG_FUNCTION_ARGS)
elog(ERROR, "%s: parameter row cannot be other than a rowtype", __func__);
ctx->row = PG_GETARG_HEAPTUPLEHEADER(1);

old_context = MemoryContextSwitchTo(ctx->trans_context);
mvt_agg_transfn(ctx);
MemoryContextSwitchTo(old_context);

PG_FREE_IF_COPY(ctx->row, 1);
PG_RETURN_POINTER(ctx);
#endif
Expand Down Expand Up @@ -203,6 +211,7 @@ Datum pgis_asmvt_serialfn(PG_FUNCTION_ARGS)
PG_RETURN_NULL();
#else
mvt_agg_context *ctx;
bytea *result;
elog(DEBUG2, "%s called", __func__);
if (!AggCheckCallContext(fcinfo, NULL))
elog(ERROR, "%s called in non-aggregate context", __func__);
Expand All @@ -215,7 +224,11 @@ Datum pgis_asmvt_serialfn(PG_FUNCTION_ARGS)
}

ctx = (mvt_agg_context *) PG_GETARG_POINTER(0);
PG_RETURN_BYTEA_P(mvt_ctx_serialize(ctx));
result = mvt_ctx_serialize(ctx);
if (ctx->trans_context)
MemoryContextDelete(ctx->trans_context);
ctx->trans_context = NULL;
PG_RETURN_BYTEA_P(result);
#endif
}

Expand Down
Loading

0 comments on commit 99c50d4

Please sign in to comment.