diff --git a/.install-lazperf.sh b/.install-lazperf.sh new file mode 100644 index 00000000..9646861a --- /dev/null +++ b/.install-lazperf.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +git clone https://github.com/verma/laz-perf.git +cd laz-perf; cmake .; make; sudo make install diff --git a/.travis.yml b/.travis.yml index 92edfbab..f561e8e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,13 @@ language: c before_install: - sudo apt-get update - sudo apt-get install -q postgresql-server-dev-9.3 libcunit1-dev valgrind + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -qq + +install: + - sudo apt-get install -qq g++-4.8 + - export CXX="g++-4.8" + - sh .install-lazperf.sh addons: postgresql: "9.3" # for "installcheck" @@ -10,9 +17,10 @@ addons: before_script: ./autogen.sh script: - - ./configure CFLAGS="-Wall -O2 -g" + - ./configure CFLAGS="-Wall -O2 -g" --with-lazperf=/usr/local/ - make - make check - valgrind --leak-check=full --error-exitcode=1 lib/cunit/cu_tester - sudo make install - make installcheck || { cat pgsql/regression.diffs && false; } + - cd tools/benchmark_compression && sh compression_benchmark.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index b5419f52..46ebd7d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,7 +102,7 @@ include_directories (${ZLIB_INCLUDE_DIR}) #------------------------------------------------------------------------------ -# cunit and ght +# cunit, ght and lazperf if (WITH_TESTS) find_package (CUnit) @@ -114,6 +114,12 @@ if (LIBGHT_FOUND) set (HAVE_LIBGHT 1) endif (LIBGHT_FOUND) +find_package (LazPerf) + +if (LAZPERF_FOUND) + set (HAVE_LAZPERF 1) +endif (LAZPERF_FOUND) + #------------------------------------------------------------------------------ # use default install location if not specified diff --git a/Makefile b/Makefile index 731fd7c5..7aae7c0b 100644 --- a/Makefile +++ b/Makefile @@ -19,3 +19,6 @@ astyle: -type f \ -exec astyle --style=ansi --indent=tab --suffix=none {} ';' +maintainer-clean: + rm -f config.log config.mk config.status lib/pc_config.h configure + rm -rf autom4te.cache build diff --git a/README.md b/README.md index 5c6df95a..012345f4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ A PostgreSQL extension for storing point cloud (LIDAR) data. - LibXML2 development packages must be installed, usually "libxml2-dev" or "libxml2-devel". - CUnit packages must be installed, or [source built and installed](http://sourceforge.net/projects/cunit/ "CUnit"). - [Optional] GHT library may be installed for GHT compression support, [built from source](http://github.com/pramsey/libght/ "LibGHT") +- [Optional] LAZPERF library may be installed for LAZ compression support, [built from source](http://github.com/hobu/laz-perf "LAZPERF") Tests can be disabled by passing ``WITH_TESTS=FALSE`` to cmake, e.g. ``cmake .. -DWITH_TESTS=FALSE``. This removes the CUnit dependency. @@ -439,6 +440,7 @@ Now that you have created two tables, you'll see entries for them in the `pointc > Allowed global compression schemes are: > - auto -- determined by pcid > - ght -- no compression config supported +> - laz -- no compression config supported > - dimensional > configuration is a comma-separated list of per-dimension > compressions from this list: @@ -506,6 +508,7 @@ There are currently three supported compressions: - **None**, which stores points and patches as byte arrays using the type and formats described in the schema document. - **Dimensional**, which stores points the same as 'none' but stores patches as collections of dimensional data arrays, with an "appropriate" compression applied. Dimensional compression makes the most sense for smaller patch sizes, since small patches will tend to have more homogeneous dimensions. - **GHT** or "GeoHash Tree", which stores the points in a tree where each node stores the common values shared by all nodes below. For larger patch sizes, GHT should provide effective compression and performance for patch-wise operations. You must build Pointcloud with libght support to make use of the GHT compression. +- **LAZ** or "LASZip". You must build Pointcloud with LAZPERF support to make use of the LAZ compression. If no compression is declared in ``, then a compression of "none" is assumed. @@ -630,6 +633,17 @@ Where simple compression schemes fail, general purpose compression is applied to GHT patches are much like dimensional patches, except their internal structure is more opaque. Use LibGHT to read the GHT data buffer out into a GHT tree in memory. +### Patch Binary (LAZ) #### + + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: 3 = LAZ compression + uint32: npoints + uint32: LAZ data size + data[]: LAZ data + +LAZ patches are much like GHT patches. Use LAZPERF library to read the LAZ data buffer out into a LAZ buffer. + ## Loading Data ## The examples above show how to form patches from array of doubles, and well-known binary. You can write your own loader, using the uncompressed WKB format, or more simply you can load existing LIDAR files using the [PDAL](http://pointcloud.org "PDAL") processing and format conversion library. diff --git a/cmake/modules/FindLazPerf.cmake b/cmake/modules/FindLazPerf.cmake new file mode 100644 index 00000000..f60a5832 --- /dev/null +++ b/cmake/modules/FindLazPerf.cmake @@ -0,0 +1,16 @@ +# Find the LazPerf headers +# +# LAZPERF_INCLUDE_DIRS - The LazPerf include directory (directory where laz-perf/las.hpp was found) +# LAZPERF_FOUND - True if LazPerf found in system + + +FIND_PATH(LAZPERF_INCLUDE_DIR NAMES laz-perf/las.hpp) + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LazPerf DEFAULT_MSG LAZPERF_INCLUDE_DIR) + +IF(LAZPERF_FOUND) + SET(LAZPERF_INCLUDE_DIRS ${LAZPERF_INCLUDE_DIR}) +ENDIF(LAZPERF_FOUND) + +MARK_AS_ADVANCED(CLEAR LAZPERF_INCLUDE_DIR) diff --git a/config.mk.in b/config.mk.in index d02483a8..074441e7 100644 --- a/config.mk.in +++ b/config.mk.in @@ -1,5 +1,6 @@ CC = @CC@ CFLAGS = @CFLAGS@ +CXXFLAGS = -fPIC -std=c++0x XML2_CPPFLAGS = @XML2_CPPFLAGS@ XML2_LDFLAGS = @XML2_LDFLAGS@ @@ -16,4 +17,5 @@ GHT_LDFLAGS = @GHT_LDFLAGS@ PG_CONFIG = @PG_CONFIG@ PGXS = @PGXS@ -LIB_A = libpc.a \ No newline at end of file +LIB_A = libpc.a +LIB_A_LAZPERF = liblazperf.a diff --git a/configure.ac b/configure.ac index 76247a1b..4b7efdcf 100644 --- a/configure.ac +++ b/configure.ac @@ -12,6 +12,7 @@ dnl ***********************************************************************/ AC_INIT() AC_CONFIG_MACRO_DIR([macros]) AC_CONFIG_HEADERS([lib/pc_config.h]) +AC_LANG([C++]) dnl dnl Compilers @@ -325,6 +326,44 @@ fi AC_SUBST([GHT_LDFLAGS]) AC_SUBST([GHT_CPPFLAGS]) +dnl =========================================================================== +dnl Detect LazPerf +dnl =========================================================================== + +AC_ARG_WITH([lazperf], + [AS_HELP_STRING([--with-lazperf=DIR], [specify the base lazperf installation directory])], + [LAZPERFDIR="$withval"], [LAZPERFDIR=""]) + +if test "x$LAZPERFDIR" = "xyes"; then + AC_MSG_ERROR([you must specify a parameter to --with-lazperf, e.g. --with-lazperf=/opt/local]) +fi + +if test "x$LAZPERFDIR" = "x"; then + AC_CHECK_HEADER([las.hpp], + [FOUND_LAZPERF="YES"], + [FOUND_LAZPERF="NO"]) +elif test "x$GHTDIR" != "xno"; then + LAZPERF_CPPFLAGS="-I${LAZPERFDIR}/include/" + + CFLAGS="$LAZPERF_CPPFLAGS" + CPPFLAGS="$LAZPERF_CPPFLAGS -std=c++0x" + + AC_CHECK_HEADER([laz-perf/las.hpp], + [FOUND_LAZPERF="YES"], + [FOUND_LAZPERF="NO"]) +fi + +if test "x$FOUND_LAZPERF" = "xYES"; then + AC_DEFINE([HAVE_LAZPERF]) + LAZPERF_STATUS="enabled" + if test $LAZPERFDIR; then + LAZPERF_STATUS="$LAZPERFDIR/include/laz-perf" + fi +else + LAZPERF_STATUS="disabled" +fi + +AC_SUBST([LAZPERF_STATUS]) dnl =========================================================================== dnl Figure out where this script is running @@ -357,5 +396,6 @@ AC_MSG_RESULT([ PostgreSQL version: ${PGSQL_FULL_VERSION}]) AC_MSG_RESULT([ Libxml2 config: ${XML2CONFIG}]) AC_MSG_RESULT([ Libxml2 version: ${LIBXML2_VERSION}]) AC_MSG_RESULT([ LibGHT status: ${GHT_STATUS}]) +AC_MSG_RESULT([ LazPerf status: ${LAZPERF_STATUS}]) AC_MSG_RESULT([ CUnit status: ${CUNIT_STATUS}]) AC_MSG_RESULT() diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index d3909b7a..a255ef64 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -9,6 +9,7 @@ set ( PC_SOURCES pc_patch.c pc_patch_dimensional.c pc_patch_ght.c + pc_patch_lazperf.c pc_patch_uncompressed.c pc_point.c pc_pointlist.c @@ -18,6 +19,10 @@ set ( PC_SOURCES pc_val.c ) +set ( LAZPERF_SOURCES + lazperf_adapter.cpp + ) + set ( PC_HEADERS hashtable.h stringbuffer.h @@ -27,6 +32,7 @@ set ( PC_HEADERS ) add_library (libpc-static STATIC ${PC_SOURCES} ${PC_HEADERS}) +add_library (liblazperf-static STATIC ${LAZPERF_SOURCES}) set_target_properties (libpc-static PROPERTIES @@ -36,12 +42,23 @@ set_target_properties (libpc-static COMPILE_FLAGS -fPIC ) +set_target_properties (liblazperf-static + PROPERTIES + OUTPUT_NAME "lazperf" + PREFIX "lib" + CLEAN_DIRECT_OUTPUT 1 + COMPILE_FLAGS "-fPIC -std=c++0x" + ) + target_link_libraries (libpc-static xml2) target_link_libraries (libpc-static z) target_link_libraries (libpc-static m) if (LIBGHT_FOUND) target_link_libraries (libpc-static ght) endif (LIBGHT_FOUND) +if (LAZPERF_FOUND) + target_link_libraries (libpc-static liblazperf-static) +endif (LAZPERF_FOUND) if (WITH_TESTS) add_subdirectory (cunit) diff --git a/lib/Makefile b/lib/Makefile index 7e4e148b..3ef0d4c6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -21,16 +21,23 @@ OBJS = \ pc_util.o \ pc_val.o \ stringbuffer.o \ - hashtable.o + hashtable.o \ + pc_patch_lazperf.o -all: $(LIB_A) +OBJS_LAZPERF = \ + lazperf_adapter.o + +all: $(LIB_A) $(LIB_A_LAZPERF) $(MAKE) -C cunit $@ $(LIB_A): $(OBJS) ar rs $@ $^ +$(LIB_A_LAZPERF): $(OBJS_LAZPERF) + ar rs $@ $^ + clean: - @rm -f $(OBJS) $(LIB_A) + @rm -f $(OBJS) $(LIB_A) $(OBJS_LAZPERF) $(LIB_A_LAZPERF) $(MAKE) -C cunit $@ install: diff --git a/lib/cunit/CMakeLists.txt b/lib/cunit/CMakeLists.txt index 45dbc6e6..42a14c34 100644 --- a/lib/cunit/CMakeLists.txt +++ b/lib/cunit/CMakeLists.txt @@ -8,6 +8,7 @@ set (PC_TEST_SOURCES cu_pc_bytes.c cu_pc_schema.c cu_pc_patch.c + cu_pc_patch_lazperf.c cu_tester.c ) diff --git a/lib/cunit/Makefile b/lib/cunit/Makefile index 5bb6d260..226ab1bd 100644 --- a/lib/cunit/Makefile +++ b/lib/cunit/Makefile @@ -13,7 +13,8 @@ OBJS = \ cu_pc_schema.o \ cu_pc_point.o \ cu_pc_patch.o \ - cu_pc_patch_ght.o + cu_pc_patch_ght.o \ + cu_pc_patch_lazperf.o ifeq ($(CUNIT_LDFLAGS),) # No cunit? Emit message and continue @@ -37,12 +38,15 @@ check: $(EXE) endif # Build the main unit test executable -$(EXE): $(OBJS) ../$(LIB_A) - $(CC) -o $@ $^ $(LDFLAGS) -lm +$(EXE): $(OBJS) ../$(LIB_A) ../$(LIB_A_LAZPERF) + $(CC) -o $@ $^ $(LDFLAGS) -lm -lstdc++ ../$(LIB_A): $(MAKE) -C .. $(LIB_A) +../$(LIB_A_LAZPERF): + $(MAKE) -C .. $(LIB_A_LAZPERF) + # Clean target clean: @rm -f $(OBJS) diff --git a/lib/cunit/cu_pc_patch_lazperf.c b/lib/cunit/cu_pc_patch_lazperf.c new file mode 100644 index 00000000..5b9c28cf --- /dev/null +++ b/lib/cunit/cu_pc_patch_lazperf.c @@ -0,0 +1,264 @@ +/*********************************************************************** +* cu_pc_patch_lazperf.c +* +* Testing for the LazPerf API functions +* +* Copyright (c) 2016 Paul Blottiere, Oslandia +* +***********************************************************************/ + +#include "CUnit/Basic.h" +#include "cu_tester.h" + +/* GLOBALS ************************************************************/ + +static PCSCHEMA *simpleschema = NULL; +static const char *simplexmlfile = "data/simple-schema.xml"; + +/* Setup/teardown for this suite */ +static int +init_suite(void) +{ + char *xmlstr = file_to_str(simplexmlfile); + int rv = pc_schema_from_xml(xmlstr, &simpleschema); + pcfree(xmlstr); + if ( rv == PC_FAILURE ) return 1; + + return 0; +} + +static int +clean_suite(void) +{ + pc_schema_free(simpleschema); + return 0; +} + +static void +test_schema_compression_lazperf(void) +{ + PCSCHEMA *myschema = NULL; + char *myxmlfile = "data/simple-schema-laz.xml"; + char *xmlstr = file_to_str(myxmlfile); + int rv = pc_schema_from_xml(xmlstr, &myschema); + + CU_ASSERT(rv == PC_SUCCESS); + int compression = myschema->compression; + CU_ASSERT_EQUAL(compression, PC_LAZPERF); + + pc_schema_free(myschema); + pcfree(xmlstr); +} + +#ifdef HAVE_LAZPERF +static void +test_patch_lazperf() +{ + PCPOINT *pt; + int i; + int npts = 400; + PCPOINTLIST *pl; + PCPATCH_LAZPERF *pal; + PCPATCH_UNCOMPRESSED *paul, *pauref; + + // build a list of points + pl = pc_pointlist_make(npts); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_point_make(simpleschema); + pc_point_set_double_by_name(pt, "x", i*2.0); + pc_point_set_double_by_name(pt, "y", i*1.9); + pc_point_set_double_by_name(pt, "Z", i*0.34); + pc_point_set_double_by_name(pt, "intensity", 10); + pc_pointlist_add_point(pl, pt); + } + + // compress the list in a lazperf patch + pal = pc_patch_lazperf_from_pointlist( pl ); + + // get an uncompressed patch from the lazperf patch + paul = pc_patch_uncompressed_from_lazperf( pal ); + + // get an uncompressed patch directly from the pointlist + pauref = pc_patch_uncompressed_from_pointlist( pl ); + + // test the number of points + CU_ASSERT_EQUAL(pal->npoints, pauref->npoints); + CU_ASSERT_EQUAL(paul->npoints, pauref->npoints); + + // test bounds + CU_ASSERT_DOUBLE_EQUAL(pal->bounds.xmax, pauref->bounds.xmax, 0.0001); + CU_ASSERT_DOUBLE_EQUAL(paul->bounds.ymax, pauref->bounds.ymax, 0.000001); + + // test type + CU_ASSERT_EQUAL(pal->type, PC_LAZPERF); + CU_ASSERT_EQUAL(paul->type, pauref->type); + + // test readonly + CU_ASSERT_EQUAL(pauref->readonly, paul->readonly); + CU_ASSERT_EQUAL(pauref->readonly, pal->readonly); + + // free + pc_pointlist_free(pl); + pc_patch_free( (PCPATCH*) pal ); + pc_patch_free((PCPATCH*) paul); + pc_patch_free((PCPATCH*) pauref); +} + +static void +test_pointlist_lazperf() +{ + PCPOINT *pt; + int i; + int npts = 400; + PCPOINTLIST *pl1, *pl2; + PCPATCH_LAZPERF *pch1; + PCPATCH_UNCOMPRESSED *pa1, *pa2; + char *wkt1, *wkt2; + + // build a list of points + pl1 = pc_pointlist_make(npts); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_point_make(simpleschema); + pc_point_set_double_by_name(pt, "x", i*2.0); + pc_point_set_double_by_name(pt, "y", i*1.9); + pc_point_set_double_by_name(pt, "Z", i*0.34); + pc_point_set_double_by_name(pt, "intensity", 10); + pc_pointlist_add_point(pl1, pt); + } + + // compress the list in a lazperf patch + pch1 = pc_patch_lazperf_from_pointlist( pl1 ); + + // decompress the lazperf patch in a pointlist + pl2 = pc_pointlist_from_lazperf(pch1); + + // test that the string representation of pointlist is equal + pa1 = pc_patch_uncompressed_from_pointlist( pl1 ); + pa2 = pc_patch_uncompressed_from_lazperf( pch1 ); + + wkt1 = pc_patch_uncompressed_to_string(pa1); + wkt2 = pc_patch_uncompressed_to_string(pa2); + + CU_ASSERT_STRING_EQUAL(wkt1, wkt2); + + pc_patch_free((PCPATCH*) pch1 ); + pc_patch_free((PCPATCH*) pa1); + pc_patch_free((PCPATCH*) pa2); + pc_pointlist_free(pl1); + pc_pointlist_free(pl2); + pcfree(wkt1); + pcfree(wkt2); +} + +static void +test_to_string_lazperf() +{ + PCPOINT *pt; + int i; + int npts = 400; + PCPOINTLIST *pl; + PCPATCH_LAZPERF *pal; + PCPATCH_UNCOMPRESSED *pau; + char *str1, *str2; + + // build a list of points + pl = pc_pointlist_make(npts); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_point_make(simpleschema); + pc_point_set_double_by_name(pt, "x", i*2.0); + pc_point_set_double_by_name(pt, "y", i*1.9); + pc_point_set_double_by_name(pt, "Z", i*0.34); + pc_point_set_double_by_name(pt, "intensity", 10); + pc_pointlist_add_point(pl, pt); + } + + // build patch + pau = pc_patch_uncompressed_from_pointlist(pl); + pal = pc_patch_lazperf_from_pointlist(pl); + + // get string + str1 = pc_patch_uncompressed_to_string(pau); + str2 = pc_patch_lazperf_to_string(pal); + + // compare + CU_ASSERT_STRING_EQUAL(str1, str2); + + // free + pc_patch_free((PCPATCH*) pal); + pc_patch_free((PCPATCH*) pau); + pc_pointlist_free(pl); + pcfree(str1); + pcfree(str2); +} + +static void +test_wkb_lazperf() +{ + PCPOINT *pt; + int i; + int npts = 400; + PCPOINTLIST *pl; + PCPATCH_LAZPERF *pal1, *pal2; + PCPATCH_UNCOMPRESSED *pau; + uint8_t *wkb1, *wkb2; + size_t wkbsize; + + // build a list of points + pl = pc_pointlist_make(npts); + + for ( i = 0; i < npts; i++ ) + { + pt = pc_point_make(simpleschema); + pc_point_set_double_by_name(pt, "x", i*2.0); + pc_point_set_double_by_name(pt, "y", i*1.9); + pc_point_set_double_by_name(pt, "Z", i*0.34); + pc_point_set_double_by_name(pt, "intensity", 10); + pc_pointlist_add_point(pl, pt); + } + + // build patch lazperf + pal1 = pc_patch_lazperf_from_pointlist(pl); + + // get the corresponding wkb + wkb1 = pc_patch_lazperf_to_wkb(pal1, &wkbsize); + + // rebuild a lazperf patch thanks to the wkb + pal2 = (PCPATCH_LAZPERF*) pc_patch_lazperf_from_wkb( pal1->schema, wkb1, wkbsize); + + // get the wkb reference + pau = pc_patch_uncompressed_from_pointlist(pl); + wkb2 = pc_patch_uncompressed_to_wkb( pau, &wkbsize ); + + // compare wkb + CU_ASSERT_STRING_EQUAL(wkb1, wkb2); + + // free + pc_patch_free((PCPATCH*) pal1); + pc_patch_free((PCPATCH*) pal2); + pc_patch_free((PCPATCH*) pau); + pc_pointlist_free(pl); + pcfree(wkb1); + pcfree(wkb2); +} +#endif + +/* REGISTER ***********************************************************/ + +CU_TestInfo lazperf_tests[] = { +#ifdef HAVE_LAZPERF + PC_TEST(test_schema_compression_lazperf), + PC_TEST(test_patch_lazperf), + PC_TEST(test_pointlist_lazperf), + PC_TEST(test_to_string_lazperf), + PC_TEST(test_wkb_lazperf), +#endif + CU_TEST_INFO_NULL +}; + +CU_SuiteInfo lazperf_suite = {"lazperf", init_suite, clean_suite, lazperf_tests}; diff --git a/lib/cunit/cu_tester.c b/lib/cunit/cu_tester.c index b65a3c7d..3510b071 100644 --- a/lib/cunit/cu_tester.c +++ b/lib/cunit/cu_tester.c @@ -18,6 +18,7 @@ extern CU_SuiteInfo patch_suite; extern CU_SuiteInfo point_suite; extern CU_SuiteInfo ght_suite; extern CU_SuiteInfo bytes_suite; +extern CU_SuiteInfo lazperf_suite; /** * CUnit error handler @@ -52,6 +53,7 @@ int main(int argc, char *argv[]) point_suite, ght_suite, bytes_suite, + lazperf_suite, CU_SUITE_INFO_NULL }; diff --git a/lib/cunit/data/simple-schema-laz.xml b/lib/cunit/data/simple-schema-laz.xml new file mode 100644 index 00000000..0b0e6329 --- /dev/null +++ b/lib/cunit/data/simple-schema-laz.xml @@ -0,0 +1,38 @@ + + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + laz + + diff --git a/lib/lazperf_adapter.cpp b/lib/lazperf_adapter.cpp new file mode 100644 index 00000000..8f4b2725 --- /dev/null +++ b/lib/lazperf_adapter.cpp @@ -0,0 +1,260 @@ +/*********************************************************************** +* lazperf_adapter.cpp +* +* LazPerf compression/decompression +* +* Copyright (c) 2016 Paul Blottiere, Oslandia +* +***********************************************************************/ + +#include "lazperf_adapter.hpp" + +#ifdef HAVE_LAZPERF + +/********************************************************************** +* C API +*/ +size_t +lazperf_compress_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa, uint8_t **compressed) +{ + size_t size = 1; + + LazPerfBuf buf; + LazPerfCompressor engine(pa->schema, buf); + + if (engine.compress(pa->data, pa->datasize) == pa->npoints) + { + size = buf.buf.size(); + *compressed = (uint8_t*) malloc(size); + *compressed = (uint8_t*) memcpy(*compressed, buf.data(), size); + } + + // log + // lazperf_dump(pa); + // lazperf_dump(*compressed, size); + + return size; +} + +size_t +lazperf_uncompress_from_compressed(const PCPATCH_LAZPERF *pa, uint8_t **decompressed) +{ + size_t size = -1; + size_t datasize = pa->schema->size * pa->npoints; + + LazPerfBuf buf; + buf.putBytes(pa->lazperf, pa->lazperfsize); + LazPerfDecompressor engine(pa->schema, buf); + + *decompressed = (uint8_t*) malloc(datasize); + + if (engine.decompress(*decompressed, datasize) == pa->npoints) + size = buf.buf.size(); + + // log + // lazperf_dump(pa); + // lazperf_dump(*decompressed, datasize); + + return size; +} + +/********************************************************************** +* INTERNAL CPP +*/ +// utility functions +void +lazperf_dump(uint8_t *data, const size_t size) +{ + std::cout << "DUMP DATA: " << std::endl; + std::cout << " - datasize: " << size << std::endl; + std::cout << " - data: "; + for (int i = 0; i < size; ++i) + printf("%02x ", data[i]); + std::cout << std::endl; +} + +void +lazperf_dump(const PCPATCH_UNCOMPRESSED* p) +{ + std::cout << std::endl; + std::cout << "DUMP UNCOMPRESSED PATCH: " << std::endl; + std::cout << " - type: " << p->type << std::endl; + std::cout << " - schema->size " << p->schema->size << std::endl; + std::cout << " - readonly: " << p->readonly << std::endl; + std::cout << " - npoints: " << p->npoints << std::endl; + std::cout << " - maxpoints: " << p->maxpoints << std::endl; + std::cout << " - datasize: " << p->datasize << std::endl; + std::cout << " - data: "; + for (int i = 0; i < p->datasize; ++i) + printf("%02x ", p->data[i]); + std::cout << std::endl; +} + +void +lazperf_dump(const PCPATCH_LAZPERF* p) +{ + std::cout << std::endl; + std::cout << "DUMP LAZPERF PATCH: " << std::endl; + std::cout << " - type: " << p->type << std::endl; + std::cout << " - schema->size " << p->schema->size << std::endl; + std::cout << " - readonly: " << p->readonly << std::endl; + std::cout << " - npoints: " << p->npoints << std::endl; + std::cout << " - lazperfsize: " << p->lazperfsize << std::endl; + std::cout << " - lazperf: "; + for (int i = 0; i < p->lazperfsize; ++i) + printf("%02x ", p->lazperf[i]); + std::cout << std::endl; +} + +// LazPerf class +template +LazPerf::LazPerf(const PCSCHEMA *pcschema, LazPerfBuf &buf) + : _pcschema(pcschema) + , _coder(buf) + , _pointsize(0) +{ +} + +template +LazPerf::~LazPerf() +{ +} + +template +void +LazPerf::initSchema() +{ + for (int i = 0; i < _pcschema->ndims; i++) + addField(_pcschema->dims[i]); +} + +template +bool +LazPerf::addField(const PCDIMENSION *dim) +{ + bool rc = true; + + switch(dim->interpretation) + { + case PC_INT8: + { + _engine->template add_field(); + break; + } + case PC_UINT8: + { + _engine->template add_field(); + break; + } + case PC_INT16: + { + _engine->template add_field(); + break; + } + case PC_UINT16: + { + _engine->template add_field(); + break; + } + case PC_INT32: + { + _engine->template add_field(); + break; + } + case PC_UINT32: + { + _engine->template add_field(); + break; + } + case PC_INT64: + { + //_engine->template add_field(); + break; + } + case PC_UINT64: + { + //_engine->template add_field(); + break; + } + case PC_DOUBLE: + { + //_engine->template add_field(); + break; + } + case PC_FLOAT: + { + //_engine->template add_field(); + break; + } + case PC_UNKNOWN: + default: + rc = false; + } + + if (rc) + _pointsize += dim->size; + + return rc; +} + +// LazPerf Compressor +LazPerfCompressor::LazPerfCompressor(const PCSCHEMA *pcschema, LazPerfBuf &output) + : LazPerf(pcschema, output) +{ + _engine = laszip::formats::make_dynamic_compressor(_coder); + initSchema(); +} + +LazPerfCompressor::~LazPerfCompressor() +{ +} + +size_t +LazPerfCompressor::compress(const uint8_t *input, const size_t inputsize) +{ + size_t size = 0; + + const uint8_t *end = input + inputsize; + + while (input + _pointsize <= end) + { + _engine->compress((const char*) input); + input += _pointsize; + size++; + } + + _coder.done(); + + return size; +} + +// LazPerf Decompressor +LazPerfDecompressor::LazPerfDecompressor(const PCSCHEMA *pcschema, LazPerfBuf &input) + : LazPerf(pcschema, input) +{ + _engine = laszip::formats::make_dynamic_decompressor(_coder); + initSchema(); +} + +LazPerfDecompressor::~LazPerfDecompressor() +{ +} + +size_t +LazPerfDecompressor::decompress(uint8_t *output, const size_t outputsize) +{ + size_t size = 0; + + const uint8_t *end = output + outputsize; + + while (output + _pointsize <= end) + { + _engine->decompress((char*) output); + output += _pointsize; + size++; + } + + return size; +} + +#endif // HAVE_LAZPERF diff --git a/lib/lazperf_adapter.hpp b/lib/lazperf_adapter.hpp new file mode 100644 index 00000000..6a4c689f --- /dev/null +++ b/lib/lazperf_adapter.hpp @@ -0,0 +1,118 @@ +/*********************************************************************** +* lazperf_adapter.hpp +* +* LazPerf compression/decompression +* +* Copyright (c) 2016 Paul Blottiere, Oslandia +* +***********************************************************************/ + +#pragma once + +#include "pc_api_internal.h" + +#ifdef HAVE_LAZPERF +#include +#include +#include +#include +#include +#include +#include + +/********************************************************************** +* C API +*/ +extern "C" { + size_t lazperf_compress_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa, uint8_t **compressed); + + size_t lazperf_uncompress_from_compressed(const PCPATCH_LAZPERF *pa, uint8_t **decompressed); +} + +/********************************************************************** +* INTERNAL CPP +*/ +// utility functions +void lazperf_dump( uint8_t* data, const size_t size ); +void lazperf_dump( const PCPATCH_UNCOMPRESSED *p ); +void lazperf_dump( const PCPATCH_LAZPERF *p ); + +// struct which capture data coming from the compressor +struct LazPerfBuf { + LazPerfBuf() : buf(), idx(0) {} + + const uint8_t* data() { + return reinterpret_cast(buf.data()); + } + + void putBytes(const unsigned char* b, size_t len) { + while(len --) { + buf.push_back(*b++); + } + } + + void putByte(const unsigned char b) { + buf.push_back(b); + } + + unsigned char getByte() { + return buf[idx++]; + } + + void getBytes(unsigned char *b, int len) { + for (int i = 0 ; i < len ; i ++) { + b[i] = getByte(); + } + } + + std::vector buf; + size_t idx; +}; + +// some typedef +typedef laszip::encoders::arithmetic Encoder; +typedef laszip::decoders::arithmetic Decoder; + +typedef laszip::formats::dynamic_field_compressor::ptr Compressor; +typedef laszip::formats::dynamic_field_decompressor::ptr Decompressor; + +// LazPerf class +template +class LazPerf { + + public: + LazPerf( const PCSCHEMA *pcschema, LazPerfBuf &buf ); + ~LazPerf(); + + size_t pointsize() const { return _pointsize; } + + protected: + void initSchema(); + bool addField(const PCDIMENSION *dim); + + const PCSCHEMA *_pcschema; + LazPerfCoder _coder; + LazPerfEngine _engine; + size_t _pointsize; +}; + +// compressor +class LazPerfCompressor : public LazPerf { + + public: + LazPerfCompressor( const PCSCHEMA *pcschema, LazPerfBuf &output); + ~LazPerfCompressor(); + + size_t compress( const uint8_t *input, const size_t inputsize ); +}; + +// decompressor +class LazPerfDecompressor : public LazPerf { + + public: + LazPerfDecompressor( const PCSCHEMA *pcschema, LazPerfBuf &input ); + ~LazPerfDecompressor(); + + size_t decompress( uint8_t *data, const size_t datasize ); +}; +#endif // HAVE_LAZPERF diff --git a/lib/pc_api.h b/lib/pc_api.h index c1b84dce..13023a8f 100644 --- a/lib/pc_api.h +++ b/lib/pc_api.h @@ -36,7 +36,8 @@ enum COMPRESSIONS { PC_NONE = 0, PC_GHT = 1, - PC_DIMENSIONAL = 2 + PC_DIMENSIONAL = 2, + PC_LAZPERF = 3 }; /** @@ -196,6 +197,12 @@ typedef struct uint8_t *ght; } PCPATCH_GHT; +typedef struct +{ + PCPATCH_COMMON + size_t lazperfsize; + uint8_t *lazperf; +} PCPATCH_LAZPERF; /* Global function signatures for memory/logging handlers. */ diff --git a/lib/pc_api_internal.h b/lib/pc_api_internal.h index cd0c4748..22a67991 100644 --- a/lib/pc_api_internal.h +++ b/lib/pc_api_internal.h @@ -197,6 +197,16 @@ PCPOINTLIST* pc_pointlist_from_ght(const PCPATCH_GHT *pag); PCPATCH_GHT* pc_patch_ght_filter(const PCPATCH_GHT *patch, uint32_t dimnum, PC_FILTERTYPE filter, double val1, double val2); +/* LAZPERF PATCHES */ +PCPATCH_LAZPERF* pc_patch_lazperf_from_pointlist(const PCPOINTLIST *pl); +PCPATCH_LAZPERF* pc_patch_lazperf_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa); +PCPOINTLIST* pc_pointlist_from_lazperf(const PCPATCH_LAZPERF *palaz); +PCPATCH_UNCOMPRESSED* pc_patch_uncompressed_from_lazperf(const PCPATCH_LAZPERF *palaz); +char* pc_patch_lazperf_to_string(const PCPATCH_LAZPERF *pa); +void pc_patch_lazperf_free(PCPATCH_LAZPERF *palaz); +uint8_t* pc_patch_lazperf_to_wkb(const PCPATCH_LAZPERF *patch, size_t *wkbsize); +PCPATCH* pc_patch_lazperf_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize); + /**************************************************************************** * BYTES */ diff --git a/lib/pc_config.h.cmake b/lib/pc_config.h.cmake index f54f7af6..600a1aac 100644 --- a/lib/pc_config.h.cmake +++ b/lib/pc_config.h.cmake @@ -4,6 +4,8 @@ #cmakedefine HAVE_LIBGHT ${HAVE_LIBGHT} +#cmakedefine HAVE_LAZPERF ${HAVE_LAZPERF} + #cmakedefine HAVE_CUNIT ${HAVE_CUNIT} #cmakedefine PROJECT_SOURCE_DIR "${PROJECT_SOURCE_DIR}" diff --git a/lib/pc_config.h.in b/lib/pc_config.h.in index 61f6ceba..2298d523 100644 --- a/lib/pc_config.h.in +++ b/lib/pc_config.h.in @@ -4,6 +4,8 @@ #undef HAVE_LIBGHT +#undef HAVE_LAZPERF + #undef HAVE_CUNIT #undef PROJECT_SOURCE_DIR diff --git a/lib/pc_filter.c b/lib/pc_filter.c index ebfec7ff..ad6bff80 100644 --- a/lib/pc_filter.c +++ b/lib/pc_filter.c @@ -275,6 +275,28 @@ pc_patch_filter(const PCPATCH *pa, uint32_t dimnum, PC_FILTERTYPE filter, double paout = (PCPATCH*)pdl; break; } + case PC_LAZPERF: + { + PCBITMAP *map; + PCPATCH_UNCOMPRESSED *pu; + PCPATCH_UNCOMPRESSED *pau; + + pau = pc_patch_uncompressed_from_lazperf( (PCPATCH_LAZPERF*) pa ); + map = pc_patch_uncompressed_bitmap(pau, dimnum, filter, val1, val2); + if ( map->nset == 0 ) + { + pc_bitmap_free(map); + return (PCPATCH*)pc_patch_uncompressed_make(pa->schema, -1); + } + + pu = pc_patch_uncompressed_filter(pau, map); + pc_bitmap_free(map); + /* pc_patch_uncompressed_filter computes stats and bounds, so we're ready to return here */ + /* TODO, it could/should compute bounds and stats while filtering the points */ + paout = (PCPATCH*)pu; + + break; + } default: pcerror("%s: failure", __func__); return NULL; diff --git a/lib/pc_patch.c b/lib/pc_patch.c index 780bb588..0f0b80f5 100644 --- a/lib/pc_patch.c +++ b/lib/pc_patch.c @@ -26,6 +26,8 @@ pc_patch_compute_extent(PCPATCH *pa) return pc_patch_ght_compute_extent((PCPATCH_GHT*)pa); case PC_DIMENSIONAL: return pc_patch_dimensional_compute_extent((PCPATCH_DIMENSIONAL*)pa); + case PC_LAZPERF: + return pc_patch_lazperf_compute_extent((PCPATCH_DIMENSIONAL*)pa); } return PC_FAILURE; } @@ -59,6 +61,14 @@ pc_patch_compute_stats(PCPATCH *pa) pc_patch_uncompressed_free(pu); return PC_SUCCESS; } + case PC_LAZPERF: + { + PCPATCH_UNCOMPRESSED *pu = pc_patch_uncompressed_from_lazperf((PCPATCH_LAZPERF*)pa); + pc_patch_uncompressed_compute_stats(pu); + pa->stats = pc_stats_clone(pu->stats); + pc_patch_uncompressed_free(pu); + return PC_SUCCESS; + } default: { pcerror("%s: unknown compression type", __func__, pa->type); @@ -95,6 +105,11 @@ pc_patch_free(PCPATCH *patch) pc_patch_dimensional_free((PCPATCH_DIMENSIONAL*)patch); break; } + case PC_LAZPERF: + { + pc_patch_lazperf_free((PCPATCH_LAZPERF*)patch); + break; + } default: { pcerror("%s: unknown compression type %d", __func__, patch->type); @@ -143,6 +158,14 @@ pc_patch_compress(const PCPATCH *patch, void *userdata) pc_patch_dimensional_free(pcdu); return (PCPATCH*)pcdc; } + else if ( patch_compression == PC_LAZPERF ) + { + PCPATCH_UNCOMPRESSED *pcu = pc_patch_uncompressed_from_lazperf( (PCPATCH_LAZPERF*) patch ); + PCPATCH_DIMENSIONAL *pal = pc_patch_dimensional_from_uncompressed( pcu ); + PCPATCH_DIMENSIONAL *palc = pc_patch_dimensional_compress( pal, NULL ); + pc_patch_dimensional_free(pal); + return (PCPATCH*) palc; + } else { pcerror("%s: unknown patch compression type %d", __func__, patch_compression); @@ -165,6 +188,11 @@ pc_patch_compress(const PCPATCH *patch, void *userdata) PCPATCH_UNCOMPRESSED *pcu = pc_patch_uncompressed_from_ght((PCPATCH_GHT*)patch); return (PCPATCH*)pcu; } + else if ( patch_compression == PC_LAZPERF ) + { + PCPATCH_UNCOMPRESSED *pcu = pc_patch_uncompressed_from_lazperf( (PCPATCH_LAZPERF*)patch ); + return (PCPATCH*)pcu; + } else { pcerror("%s: unknown patch compression type %d", __func__, patch_compression); @@ -189,6 +217,42 @@ pc_patch_compress(const PCPATCH *patch, void *userdata) { return (PCPATCH*)patch; } + else if ( patch_compression == PC_LAZPERF ) + { + PCPATCH_LAZPERF *pal = pc_patch_lazperf_from_uncompressed((PCPATCH_UNCOMPRESSED*)patch); + return (PCPATCH*)pal; + } + else + { + pcerror("%s: unknown patch compression type %d", __func__, patch_compression); + } + } + case PC_LAZPERF: + { + if ( patch_compression == PC_NONE ) + { + PCPATCH_LAZPERF *pgc = pc_patch_lazperf_from_uncompressed((PCPATCH_UNCOMPRESSED*)patch); + if ( ! pgc ) pcerror("%s: lazperf compression failed", __func__); + return (PCPATCH*)pgc; + } + else if ( patch_compression == PC_DIMENSIONAL ) + { + PCPATCH_UNCOMPRESSED *pad = pc_patch_uncompressed_from_dimensional((PCPATCH_DIMENSIONAL*)patch); + PCPATCH_LAZPERF *pal = pc_patch_lazperf_from_uncompressed( (PCPATCH_UNCOMPRESSED*) pad ); + pc_patch_uncompressed_free( pad ); + return (PCPATCH*)pal; + } + else if ( patch_compression == PC_GHT ) + { + PCPATCH_UNCOMPRESSED *pcu = pc_patch_uncompressed_from_ght((PCPATCH_GHT*)patch); + PCPATCH_LAZPERF *pal = pc_patch_lazperf_from_uncompressed((PCPATCH_UNCOMPRESSED*)patch); + pc_patch_uncompressed_free(pcu); + return (PCPATCH*)pcu; + } + else if ( patch_compression == PC_LAZPERF ) + { + return (PCPATCH*)patch; + } else { pcerror("%s: unknown patch compression type %d", __func__, patch_compression); @@ -227,6 +291,12 @@ pc_patch_uncompress(const PCPATCH *patch) return (PCPATCH*)pu; } + if ( patch_compression == PC_LAZPERF ) + { + PCPATCH_UNCOMPRESSED *pu = pc_patch_uncompressed_from_lazperf( (PCPATCH_LAZPERF*)patch ); + return (PCPATCH*) pu; + } + return NULL; } @@ -279,6 +349,11 @@ pc_patch_from_wkb(const PCSCHEMA *s, uint8_t *wkb, size_t wkbsize) patch = pc_patch_ght_from_wkb(s, wkb, wkbsize); break; } + case PC_LAZPERF: + { + patch = pc_patch_lazperf_from_wkb(s, wkb, wkbsize); + break; + } default: { /* Don't get here */ @@ -322,6 +397,10 @@ pc_patch_to_wkb(const PCPATCH *patch, size_t *wkbsize) { return pc_patch_ght_to_wkb((PCPATCH_GHT*)patch, wkbsize); } + case PC_LAZPERF: + { + return pc_patch_lazperf_to_wkb((PCPATCH_LAZPERF*)patch, wkbsize); + } } pcerror("%s: unknown compression requested '%d'", __func__, patch->schema->compression); return NULL; @@ -338,6 +417,8 @@ pc_patch_to_string(const PCPATCH *patch) return pc_patch_dimensional_to_string((PCPATCH_DIMENSIONAL*)patch); case PC_GHT: return pc_patch_ght_to_string((PCPATCH_GHT*)patch); + case PC_LAZPERF: + return pc_patch_lazperf_to_string( (PCPATCH_LAZPERF*)patch ); } pcerror("%s: unsupported compression %d requested", __func__, patch->type); return NULL; @@ -413,6 +494,15 @@ pc_patch_from_patchlist(PCPATCH **palist, int numpatches) buf += sz; break; } + case PC_LAZPERF: + { + PCPATCH_UNCOMPRESSED *pu = pc_patch_uncompressed_from_lazperf((const PCPATCH_LAZPERF*)pa); + size_t sz = pu->schema->size * pu->npoints; + memcpy(buf, pu->data, sz); + buf += sz; + pc_patch_uncompressed_free(pu); + break; + } default: { pcerror("%s: unknown compression type (%d)", __func__, pa->type); diff --git a/lib/pc_patch_lazperf.c b/lib/pc_patch_lazperf.c new file mode 100644 index 00000000..075d720f --- /dev/null +++ b/lib/pc_patch_lazperf.c @@ -0,0 +1,233 @@ +/*********************************************************************** +* pc_patch_lazperf.c +* +* PgSQL Pointcloud is free and open source software provided +* by the Government of Canada +* +* Copyright (c) 2016 Paul Blottiere, Oslandia +* +***********************************************************************/ + +#include "pc_api_internal.h" +#include + +void +pc_patch_lazperf_free(PCPATCH_LAZPERF *pal) +{ + assert(pal); + pcfree(pal->lazperf); + pcfree(pal); +} + +PCPATCH_LAZPERF * +pc_patch_lazperf_from_pointlist(const PCPOINTLIST *pdl) +{ + PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_pointlist(pdl); + PCPATCH_LAZPERF *lazperfpatch = pc_patch_lazperf_from_uncompressed(patch); + pc_patch_free((PCPATCH*) patch); + + return lazperfpatch; +} + +PCPATCH_LAZPERF* +pc_patch_lazperf_from_uncompressed(const PCPATCH_UNCOMPRESSED *pa) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#endif + + PCPATCH_LAZPERF *palaz = NULL; + uint8_t *compressed; + + // cpp call to get compressed data from pcpatch + size_t compressSize = lazperf_compress_from_uncompressed(pa, &compressed); + + if (compressSize != -1) + { + palaz = pcalloc(sizeof(PCPATCH_LAZPERF)); + palaz->type = PC_LAZPERF; + palaz->readonly = PC_FALSE; + palaz->schema = pa->schema; + + // not optimal but we have to pass by the context manager otherwise + // a segfault happenned (sometimes) during a pcfree of lazperf field + palaz->lazperf = (uint8_t*) pcalloc(compressSize); + memcpy(palaz->lazperf, compressed, compressSize); + free(compressed); + + palaz->npoints = pa->npoints; + palaz->bounds = pa->bounds; + palaz->stats = pc_stats_clone(pa->stats); + palaz->lazperfsize = compressSize; + } + else + pcerror("%s: LAZ compressionf failed", __func__); + + return palaz; +} + +PCPOINTLIST * +pc_pointlist_from_lazperf(const PCPATCH_LAZPERF *palaz) +{ + PCPATCH_UNCOMPRESSED *pu = NULL; + pu = pc_patch_uncompressed_from_lazperf(palaz); + PCPOINTLIST *pl = pc_pointlist_from_uncompressed(pu); + pc_patch_free((PCPATCH *)pu); + return pl; +} + +PCPATCH_UNCOMPRESSED* +pc_patch_uncompressed_from_lazperf(const PCPATCH_LAZPERF *palaz) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#endif + + PCPATCH_UNCOMPRESSED *pcu = NULL; + uint8_t *decompressed; + int i; + + // cpp call to uncompressed data + size_t size = lazperf_uncompress_from_compressed(palaz, &decompressed); + + if (size != -1) + { + pcu = pcalloc(sizeof(PCPATCH_UNCOMPRESSED)); + pcu->type = PC_NONE; + pcu->readonly = PC_FALSE; + pcu->schema = palaz->schema; + pcu->npoints = palaz->npoints; + pcu->bounds = palaz->bounds; + pcu->stats = pc_stats_clone(palaz->stats); + + // not optimal but we have to pass by the context manager otherwise + // a segfault happenned (sometimes) during a pcfree of lazperf field + pcu->data = (uint8_t*) pcalloc(palaz->schema->size * palaz->npoints); + memcpy(pcu->data, decompressed, palaz->schema->size * palaz->npoints); + free(decompressed); + + pcu->maxpoints = palaz->npoints; + } + else + pcerror("%s: lazperf uncompression failed", __func__); + + return pcu; +} + +char * +pc_patch_lazperf_to_string(const PCPATCH_LAZPERF *pa) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#endif + PCPATCH_UNCOMPRESSED *patch = pc_patch_uncompressed_from_lazperf(pa); + char *str = pc_patch_uncompressed_to_string(patch); + pc_patch_free((PCPATCH*) patch); + return str; +} + +uint8_t * +pc_patch_lazperf_to_wkb(const PCPATCH_LAZPERF *patch, size_t *wkbsize) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#else + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression + uint32: npoints + uint32: lazperfsize + uint8[]: lazperfbuffer + */ + + uint8_t *buf; + char endian = machine_endian(); + /* endian + pcid + compression + npoints + lazperfsize + lazperf */ + size_t size = 1 + 4 + 4 + 4 + 4 + patch->lazperfsize; + + uint8_t *wkb = pcalloc(size); + uint32_t compression = patch->type; + uint32_t npoints = patch->npoints; + uint32_t pcid = patch->schema->pcid; + uint32_t lazperfsize = patch->lazperfsize; + wkb[0] = endian; /* Write endian flag */ + memcpy(wkb + 1, &pcid, 4); /* Write PCID */ + memcpy(wkb + 5, &compression, 4); /* Write compression */ + memcpy(wkb + 9, &npoints, 4); /* Write npoints */ + memcpy(wkb + 13, &lazperfsize, 4); /* Write lazperf buffer size */ + + buf = wkb + 17; + memcpy(buf, patch->lazperf, patch->lazperfsize); + if (wkbsize) + *wkbsize = size; + + return wkb; +#endif +} + +PCPATCH * +pc_patch_lazperf_from_wkb(const PCSCHEMA *schema, const uint8_t *wkb, size_t wkbsize) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#else + /* + byte: endianness (1 = NDR, 0 = XDR) + uint32: pcid (key to POINTCLOUD_SCHEMAS) + uint32: compression (0 = no compression, 1 = dimensional, 2 = GHT) + uint32: npoints + uint32: lazperfsize + uint8[]: lazerperfbuffer + */ + static size_t hdrsz = 1+4+4+4; /* endian + pcid + compression + npoints */ + PCPATCH_LAZPERF *patch; + uint8_t swap_endian = (wkb[0] != machine_endian()); + uint32_t npoints; + size_t datasize, lazperfsize; + const uint8_t *buf; + + if (wkb_get_compression(wkb) != PC_LAZPERF) + { + pcerror("%s: call with wkb that is not LAZPERF compressed", __func__); + return NULL; + } + + npoints = wkb_get_npoints(wkb); + + patch = pcalloc(sizeof(PCPATCH_LAZPERF)); + patch->type = PC_LAZPERF; + patch->readonly = PC_FALSE; + patch->schema = schema; + patch->npoints = npoints; + + /* Start on the LAZPERF */ + buf = wkb+hdrsz; + lazperfsize = wkb_get_int32(buf, swap_endian); + buf += 4; + + /* Copy in the tree buffer */ + patch->lazperfsize = lazperfsize; + patch->lazperf = pcalloc(lazperfsize); + memcpy(patch->lazperf, buf, lazperfsize); + + return (PCPATCH*)patch; +#endif +} + +int +pc_patch_lazperf_compute_extent(PCPATCH_LAZPERF *patch) +{ +#ifndef HAVE_LAZPERF + pcerror("%s: lazperf support is not enabled", __func__); + return NULL; +#endif + + PCPATCH_UNCOMPRESSED *pau = pc_patch_uncompressed_from_lazperf(patch); + return pc_patch_uncompressed_compute_extent(pau); +} diff --git a/lib/pc_pointlist.c b/lib/pc_pointlist.c index 70a750e0..4b01a566 100644 --- a/lib/pc_pointlist.c +++ b/lib/pc_pointlist.c @@ -124,6 +124,10 @@ pc_pointlist_from_patch(const PCPATCH *patch) { return pc_pointlist_from_dimensional((PCPATCH_DIMENSIONAL*)patch); } + case PC_LAZPERF: + { + return pc_pointlist_from_lazperf((PCPATCH_LAZPERF*)patch); + } } /* Don't get here */ diff --git a/lib/pc_schema.c b/lib/pc_schema.c index 0e0551b0..2384f2bc 100644 --- a/lib/pc_schema.c +++ b/lib/pc_schema.c @@ -100,6 +100,8 @@ pc_compression_name(int num) return "ght"; case PC_DIMENSIONAL: return "dimensional"; + case PC_LAZPERF: + return "laz"; default: return "UNKNOWN"; } @@ -117,6 +119,12 @@ pc_compression_number(const char *str) return PC_DIMENSIONAL; } + if ( (str[0] == 'l' || str[0] == 'L') && + (strcasecmp(str, "laz") == 0) ) + { + return PC_LAZPERF; + } + if ( (str[0] == 'g' || str[0] == 'G') && (strcasecmp(str, "ght") == 0) ) { diff --git a/pgsql/Makefile b/pgsql/Makefile index 203d7d55..60035a21 100644 --- a/pgsql/Makefile +++ b/pgsql/Makefile @@ -33,9 +33,13 @@ ifneq ($(GHT_LDFLAGS),) REGRESS += pointcloud-ght endif +ifneq ($(LAZPERF_STATUS), "disabled") +REGRESS += pointcloud-laz +endif + # Add in build/link flags for lib PG_CPPFLAGS += -I../lib -SHLIB_LINK += ../lib/$(LIB_A) $(filter -lm, $(LIBS)) $(XML2_LDFLAGS) $(ZLIB_LDFLAGS) $(GHT_LDFLAGS) +SHLIB_LINK += ../lib/$(LIB_A) ../lib/$(LIB_A_LAZPERF) -lstdc++ $(filter -lm, $(LIBS)) $(XML2_LDFLAGS) $(ZLIB_LDFLAGS) $(GHT_LDFLAGS) # We are going to use PGXS for sure include $(PGXS) diff --git a/pgsql/expected/pointcloud-laz.out b/pgsql/expected/pointcloud-laz.out new file mode 100644 index 00000000..ccc9bece --- /dev/null +++ b/pgsql/expected/pointcloud-laz.out @@ -0,0 +1,276 @@ +INSERT INTO pointcloud_formats (pcid, srid, schema) +VALUES (5, 0, +' + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + laz + +' +); +CREATE TABLE IF NOT EXISTS pa_test_laz ( + pa PCPATCH(5) +); +\d pa_test_laz + Table "public.pa_test_laz" + Column | Type | Modifiers +--------+------------+----------- + pa | pcpatch(5) | + +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050008'); +INSERT INTO pa_test_laz (pa) VALUES ('000000000500000000000000020000000600000007000000050006000000090000000A00000005000A'); +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050003'); +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050001'); +SELECT pc_explode(pa) FROM pa_test_laz; + pc_explode +---------------------------------------- + 01050000000200000003000000050000000600 + 01050000000200000003000000050000000800 + 01050000000600000007000000050000000600 + 0105000000090000000A000000050000000A00 + 01050000000200000003000000050000000600 + 01050000000200000003000000050000000300 + 01050000000200000003000000050000000600 + 01050000000200000003000000050000000100 +(8 rows) + +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz; + pc_astext +------------------------------------ + {"pcid":5,"pt":[0.02,0.03,0.05,6]} + {"pcid":5,"pt":[0.02,0.03,0.05,8]} + {"pcid":5,"pt":[0.06,0.07,0.05,6]} + {"pcid":5,"pt":[0.09,0.1,0.05,10]} + {"pcid":5,"pt":[0.02,0.03,0.05,6]} + {"pcid":5,"pt":[0.02,0.03,0.05,3]} + {"pcid":5,"pt":[0.02,0.03,0.05,6]} + {"pcid":5,"pt":[0.02,0.03,0.05,1]} +(8 rows) + +SELECT * FROM pa_test_laz; + pa +------------------------------------------------------------------------------ + 01050000000300000002000000140000000200000003000000050000000600000005A10000 + 0105000000030000000200000015000000060000000700000005000000060013884A3A000000 + 01050000000300000002000000150000000200000003000000050000000600000006D8000000 + 0105000000030000000200000015000000020000000300000005000000060000000B1E000000 +(4 rows) + +SELECT Sum(PC_NumPoints(pa)) FROM pa_test_laz; + sum +----- + 8 +(1 row) + +SELECT Sum(PC_MemSize(pa)) FROM pa_test_laz; + sum +----- + 487 +(1 row) + +SELECT Sum(PC_PatchMax(pa,'x')) FROM pa_test_laz; + sum +------ + 0.15 +(1 row) + +SELECT Sum(PC_PatchMin(pa,'x')) FROM pa_test_laz; + sum +------ + 0.12 +(1 row) + +DELETE FROM pa_test_laz; +INSERT INTO pa_test_laz (pa) +SELECT PC_Patch(PC_MakePoint(5, ARRAY[x,y,z,intensity])) +FROM ( + SELECT + -127+a/100.0 AS x, + 45+a/100.0 AS y, + 1.0*a AS z, + a/10 AS intensity, + a/400 AS gid + FROM generate_series(1,1600) AS a +) AS values GROUP BY gid; +SELECT pc_explode(pa) FROM pa_test_laz LIMIT 20; + pc_explode +---------------------------------------- + 0105000000A4D4FFFFD417000000710200A000 + 0105000000F4CFFFFF24130000409C00002800 + 0105000000F5CFFFFF25130000A49C00002800 + 0105000000F6CFFFFF26130000089D00002800 + 0105000000F7CFFFFF271300006C9D00002800 + 0105000000F8CFFFFF28130000D09D00002800 + 0105000000F9CFFFFF29130000349E00002800 + 0105000000FACFFFFF2A130000989E00002800 + 0105000000FBCFFFFF2B130000FC9E00002800 + 0105000000FCCFFFFF2C130000609F00002800 + 0105000000FDCFFFFF2D130000C49F00002800 + 0105000000FECFFFFF2E13000028A000002900 + 0105000000FFCFFFFF2F1300008CA000002900 + 010500000000D0FFFF30130000F0A000002900 + 010500000001D0FFFF3113000054A100002900 + 010500000002D0FFFF32130000B8A100002900 + 010500000003D0FFFF331300001CA200002900 + 010500000004D0FFFF3413000080A200002900 + 010500000005D0FFFF35130000E4A200002900 + 010500000006D0FFFF3613000048A300002900 +(20 rows) + +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz LIMIT 20; + pc_astext +---------------------------------------- + {"pcid":5,"pt":[-111,61,1600,160]} + {"pcid":5,"pt":[-123,49,400,40]} + {"pcid":5,"pt":[-122.99,49.01,401,40]} + {"pcid":5,"pt":[-122.98,49.02,402,40]} + {"pcid":5,"pt":[-122.97,49.03,403,40]} + {"pcid":5,"pt":[-122.96,49.04,404,40]} + {"pcid":5,"pt":[-122.95,49.05,405,40]} + {"pcid":5,"pt":[-122.94,49.06,406,40]} + {"pcid":5,"pt":[-122.93,49.07,407,40]} + {"pcid":5,"pt":[-122.92,49.08,408,40]} + {"pcid":5,"pt":[-122.91,49.09,409,40]} + {"pcid":5,"pt":[-122.9,49.1,410,41]} + {"pcid":5,"pt":[-122.89,49.11,411,41]} + {"pcid":5,"pt":[-122.88,49.12,412,41]} + {"pcid":5,"pt":[-122.87,49.13,413,41]} + {"pcid":5,"pt":[-122.86,49.14,414,41]} + {"pcid":5,"pt":[-122.85,49.15,415,41]} + {"pcid":5,"pt":[-122.84,49.16,416,41]} + {"pcid":5,"pt":[-122.83,49.17,417,41]} + {"pcid":5,"pt":[-122.82,49.18,418,41]} +(20 rows) + +SELECT * FROM pa_test_laz LIMIT 20; + pa +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0105000000030000000100000012000000A4D4FFFFD417000000710200A00001000000 + 01050000000300000090010000F4000000F4CFFFFF24130000409C0000280003F27D65F909E1B4FF163125F3217D67B1A6E9728B7246DB622DAD94954405C57B953BE39649652F560A1AC414A371CE3967FBC090345EE0B9E0B3E7B024BBE31373EAA273E1696F7D46E70DCAB367103ECFDA0DF36E4F77901A2C8CE2F4718BDF6680076FFB02B320FA9ED30DFF79A17102E4E48D349A3098EC43031CE4E6B294D0A4442354A6626BCDEF20DF2C154144DA5FD99CE0DADBF4C7153ED51B4D43A3A5A0DBF1FBA5CA5C6C9630C1CA7662A002A217E22CF7357FA19EFCBA6F6C7E91035176B2DF753BD5954A923FA6A95C848053BCE36381D7EA15D928A88612B1E49BD6000000 + 01050000000300000090010000F400000014D3FFFF44160000C0D40100780003F27D65F909E1B4FF163125F3217D67B1A6E9728B7246DB622DAD94954405C57B953BE39649652F560A1AC414A371CE3967FBC090345EE0B9E0B3E7B024BBE31373EAA273E1696F7D46E70DCAB367103ECFDA0DF36E4F77901A2C8CE2F4718BDF6680076FFB02B320FA9ED30DFF79A17102E4E48D349A3098EC43031CE4E6B294D0A4442354A6626BCDEF20DF2C154144DA5FD99CE0DADBF4C7153ED51B4D43A3A5A0DBF1FBA5CA5C6C9630C1CA7662A002A217E22CF7357FA19EFCBA6F6C7E91035176B2DF753BD5954A923FA6A95C848053BCE36381D7EA15D928A88612B1E49BD6000000 + 0105000000030000008F010000F400000065CEFFFF9511000064000000000003F27D65F909E1B4FF163125F3217D67B1A6E9728B7246DB622DAD94954405D5CF50EBF47F3F94647B9C1238B6EFFF7200D0A51A36EC28ABD9AAAB0C6351B91042FA4DFB9A4C97A00E088A4033DAF7FC2FC798FFDEF31FC02D5C881C2A92086291FDC6832A9E966831DC62112CFBE18094BF4B398D29C9A29EB9868F208DBB735025F1EAA03FD8D064FB7B012CD72D8E6CC5E82B221E17B55000071A6B70136AAB145FB0F48BE8ADB1D33E88E13D7DE876B582245BC78DB10A9CB31673C349380CF301873FF20D65406448D9889258A582908A6D2E85B58918B4B98E50BC5ADB5B11DB000000 + 01050000000300000090010000F400000084D1FFFFB414000080380100500003F27D65F909E1B4FF163125F3217D67B1A6E9728B7246DB622DAD94954405C57B953BE39649652F560A1AC414A371CE3967FBC090345EE0B9E0B3E7B024BBE31373EAA273E1696F7D46E70DCAB367103ECFDA0DF36E4F77901A2C8CE2F4718BDF6680076FFB02B320FA9ED30DFF79A17102E4E48D349A3098EC43031CE4E6B294D0A4442354A6626BCDEF20DF2C154144DA5FD99CE0DADBF4C7153ED51B4D43A3A5A0DBF1FBA5CA5C6C9630C1CA7662A002A217E22CF7357FA19EFCBA6F6C7E91035176B2DF753BD5954A923FA6A95C848053BCE36381D7EA15D928A88612B1E49BD6000000 +(5 rows) + +SELECT Sum(PC_NumPoints(pa)) FROM pa_test_laz; + sum +------ + 1600 +(1 row) + +SELECT Sum(PC_MemSize(pa)) FROM pa_test_laz; + sum +------ + 1499 +(1 row) + +SELECT Max(PC_PatchMax(pa,'x')) FROM pa_test_laz; + max +------ + -111 +(1 row) + +SELECT Min(PC_PatchMin(pa,'x')) FROM pa_test_laz; + min +--------- + -126.99 +(1 row) + +SELECT Min(PC_PatchMin(pa,'z')) FROM pa_test_laz; + min +----- + 1 +(1 row) + +SELECT pc_astext(PC_FilterLessThan(pa, 'z', 5)) FROM pa_test_laz; + pc_astext +---------------------------------------------------------------------------------------------------- + + + + {"pcid":5,"pts":[[-126.99,45.01,1,0],[-126.98,45.02,2,0],[-126.97,45.03,3,0],[-126.96,45.04,4,0]]} + +(5 rows) + +SELECT pc_astext(PC_FilterGreaterThan(pa, 'z', 1595)) FROM pa_test_laz; + pc_astext +------------------------------------------------------------------------------------------------------------------------ + {"pcid":5,"pts":[[-111,61,1600,160]]} + + {"pcid":5,"pts":[[-111.04,60.96,1596,159],[-111.03,60.97,1597,159],[-111.02,60.98,1598,159],[-111.01,60.99,1599,159]]} + + +(5 rows) + +SELECT pc_astext(PC_FilterEquals(pa, 'z', 500)) FROM pa_test_laz; + pc_astext +------------------------------------- + + {"pcid":5,"pts":[[-122,50,500,50]]} + + + +(5 rows) + +SELECT pc_astext(PC_FilterBetween(pa, 'z', 500, 505)) FROM pa_test_laz; + pc_astext +---------------------------------------------------------------------------------------------------------------- + + {"pcid":5,"pts":[[-121.99,50.01,501,50],[-121.98,50.02,502,50],[-121.97,50.03,503,50],[-121.96,50.04,504,50]]} + + + +(5 rows) + +DELETE FROM pa_test_laz; +INSERT INTO pa_test_laz( pa ) VALUES ('01050000000300000004000000210000000000000000000000000000000a004417593a34c1c5f74f83179fc2448960000000'); +SELECT pc_explode(pa) FROM pa_test_laz; + pc_explode +---------------------------------------- + 01050000000000000000000000000000000A00 + 0105000000C8000000BE000000220000000A00 + 0105000000900100007C010000440000000A00 + 0105000000580200003A020000660000000A00 +(4 rows) + +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz; + pc_astext +--------------------------------- + {"pcid":5,"pt":[0,0,0,10]} + {"pcid":5,"pt":[2,1.9,0.34,10]} + {"pcid":5,"pt":[4,3.8,0.68,10]} + {"pcid":5,"pt":[6,5.7,1.02,10]} +(4 rows) + +TRUNCATE pointcloud_formats; diff --git a/pgsql/expected/pointcloud.out b/pgsql/expected/pointcloud.out index bd0a06ce..4fa56e28 100644 --- a/pgsql/expected/pointcloud.out +++ b/pgsql/expected/pointcloud.out @@ -470,7 +470,8 @@ FROM p1, ( values ('dimensional','rle'), ('dimensional','zlib'), ('dimensional','sigbits'), - ('dimensional','auto') + ('dimensional','auto'), + ('laz','null') -- ,('ght',null) -- fails due to https://github.com/pgpointcloud/pointcloud/issues/35 ) dimcompr(compr,sc) ORDER BY compr,sc,v; @@ -536,7 +537,22 @@ ORDER BY compr,sc,v; compr | 5 | dimensional | zlib | t compr | 6 | dimensional | zlib | t compr | 7 | dimensional | zlib | t -(60 rows) + compr | -7 | laz | null | f + compr | -6 | laz | null | f + compr | -5 | laz | null | f + compr | -4 | laz | null | f + compr | -3 | laz | null | f + compr | -2 | laz | null | f + compr | -1 | laz | null | f + compr | 0 | laz | null | f + compr | 1 | laz | null | f + compr | 2 | laz | null | f + compr | 3 | laz | null | f + compr | 4 | laz | null | f + compr | 5 | laz | null | f + compr | 6 | laz | null | f + compr | 7 | laz | null | f +(75 rows) SELECT PC_Summary(PC_Compress(PC_Patch(PC_MakePoint(10,ARRAY[1,1,1,1,1,1,1])), 'dimensional'))::json->'compr'; diff --git a/pgsql/pc_access.c b/pgsql/pc_access.c index 1ac26b84..0fce3e5c 100644 --- a/pgsql/pc_access.c +++ b/pgsql/pc_access.c @@ -625,6 +625,9 @@ Datum pcpatch_compress(PG_FUNCTION_ARGS) else if ( strcmp(compr_in, "ght") == 0 ) { schema->compression = PC_GHT; } + else if ( strcmp(compr_in, "laz") == 0 ) { + schema->compression = PC_LAZPERF; + } else { elog(ERROR, "Unrecognized compression '%s'. Please specify 'auto','dimensional' or 'ght'", compr_in); } diff --git a/pgsql/pc_pgsql.c b/pgsql/pc_pgsql.c index 44a749f0..4aed31ec 100644 --- a/pgsql/pc_pgsql.c +++ b/pgsql/pc_pgsql.c @@ -466,6 +466,12 @@ pc_patch_serialized_size(const PCPATCH *patch) { return common_size + stats_size + pc_patch_dimensional_serialized_size((PCPATCH_DIMENSIONAL*)patch); } + case PC_LAZPERF: + { + static size_t lazsize_size = 4; + PCPATCH_LAZPERF *pg = (PCPATCH_LAZPERF*)patch; + return common_size + stats_size + lazsize_size + pg->lazperfsize; + } default: { pcerror("%s: unknown compresed %d", __func__, patch->type); @@ -606,6 +612,45 @@ pc_patch_ght_serialize(const PCPATCH *patch_in) return serpch; } +static SERIALIZED_PATCH * +pc_patch_lazperf_serialize(const PCPATCH *patch_in) +{ + size_t serpch_size = pc_patch_serialized_size(patch_in); + SERIALIZED_PATCH *serpch = pcalloc(serpch_size); + const PCPATCH_LAZPERF *patch = (PCPATCH_LAZPERF*)patch_in; + uint32_t lazsize = patch->lazperfsize; + uint8_t *buf = serpch->data; + + assert(patch); + assert(patch->type == PC_LAZPERF); + + /* Copy basics */ + serpch->pcid = patch->schema->pcid; + serpch->npoints = patch->npoints; + serpch->bounds = patch->bounds; + serpch->compression = patch->type; + + /* Write stats into the buffer first */ + if ( patch->stats ) + { + buf += pc_patch_stats_serialize(buf, patch->schema, patch->stats); + } + else + { + pcerror("%s: stats missing!", __func__); + } + + /* Write buffer size */ + memcpy(buf, &(lazsize), 4); + buf += 4; + + /* Write buffer */ + memcpy(buf, patch->lazperf, patch->lazperfsize); + SET_VARSIZE(serpch, serpch_size); + + return serpch; +} + static SERIALIZED_PATCH * pc_patch_uncompressed_serialize(const PCPATCH *patch_in) { @@ -693,6 +738,11 @@ pc_patch_serialize(const PCPATCH *patch_in, void *userdata) serpatch = pc_patch_ght_serialize(patch); break; } + case PC_LAZPERF: + { + serpatch = pc_patch_lazperf_serialize(patch); + break; + } default: { pcerror("%s: unsupported compression type %d", __func__, patch->type); @@ -880,6 +930,39 @@ pc_patch_ght_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schem return (PCPATCH*)patch; } +static PCPATCH * +pc_patch_lazperf_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) +{ + PCPATCH_LAZPERF *patch; + uint32_t lazperfsize; + int npoints = serpatch->npoints; + size_t stats_size = pc_stats_size(schema); + uint8_t *buf = (uint8_t*)serpatch->data + stats_size; + + /* Reference the external data */ + patch = pcalloc(sizeof(PCPATCH_LAZPERF)); + + /* Set up basic info */ + patch->type = serpatch->compression; + patch->schema = schema; + patch->readonly = true; + patch->npoints = npoints; + patch->bounds = serpatch->bounds; + + /* Point into the stats area */ + patch->stats = pc_patch_stats_deserialize(schema, serpatch->data); + + /* Set up buffer */ + memcpy(&lazperfsize, buf, 4); + patch->lazperfsize = lazperfsize; + buf += 4; + + patch->lazperf = pcalloc( patch->lazperfsize ); + memcpy(patch->lazperf, buf, patch->lazperfsize); + + return (PCPATCH*)patch; +} + PCPATCH * pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) { @@ -891,6 +974,8 @@ pc_patch_deserialize(const SERIALIZED_PATCH *serpatch, const PCSCHEMA *schema) return pc_patch_dimensional_deserialize(serpatch, schema); case PC_GHT: return pc_patch_ght_deserialize(serpatch, schema); + case PC_LAZPERF: + return pc_patch_lazperf_deserialize(serpatch, schema); } pcerror("%s: unsupported compression type", __func__); return NULL; diff --git a/pgsql/sql/pointcloud-laz.sql b/pgsql/sql/pointcloud-laz.sql new file mode 100644 index 00000000..33ef7c1e --- /dev/null +++ b/pgsql/sql/pointcloud-laz.sql @@ -0,0 +1,98 @@ +INSERT INTO pointcloud_formats (pcid, srid, schema) +VALUES (5, 0, +' + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + laz + +' +); + + +CREATE TABLE IF NOT EXISTS pa_test_laz ( + pa PCPATCH(5) +); +\d pa_test_laz + +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050008'); +INSERT INTO pa_test_laz (pa) VALUES ('000000000500000000000000020000000600000007000000050006000000090000000A00000005000A'); +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050003'); +INSERT INTO pa_test_laz (pa) VALUES ('0000000005000000000000000200000002000000030000000500060000000200000003000000050001'); + +SELECT pc_explode(pa) FROM pa_test_laz; +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz; +SELECT * FROM pa_test_laz; + +SELECT Sum(PC_NumPoints(pa)) FROM pa_test_laz; +SELECT Sum(PC_MemSize(pa)) FROM pa_test_laz; +SELECT Sum(PC_PatchMax(pa,'x')) FROM pa_test_laz; +SELECT Sum(PC_PatchMin(pa,'x')) FROM pa_test_laz; + +DELETE FROM pa_test_laz; +INSERT INTO pa_test_laz (pa) +SELECT PC_Patch(PC_MakePoint(5, ARRAY[x,y,z,intensity])) +FROM ( + SELECT + -127+a/100.0 AS x, + 45+a/100.0 AS y, + 1.0*a AS z, + a/10 AS intensity, + a/400 AS gid + FROM generate_series(1,1600) AS a +) AS values GROUP BY gid; + +SELECT pc_explode(pa) FROM pa_test_laz LIMIT 20; +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz LIMIT 20; +SELECT * FROM pa_test_laz LIMIT 20; + +SELECT Sum(PC_NumPoints(pa)) FROM pa_test_laz; +SELECT Sum(PC_MemSize(pa)) FROM pa_test_laz; + +SELECT Max(PC_PatchMax(pa,'x')) FROM pa_test_laz; +SELECT Min(PC_PatchMin(pa,'x')) FROM pa_test_laz; +SELECT Min(PC_PatchMin(pa,'z')) FROM pa_test_laz; + +SELECT pc_astext(PC_FilterLessThan(pa, 'z', 5)) FROM pa_test_laz; +SELECT pc_astext(PC_FilterGreaterThan(pa, 'z', 1595)) FROM pa_test_laz; +SELECT pc_astext(PC_FilterEquals(pa, 'z', 500)) FROM pa_test_laz; +SELECT pc_astext(PC_FilterBetween(pa, 'z', 500, 505)) FROM pa_test_laz; + +DELETE FROM pa_test_laz; +INSERT INTO pa_test_laz( pa ) VALUES ('01050000000300000004000000210000000000000000000000000000000a004417593a34c1c5f74f83179fc2448960000000'); + +SELECT pc_explode(pa) FROM pa_test_laz; +SELECT pc_astext(pc_explode(pa)) FROM pa_test_laz; + +TRUNCATE pointcloud_formats; diff --git a/pgsql/sql/pointcloud.sql b/pgsql/sql/pointcloud.sql index 0d295f55..a0669abc 100644 --- a/pgsql/sql/pointcloud.sql +++ b/pgsql/sql/pointcloud.sql @@ -318,7 +318,8 @@ FROM p1, ( values ('dimensional','rle'), ('dimensional','zlib'), ('dimensional','sigbits'), - ('dimensional','auto') + ('dimensional','auto'), + ('laz','null') -- ,('ght',null) -- fails due to https://github.com/pgpointcloud/pointcloud/issues/35 ) dimcompr(compr,sc) ORDER BY compr,sc,v; diff --git a/tools/benchmark_compression/compression_benchmark.sh b/tools/benchmark_compression/compression_benchmark.sh new file mode 100644 index 00000000..2b510b5f --- /dev/null +++ b/tools/benchmark_compression/compression_benchmark.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +DB=compression_benchmark + +createdb $DB + +psql -d $DB -f pointcloud.sql > /dev/null 2>&1 +psql -d $DB -f pointcloud-laz.sql > /dev/null 2>&1 +psql -d $DB -f pointcloud-dim.sql > /dev/null 2>&1 +psql -d $DB -f getsize.sql + +dropdb $DB diff --git a/tools/benchmark_compression/getsize.sql b/tools/benchmark_compression/getsize.sql new file mode 100644 index 00000000..768bff5b --- /dev/null +++ b/tools/benchmark_compression/getsize.sql @@ -0,0 +1,5 @@ +SELECT + relname as "Table", + pg_size_pretty(pg_total_relation_size(relid)) As "Size", + pg_size_pretty(pg_total_relation_size(relid) - pg_relation_size(relid)) as "External Size" + FROM pg_catalog.pg_statio_user_tables ORDER BY pg_total_relation_size(relid) DESC; diff --git a/tools/benchmark_compression/pointcloud-dim.sql b/tools/benchmark_compression/pointcloud-dim.sql new file mode 100644 index 00000000..5e9ccc14 --- /dev/null +++ b/tools/benchmark_compression/pointcloud-dim.sql @@ -0,0 +1,63 @@ +create EXTENSION if not exists pointcloud; + +INSERT INTO pointcloud_formats (pcid, srid, schema) +VALUES (5, 0, +' + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + dimensional + +' +); + + +CREATE TABLE IF NOT EXISTS pa_compression_dimensional ( + pa PCPATCH(5) +); +\d pa_compression_dimensional + +INSERT INTO pa_compression_dimensional (pa) +SELECT PC_Patch(PC_MakePoint(5, ARRAY[x,y,z,intensity])) +FROM ( + SELECT + -127+a/100.0 AS x, + 45+a/100.0 AS y, + 1.0*a AS z, + a/10 AS intensity, + a/400 AS gid + FROM generate_series(1,100000) AS a +) AS values GROUP BY gid; + +TRUNCATE pointcloud_formats; diff --git a/tools/benchmark_compression/pointcloud-laz.sql b/tools/benchmark_compression/pointcloud-laz.sql new file mode 100644 index 00000000..2f994c49 --- /dev/null +++ b/tools/benchmark_compression/pointcloud-laz.sql @@ -0,0 +1,63 @@ +create EXTENSION if not exists pointcloud; + +INSERT INTO pointcloud_formats (pcid, srid, schema) +VALUES (5, 0, +' + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + laz + +' +); + + +CREATE TABLE IF NOT EXISTS pa_compression_laz ( + pa PCPATCH(5) +); +\d pa_compression_laz + +INSERT INTO pa_compression_laz (pa) +SELECT PC_Patch(PC_MakePoint(5, ARRAY[x,y,z,intensity])) +FROM ( + SELECT + -127+a/100.0 AS x, + 45+a/100.0 AS y, + 1.0*a AS z, + a/10 AS intensity, + a/400 AS gid + FROM generate_series(1,100000) AS a +) AS values GROUP BY gid; + +TRUNCATE pointcloud_formats; diff --git a/tools/benchmark_compression/pointcloud.sql b/tools/benchmark_compression/pointcloud.sql new file mode 100644 index 00000000..531a8dc8 --- /dev/null +++ b/tools/benchmark_compression/pointcloud.sql @@ -0,0 +1,63 @@ +create EXTENSION if not exists pointcloud; + +INSERT INTO pointcloud_formats (pcid, srid, schema) +VALUES (5, 0, +' + + + 1 + 4 + X coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + X + int32_t + 0.01 + + + 2 + 4 + Y coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Y + int32_t + 0.01 + + + 3 + 4 + Z coordinate as a long integer. You must use the scale and offset information of the header to determine the double value. + Z + int32_t + 0.01 + + + 4 + 2 + The intensity value is the integer representation of the pulse return magnitude. This value is optional and system specific. However, it should always be included if available. + Intensity + uint16_t + 1 + + + none + +' +); + + +CREATE TABLE IF NOT EXISTS pa_compression_none ( + pa PCPATCH(5) +); +\d pa_compression_none + +INSERT INTO pa_compression_none (pa) +SELECT PC_Patch(PC_MakePoint(5, ARRAY[x,y,z,intensity])) +FROM ( + SELECT + -127+a/100.0 AS x, + 45+a/100.0 AS y, + 1.0*a AS z, + a/10 AS intensity, + a/400 AS gid + FROM generate_series(1,100000) AS a +) AS values GROUP BY gid; + +TRUNCATE pointcloud_formats;