Compression (#1736)

* Add general deflate/LZMA support. * Add lzma cmake file * Add xz to dockerfile. * Try xz-dev. * Put xz in the right list. * Modify travis build script for xz(lzma) compression. * Update Dockerfile for xz(lzma) * Use proper casting to avoid warnings on Linux. * Make interface consistent for compressors. * Update SQLite for new compression API. * Update osgeo4w install to grab xz-devel. * Add support for zstd compression. * Add zstd to alpine build. * Add zstd. * Add zstd-dev pkg. * Update docker/build files for zstd. * Convert LazPerfDecompressor to new interface. * Update Dockerfile(s) for lzma(xz) and zstd. * Update compression for GreyhoundReader. * Made compression modules optional. * Allow compression types to be optional. * Provide appropriate defaults for compression options.
PDAL · Nov 30, 2017 · 813caf8 · 813caf8
1 parent 88b102e
commit 813caf8
Show file tree

Hide file tree

Showing 24 changed files with 711 additions and 479 deletions.
diff --git a/cmake/lzma.cmake b/cmake/lzma.cmake
@@ -1,14 +1,18 @@
 #
 # LZMA support
 #
-find_package(LibLZMA REQUIRED)
-set_package_properties(LibLZMA PROPERTIES TYPE REQUIRED
-        PURPOSE "General compression support")
-if(LIBLZMA_FOUND)
-    set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}
-        "${LIBLZMA_LIBRARIES}")
-    include_directories(${LIBLZMA_INCLUDE_DIRS})
-    mark_as_advanced(CLEAR LIBLZMA_INCLUDE_DIRS)
-    mark_as_advanced(CLEAR LIBLZMA_LIBRARIES)
-    set(PDAL_HAVE_LZMA 1)
+option(WITH_LZMA
+    "Build support for compression/decompression with LZMA" FALSE)
+if (WITH_LZMA)
+    find_package(LibLZMA REQUIRED)
+    set_package_properties(LibLZMA PROPERTIES TYPE REQUIRED
+            PURPOSE "General compression support")
+    if(LIBLZMA_FOUND)
+        set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}
+            "${LIBLZMA_LIBRARIES}")
+        include_directories(${LIBLZMA_INCLUDE_DIRS})
+        mark_as_advanced(CLEAR LIBLZMA_INCLUDE_DIRS)
+        mark_as_advanced(CLEAR LIBLZMA_LIBRARIES)
+        set(PDAL_HAVE_LZMA 1)
+    endif()
 endif()
diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake
@@ -1,13 +1,18 @@
 #
 # ZLIB support
 #
-find_package(ZLIB REQUIRED)
-set_package_properties(ZLIB PROPERTIES TYPE REQUIRED
+option(WITH_ZLIB
+    "Build support for compression/decompression with zlib/deflate." TRUE)
+if (WITH_ZLIB)
+    find_package(ZLIB REQUIRED)
+    set_package_properties(ZLIB PROPERTIES TYPE REQUIRED
         PURPOSE "Compression support in BPF")
-if(ZLIB_FOUND)
-    set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} "${ZLIB_LIBRARY}")
-    include_directories(${ZLIB_INCLUDE_DIR})
-    mark_as_advanced(CLEAR ZLIB_INCLUDE_DIR)
-    mark_as_advanced(CLEAR ZLIB_LIBRARY)
-    set(PDAL_HAVE_ZLIB 1)
-endif()
+    if(ZLIB_FOUND)
+        set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}
+            "${ZLIB_LIBRARY}")
+        include_directories(${ZLIB_INCLUDE_DIR})
+        mark_as_advanced(CLEAR ZLIB_INCLUDE_DIR)
+        mark_as_advanced(CLEAR ZLIB_LIBRARY)
+        set(PDAL_HAVE_ZLIB 1)
+    endif(ZLIB_FOUND)
+endif(WITH_ZLIB)
diff --git a/cmake/zstd.cmake b/cmake/zstd.cmake
@@ -1,13 +1,17 @@
 #
 # LZMA support
 #
-find_package(ZSTD REQUIRED)
-set_package_properties(ZSTD PROPERTIES TYPE REQUIRED
+option(WITH_ZSTD
+    "Build support for compression/decompression with Zstd." FALSE)
+if (WITH_ZSTD)
+    find_package(ZSTD REQUIRED)
+    set_package_properties(ZSTD PROPERTIES TYPE REQUIRED
         PURPOSE "General compression support")
-if(ZSTD_FOUND)
-    set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}
-        "${ZSTD_STATIC_LIB}")
-    mark_as_advanced(CLEAR ZSTD_INCLUDE_DIRS)
-    mark_as_advanced(CLEAR ZSTD_LIBRARIES)
-    set(PDAL_HAVE_ZSTD 1)
-endif()
+    if (ZSTD_FOUND)
+        set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}
+            "${ZSTD_STATIC_LIB}")
+        mark_as_advanced(CLEAR ZSTD_INCLUDE_DIRS)
+        mark_as_advanced(CLEAR ZSTD_LIBRARIES)
+        set(PDAL_HAVE_ZSTD 1)
+    endif(ZSTD_FOUND)
+endif(WITH_ZSTD)
diff --git a/io/BpfCompressor.cpp b/io/BpfCompressor.cpp
@@ -34,11 +34,13 @@
 
 #include "BpfCompressor.hpp"
 
+
 #include <pdal/pdal_internal.hpp>
 
 namespace pdal
 {
 
+#ifdef PDAL_HAVE_ZLIB
 void BpfCompressor::startBlock()
 {
     // Initialize the stream.
@@ -129,4 +131,16 @@ void BpfCompressor::finish()
     blockEnd.rewind();
 }
 
+#else
+
+void BpfCompressor::startBlock()
+{}
+void BpfCompressor::compress()
+{}
+void BpfCompressor::finish()
+{}
+
+#endif // PDAL_HAVE_ZLIB
+
 } // namespace pdal
+
diff --git a/io/BpfCompressor.hpp b/io/BpfCompressor.hpp
@@ -34,9 +34,12 @@
 
 #pragma once
 
+
 #include <stdexcept>
 #include <ostream>
+#ifdef PDAL_HAVE_ZLIB
 #include <zlib.h>
+#endif // PDAL_HAVE_ZLIB
 
 #include <pdal/util/Charbuf.hpp>
 #include <pdal/util/OStream.hpp>
@@ -53,17 +56,24 @@ class BpfCompressor
         {}
     };
 
+#ifdef PDAL_HAVE_ZLIB
     BpfCompressor(OLeStream& out, size_t maxSize) :
         m_out(out), m_inbuf(maxSize), m_blockStart(out), m_rawSize(0),
         m_compressedSize(0)
     {}
+#else
+    BpfCompressor(OLeStream&, size_t)
+    {}
+#endif // PDAL_HAVE_ZLIB
+
     void startBlock();
     void finish();
     void compress();
 
 private:
     static const int CHUNKSIZE = 1000000;
 
+#ifdef PDAL_HAVE_ZLIB
     OLeStream& m_out;
     Charbuf m_charbuf;
     std::vector<char> m_inbuf;
@@ -72,6 +82,8 @@ class BpfCompressor
     OStreamMarker m_blockStart;
     size_t m_rawSize;
     size_t m_compressedSize;
+#endif // PDAL_HAVE_ZLIB
 };
 
 } // namespace pdal
+
diff --git a/io/BpfReader.cpp b/io/BpfReader.cpp
@@ -115,6 +115,11 @@ void BpfReader::initialize()
     {
         throwError(err.what());
     }
+#ifndef PDAL_HAVE_ZLIB
+    if (m_header.m_compression)
+        throwError("Can't read compressed BPF. PDAL wasn't built with "
+            "Zlib support.");
+#endif
 
     std::string code;
     if (m_header.m_coordType == static_cast<int>(BpfCoordType::Cartesian))
@@ -256,6 +261,7 @@ void BpfReader::ready(PointTableRef)
     m_stream.seek(m_header.m_len);
     m_index = 0;
     m_start = m_stream.position();
+#ifdef PDAL_HAVE_ZLIB
     if (m_header.m_compression)
     {
         m_deflateBuf.resize(numPoints() * m_dims.size() * sizeof(float));
@@ -269,6 +275,7 @@ void BpfReader::ready(PointTableRef)
         m_charbuf.initialize(m_deflateBuf.data(), m_deflateBuf.size(), m_start);
         m_stream.pushStream(new std::istream(&m_charbuf));
     }
+#endif // PDAL_HAVE_ZLIB
 }
 
 
@@ -313,24 +320,6 @@ point_count_t BpfReader::read(PointViewPtr data, point_count_t count)
 }
 
 
-size_t BpfReader::readBlock(std::vector<char>& outBuf, size_t index)
-{
-    uint32_t finalBytes;
-    uint32_t compressBytes;
-
-    m_stream >> finalBytes;
-    m_stream >> compressBytes;
-
-    std::vector<char> in(compressBytes);
-
-    // Fill the input bytes from the stream.
-    m_stream.get(in);
-    int ret = inflate(in.data(), compressBytes,
-        outBuf.data() + index, finalBytes);
-    return (ret ? 0 : finalBytes);
-}
-
-
 bool BpfReader::eof()
 {
     return m_index >= numPoints();
@@ -413,6 +402,7 @@ void BpfReader::readDimMajor(PointRef& point)
             m_streams.emplace_back(new ILeStream());
             m_streams.back()->open(m_filename);
 
+#ifdef PDAL_HAVE_ZLIB
             if (m_header.m_compression)
             {
                 m_charbufs.emplace_back(new Charbuf());
@@ -422,6 +412,7 @@ void BpfReader::readDimMajor(PointRef& point)
                 m_streams.back()->pushStream(
                         new std::istream(m_charbufs.back().get()));
             }
+#endif // PDAL_HAVE_ZLIB
 
             m_streams.back()->seek(m_start + offset);
         }
@@ -622,6 +613,25 @@ void BpfReader::seekByteMajor(size_t dimIdx, size_t byteIdx, PointId ptIdx)
 }
 
 
+#ifdef PDAL_HAVE_ZLIB
+size_t BpfReader::readBlock(std::vector<char>& outBuf, size_t index)
+{
+    uint32_t finalBytes;
+    uint32_t compressBytes;
+
+    m_stream >> finalBytes;
+    m_stream >> compressBytes;
+
+    std::vector<char> in(compressBytes);
+
+    // Fill the input bytes from the stream.
+    m_stream.get(in);
+    int ret = inflate(in.data(), compressBytes,
+        outBuf.data() + index, finalBytes);
+    return (ret ? 0 : finalBytes);
+}
+
+
 int BpfReader::inflate(char *buf, uint32_t insize,
     char *outbuf, uint32_t outsize)
 {
@@ -649,5 +659,6 @@ int BpfReader::inflate(char *buf, uint32_t insize,
     (void)inflateEnd(&strm);
     return ret == Z_STREAM_END ? 0 : -1;
 }
+#endif // PDAL_HAVE_ZLIB
 
 } //namespace pdal
diff --git a/io/BpfWriter.cpp b/io/BpfWriter.cpp
@@ -41,8 +41,6 @@
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/ProgramArgs.hpp>
 
-#include <zlib.h>
-
 #include "BpfCompressor.hpp"
 #include <pdal/pdal_macros.hpp>
 #include <pdal/util/Utils.hpp>
@@ -102,6 +100,11 @@ void BpfWriter::initialize()
     m_header.m_coordId = m_coordId.m_val;
     m_header.m_coordType = Utils::toNative(m_header.m_coordId ?
         BpfCoordType::UTM : BpfCoordType::Cartesian);
+#ifndef PDAL_HAVE_ZLIB
+    if (m_compression)
+        throwError("Can't write compressed BPF. PDAL wasn't built with "
+            "Zlib support.");
+#endif
     m_header.m_compression = Utils::toNative(
             m_compression ? BpfCompression::Zlib : BpfCompression::None);
     m_extraData = Utils::base64_decode(m_extraDataSpec);
@@ -290,12 +293,11 @@ void BpfWriter::writePointMajor(const PointView* data)
 
 void BpfWriter::writeDimMajor(const PointView* data)
 {
-    // We're going to pretend for now that we only even have one point buffer.
+    // We're going to pretend for now that we only ever have one point buffer.
     BpfCompressor compressor(m_stream, data->size() * sizeof(float));
 
     for (auto & bpfDim : m_dims)
     {
-
         if (m_header.m_compression)
             compressor.startBlock();
         for (PointId idx = 0; idx < data->size(); ++idx)

diff --git a/io/LasReader.hpp b/io/LasReader.hpp
@@ -36,9 +36,10 @@
 
 #include <pdal/pdal_export.hpp>
 #include <pdal/plugin.hpp>
-#include <pdal/Compression.hpp>
 #include <pdal/PDALUtils.hpp>
 #include <pdal/Reader.hpp>
+#include <pdal/compression/LazPerfCompression.hpp>
+
 #ifdef PDAL_HAVE_LASZIP
 #include <laszip/laszip_api.h>
 #else

diff --git a/io/LasWriter.cpp b/io/LasWriter.cpp
@@ -38,7 +38,6 @@
 #include <iostream>
 #include <vector>
 
-#include <pdal/Compression.hpp>
 #include <pdal/DimUtil.hpp>
 #include <pdal/PDALUtils.hpp>
 #include <pdal/PointView.hpp>

diff --git a/io/LasWriter.hpp b/io/LasWriter.hpp
@@ -34,10 +34,11 @@
 
 #pragma once
 
-#include <pdal/Compression.hpp>
-#include <pdal/FlexWriter.hpp>
 #include <pdal/plugin.hpp>
 
+#include <pdal/FlexWriter.hpp>
+#include <pdal/compression/LazPerfCompression.hpp>
+
 #include "HeaderVal.hpp"
 #include "LasError.hpp"
 #include "LasHeader.hpp"

diff --git a/pdal/compression/Compression.hpp b/pdal/compression/Compression.hpp
@@ -0,0 +1,65 @@
+/******************************************************************************
+* Copyright (c) 2014, Howard Butler (howard@hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+*     * Redistributions of source code must retain the above copyright
+*       notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above copyright
+*       notice, this list of conditions and the following disclaimer in
+*       the documentation and/or other materials provided
+*       with the distribution.
+*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+*       names of its contributors may be used to endorse or promote
+*       products derived from this software without specific prior
+*       written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+#pragma once
+
+#include <pdal/pdal_internal.hpp>
+
+namespace pdal
+{
+
+enum class CompressionType
+{
+    None = 0,
+    Ght = 1,
+    Dimensional = 2,
+    Lazperf = 3,
+    Unknown = 256
+};
+
+using BlockCb = std::function<void(char *buf, size_t bufsize)>;
+const size_t CHUNKSIZE(1000000);
+
+class compression_error : public std::runtime_error
+{
+public:
+    compression_error() : std::runtime_error("General compression error")
+    {}
+
+    compression_error(const std::string& s) :
+        std::runtime_error("Compression: " + s)
+    {}
+};
+
+} // namespace pdal
+