From f2bf939bcd5d43a5a738b4e0ac07342f1897d702 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 17 Sep 2015 08:15:45 -0500 Subject: [PATCH] Fix chunking of LazPerf writer. Add LazPerf writer test. Add documentation regarding compressor/decompressor selection. --- doc/stages/readers.las.rst | 6 +++ doc/stages/writers.las.rst | 7 +-- include/pdal/Compression.hpp | 25 ++++++---- io/las/LasReader.hpp | 2 - io/las/LasWriter.cpp | 6 +-- test/unit/io/las/LasReaderTest.cpp | 2 + test/unit/io/las/LasWriterTest.cpp | 75 ++++++++++++++++++++++++++++-- 7 files changed, 102 insertions(+), 21 deletions(-) diff --git a/doc/stages/readers.las.rst b/doc/stages/readers.las.rst index 0a462946d3..6e0ff8da6d 100644 --- a/doc/stages/readers.las.rst +++ b/doc/stages/readers.las.rst @@ -47,3 +47,9 @@ extra_dims .. _LAS format: http://asprs.org/Committee-General/LASer-LAS-File-Format-Exchange-Activities.html +compression + May be set to "lazperf" or "laszip" to choose either the LazPerf decompressor + or the LasZip decompressor for LAZ files. PDAL must have been build with + support for the decompressor being requested. The LazPerf decompressor + doesn't support version 1 LAZ files or version 1.4 of LAS. + [Default: "laszip"] diff --git a/doc/stages/writers.las.rst b/doc/stages/writers.las.rst index b929a4f4ac..6b8f25e701 100644 --- a/doc/stages/writers.las.rst +++ b/doc/stages/writers.las.rst @@ -112,9 +112,10 @@ project_id UID reserved for the user [Default: Nil UID] compression - Set to true to apply compression to the output, creating a LAZ file instead - of an LAS file. Requires PDAL to have been built with compression support - by linking with LASzip. [Default: false] + Set to "lazperf" or "laszip" to apply compression to the output, creating + a LAZ file instead of an LAS file. "lazperf" selects the LazPerf compressor + and "laszip" (or "true") selects the LasZip compressor. PDAL must have + been built with support for the requested compressor. [Default: "none"] scale_x, scale_y, scale_z Scale to be divided from the X, Y and Z nominal values, respectively, after diff --git a/include/pdal/Compression.hpp b/include/pdal/Compression.hpp index 3b3dcc508e..3113efe3b4 100644 --- a/include/pdal/Compression.hpp +++ b/include/pdal/Compression.hpp @@ -207,10 +207,17 @@ class LazPerfCompressor class LazPerfVlrCompressor { + typedef laszip::io::__ofstream_wrapper OutputStream; + typedef laszip::encoders::arithmetic Encoder; + typedef laszip::formats::dynamic_compressor Compressor; + typedef laszip::factory::record_schema Schema; + public: - LazPerfVlrCompressor(std::ostream& stream) : - m_stream(stream), m_outputStream(stream), m_chunksize(50000), - m_chunkPointsWritten(0), m_chunkInfoPos(0), m_chunkOffset(0) + LazPerfVlrCompressor(std::ostream& stream, const Schema& schema, + uint32_t chunksize) : + m_stream(stream), m_outputStream(stream), m_schema(schema), + m_chunksize(chunksize), m_chunkPointsWritten(0), m_chunkInfoPos(0), + m_chunkOffset(0) {} ~LazPerfVlrCompressor() @@ -247,6 +254,8 @@ class LazPerfVlrCompressor m_encoder->done(); m_encoder.reset(); + newChunk(); + // Save our current position. Go to the location where we need // to write the chunk table offset at the beginning of the point data. std::streampos chunkTablePos = m_stream.tellp(); @@ -265,6 +274,7 @@ class LazPerfVlrCompressor OutputStream outputStream(m_stream); Encoder encoder(outputStream); laszip::compressors::integer compressor(32, 2); + compressor.init(); uint32_t predictor = 0; for (uint32_t offset : m_chunkTable) @@ -279,7 +289,8 @@ class LazPerfVlrCompressor private: void resetCompressor() { - m_encoder->done(); + if (m_encoder) + m_encoder->done(); m_encoder.reset(new Encoder(m_outputStream)); m_compressor = laszip::factory::build_compressor(*m_encoder, m_schema); } @@ -292,11 +303,6 @@ class LazPerfVlrCompressor m_chunkPointsWritten = 0; } - typedef laszip::io::__ofstream_wrapper OutputStream; - typedef laszip::encoders::arithmetic Encoder; - typedef laszip::formats::dynamic_compressor Compressor; - typedef laszip::factory::record_schema Schema; - std::ostream& m_stream; OutputStream m_outputStream; std::unique_ptr m_encoder; @@ -396,6 +402,7 @@ class LazPerfVlrDecompressor #else +typedef char LazPerfVlrCompressor; typedef char LazPerfVlrDecompressor; #endif // PDAL_HAVE_LAZPERF diff --git a/io/las/LasReader.hpp b/io/las/LasReader.hpp index 3baf8167df..27867cf65c 100644 --- a/io/las/LasReader.hpp +++ b/io/las/LasReader.hpp @@ -103,9 +103,7 @@ class PDAL_DLL LasReader : public pdal::Reader LasHeader m_lasHeader; std::unique_ptr m_zipPoint; std::unique_ptr m_unzipper; -#ifdef PDAL_HAVE_LAZPERF std::unique_ptr m_decompressor; -#endif point_count_t m_index; std::istream* m_istream; VlrList m_vlrs; diff --git a/io/las/LasWriter.cpp b/io/las/LasWriter.cpp index 4fac74ff3c..d7f58300b5 100644 --- a/io/las/LasWriter.cpp +++ b/io/las/LasWriter.cpp @@ -130,7 +130,6 @@ void LasWriter::processOptions(const Options& options) std::string compression = options.getValueOrDefault("compression"); compression = Utils::toupper(compression); -std::cerr << "Compression = " << compression << "!\n"; if (compression == "LASZIP" || compression == "TRUE") m_compression = LasCompression::LasZip; else if (compression == "LAZPERF") @@ -357,7 +356,7 @@ void LasWriter::prepOutput(std::ostream *outStream) if (m_lasHeader.versionEquals(1, 0)) out << (uint16_t)0xCCDD; m_lasHeader.setPointOffset((uint32_t)m_ostream->tellp()); - if (m_lasHeader.compressed()) + if (m_compression == LasCompression::LasZip) openCompression(); m_error.setLog(log()); @@ -653,7 +652,8 @@ void LasWriter::readyLazPerfCompression() zipvlr.extract((char *)data.data()); addVlr(LASZIP_USER_ID, LASZIP_RECORD_ID, "http://laszip.org", data); - m_compressor.reset(new LazPerfVlrCompressor(*m_ostream)); + m_compressor.reset(new LazPerfVlrCompressor(*m_ostream, schema, + zipvlr.chunk_size)); #endif } diff --git a/test/unit/io/las/LasReaderTest.cpp b/test/unit/io/las/LasReaderTest.cpp index 2eb2fc5c58..269671ca6e 100644 --- a/test/unit/io/las/LasReaderTest.cpp +++ b/test/unit/io/las/LasReaderTest.cpp @@ -393,6 +393,7 @@ TEST(LasReaderTest, callback) EXPECT_EQ(count, (point_count_t)1065); } +#ifdef PDAL_HAVE_LAZPERF // LAZ files are normally written in chunks of 50,000, so a file of size // 110,000 ensures we read some whole chunks and a partial. TEST(LasReaderTest, lazperf) @@ -437,6 +438,7 @@ TEST(LasReaderTest, lazperf) EXPECT_EQ(memcmp(buf1.get(), buf2.get(), pointSize), 0); } } +#endif // The header of 1.2-with-color-clipped says that it has 1065 points, diff --git a/test/unit/io/las/LasWriterTest.cpp b/test/unit/io/las/LasWriterTest.cpp index 7f2c96ffa6..0c15ac6e99 100644 --- a/test/unit/io/las/LasWriterTest.cpp +++ b/test/unit/io/las/LasWriterTest.cpp @@ -207,12 +207,13 @@ TEST(LasWriterTest, extra_dims) Options reader2Ops; reader2Ops.add("filename", Support::temppath("simple.las")); reader2Ops.add("extra_dims", "R1 =int32, B1= int16 ,G1=int32_t"); - std::shared_ptr reader2(new LasReader); - reader2->setOptions(reader2Ops); + + LasReader reader2; + reader2.setOptions(reader2Ops); PointTable readTable; - reader2->prepare(readTable); - viewSet = reader2->execute(readTable); + reader2.prepare(readTable); + viewSet = reader2.execute(readTable); pb = *viewSet.begin(); Dimension::Id::Enum r1 = readTable.layout()->findDim("R1"); EXPECT_TRUE(r1 != Dimension::Id::Unknown); @@ -461,6 +462,72 @@ TEST(LasWriterTest, flex2) EXPECT_EQ(r.preview().m_pointCount, 1065u); } +#ifdef PDAL_HAVE_LAZPERF +// LAZ files are normally written in chunks of 50,000, so a file of size +// 110,000 ensures we read some whole chunks and a partial. +TEST(LasWriterTest, lazperf) +{ + Options readerOps; + readerOps.add("filename", Support::datapath("las/autzen_trim.las")); + + LasReader lazReader; + lazReader.setOptions(readerOps); + + Options writerOps; + writerOps.add("filename", Support::temppath("temp.laz")); + writerOps.add("compression", "lazperf"); + + LasWriter lazWriter; + lazWriter.setOptions(writerOps); + lazWriter.setInput(lazReader); + + PointTable t; + lazWriter.prepare(t); + lazWriter.execute(t); + + // Now test the points were properly written. Use laszip. + Options ops1; + ops1.add("filename", Support::temppath("temp.laz")); + + LasReader r1; + r1.setOptions(ops1); + + PointTable t1; + r1.prepare(t1); + PointViewSet set1 = r1.execute(t1); + PointViewPtr view1 = *set1.begin(); + + Options ops2; + ops2.add("filename", Support::datapath("las/autzen_trim.las")); + + LasReader r2; + r2.setOptions(ops2); + + PointTable t2; + r2.prepare(t2); + PointViewSet set2 = r2.execute(t2); + PointViewPtr view2 = *set2.begin(); + + EXPECT_EQ(view1->size(), view2->size()); + EXPECT_EQ(view1->size(), (point_count_t)110000); + + DimTypeList dims = view1->dimTypes(); + size_t pointSize = view1->pointSize(); + EXPECT_EQ(view1->pointSize(), view2->pointSize()); + + // Validate some point data. + std::unique_ptr buf1(new char[pointSize]); + std::unique_ptr buf2(new char[pointSize]); + for (PointId i = 0; i < view1->pointSize(); i += 100) + { + view1->getPackedPoint(dims, i, buf1.get()); + view2->getPackedPoint(dims, i, buf2.get()); + EXPECT_EQ(memcmp(buf1.get(), buf2.get(), pointSize), 0); + } +} +#endif + + /** namespace {