Skip to content
Permalink
Browse files

Update Untwine to the latest main branch

This lowers the required minimum version of PDAL to >= 1.7 which makes
it much easier to use in many linux distributions which are not yet
shipping PDAL 2.2.
  • Loading branch information
wonder-sk authored and nyalldawson committed Dec 22, 2020
1 parent 8177449 commit b59270850687053a1b3ba52e660275f9312552d5
@@ -4,6 +4,7 @@
#include <string>
#include <vector>

#include <pdal/util/FileUtils.hpp>
#include <pdal/util/ProgramArgs.hpp>

#include "BuPyramid.hpp"
@@ -14,7 +14,7 @@

#include <list>

#include <pdal/util/FileUtils.hpp>
#include "../untwine/MapFile.hpp"

namespace untwine
{
@@ -44,16 +44,16 @@ class FileInfo
char *address() const
{ return reinterpret_cast<char *>(m_ctx.addr()); }

pdal::FileUtils::MapContext context() const
MapContext context() const
{ return m_ctx; }
void setContext(const pdal::FileUtils::MapContext& ctx)
void setContext(const MapContext& ctx)
{ m_ctx = ctx; }

private:
std::string m_filename;
int m_numPoints;
int m_start;
pdal::FileUtils::MapContext m_ctx;
MapContext m_ctx;
};
using FileInfoList = std::list<FileInfo>;

@@ -13,6 +13,7 @@
#pragma once

#include "../untwine/Common.hpp"
#include "../untwine/MapFile.hpp"
#include "../untwine/Point.hpp"

#include "BuTypes.hpp"
@@ -32,13 +33,11 @@ class PointAccessor
~PointAccessor()
{
for (FileInfo *fi : m_fileInfos)
pdal::FileUtils::unmapFile(fi->context());
unmapFile(fi->context());
}

void read(FileInfo& fi)
{
using namespace pdal::FileUtils;

std::string filename = m_b.inputDir + "/" + fi.filename();
auto ctx = mapFile(filename, true, 0, fi.numPoints() * m_b.pointSize);
if (ctx.m_addr == nullptr)
@@ -17,10 +17,9 @@
#include <unordered_map>
#include <vector>

#include <pdal/util/ThreadPool.hpp>

#include "BuTypes.hpp"
#include "OctantInfo.hpp"
#include "../untwine/ThreadPool.hpp"

namespace untwine
{
@@ -55,7 +54,7 @@ class PyramidManager
std::condition_variable m_cv;
std::unordered_map<VoxelKey, OctantInfo> m_completes;
std::queue<OctantInfo> m_queue;
pdal::ThreadPool m_pool;
ThreadPool m_pool;
uint64_t m_totalPoints;
ProgressWriter *m_progress;
//
@@ -88,8 +88,8 @@ void Epf::run(const Options& options, ProgressWriter& progress)
m_grid.resetLevel(options.level);

// This is just a debug thing that will allow the number of input files to be limited.
if (fileInfos.size() > m_fileLimit)
fileInfos.resize(m_fileLimit);
if (fileInfos.size() > options.fileLimit)
fileInfos.resize(options.fileLimit);

// Stick all the dimension names from each input file in a set.
std::unordered_set<std::string> allDimNames;
@@ -17,11 +17,11 @@
#include <vector>

#include <pdal/SpatialReference.hpp>
#include <pdal/util/ThreadPool.hpp>

#include "EpfTypes.hpp"
#include "Grid.hpp"
#include "../untwine/ProgressWriter.hpp"
#include "../untwine/ThreadPool.hpp"

namespace untwine
{
@@ -49,8 +49,7 @@ class Epf

Grid m_grid;
std::unique_ptr<Writer> m_writer;
pdal::ThreadPool m_pool;
size_t m_fileLimit;
ThreadPool m_pool;
FileInfo m_srsFileInfo;
};

@@ -15,6 +15,7 @@

#include "Reprocessor.hpp"
#include "../untwine/Common.hpp"
#include "../untwine/MapFile.hpp"

namespace untwine
{
@@ -48,7 +49,7 @@ Reprocessor::Reprocessor(const VoxelKey& k, int numPoints, int pointSize,

void Reprocessor::run()
{
auto ctx = pdal::FileUtils::mapFile(m_filename, true, 0, m_fileSize);
auto ctx = mapFile(m_filename, true, 0, m_fileSize);
if (ctx.addr() == nullptr)
{
std::cerr << "FATAL: " + m_filename + ": " + ctx.what();
@@ -66,7 +67,7 @@ void Reprocessor::run()
cell->advance();
pos += m_pointSize;
}
pdal::FileUtils::unmapFile(ctx);
unmapFile(ctx);
pdal::FileUtils::deleteFile(m_filename);
}

@@ -18,10 +18,9 @@
#include <mutex>
#include <string>

#include <pdal/util/ThreadPool.hpp>

#include "EpfTypes.hpp"
#include "BufferCache.hpp"
#include "../untwine/ThreadPool.hpp"
#include "../untwine/VoxelKey.hpp"

namespace untwine
@@ -54,7 +53,7 @@ class Writer
void run();

std::string m_directory;
pdal::ThreadPool m_pool;
ThreadPool m_pool;
BufferCache m_bufferCache;
bool m_stop;
size_t m_pointSize;
@@ -0,0 +1,109 @@
/******************************************************************************
* Copyright (c) 2020, Hobu Inc.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided
* with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
* OF SUCH DAMAGE.
****************************************************************************/

#include "MapFile.hpp"

namespace untwine
{

MapContext mapFile(const std::string& filename, bool readOnly,
size_t pos, size_t size)
{
MapContext ctx;

if (!readOnly)
{
ctx.m_error = "readOnly must be true.";
return ctx;
}

#ifndef WIN32
ctx.m_fd = ::open(filename.c_str(), readOnly ? O_RDONLY : O_RDWR);
#else
ctx.m_fd = ::_open(filename.c_str(), readOnly ? O_RDONLY : O_RDWR);
#endif

if (ctx.m_fd == -1)
{
ctx.m_error = "Mapped file couldn't be opened.";
return ctx;
}
ctx.m_size = size;

#ifndef _WIN32
ctx.m_addr = ::mmap(0, size, PROT_READ, MAP_SHARED, ctx.m_fd, (off_t)pos);
if (ctx.m_addr == MAP_FAILED)
{
ctx.m_addr = nullptr;
ctx.m_error = "Couldn't map file";
}
#else
ctx.m_handle = CreateFileMapping((HANDLE)_get_osfhandle(ctx.m_fd),
NULL, PAGE_READONLY, 0, 0, NULL);
uint32_t low = pos & 0xFFFFFFFF;
uint32_t high = (pos >> 8);
ctx.m_addr = MapViewOfFile(ctx.m_handle, FILE_MAP_READ, high, low,
ctx.m_size);
if (ctx.m_addr == nullptr)
ctx.m_error = "Couldn't map file";
#endif

return ctx;
}

MapContext unmapFile(MapContext ctx)
{
#ifndef _WIN32
if (::munmap(ctx.m_addr, ctx.m_size) == -1)
ctx.m_error = "Couldn't unmap file.";
else
{
ctx.m_addr = nullptr;
ctx.m_size = 0;
ctx.m_error = "";
}
::close(ctx.m_fd);
#else
if (UnmapViewOfFile(ctx.m_addr) == 0)
ctx.m_error = "Couldn't unmap file.";
else
{
ctx.m_addr = nullptr;
ctx.m_size = 0;
ctx.m_error = "";
}
CloseHandle(ctx.m_handle);
::_close(ctx.m_fd);
#endif
return ctx;
}

} // namespace untwine

@@ -0,0 +1,84 @@
/******************************************************************************
* Copyright (c) 2020, Hobu Inc.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided
* with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
* OF SUCH DAMAGE.
****************************************************************************/

#pragma once

#ifdef _WIN32
#include <Windows.h>
#else
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#endif

#include <string>

namespace untwine
{
struct MapContext
{
public:
MapContext() : m_fd(-1), m_addr(nullptr)
{}

void *addr() const
{ return m_addr; }
std::string what() const
{ return m_error; }

int m_fd;
size_t m_size;
void *m_addr;
std::string m_error;
#ifdef _WIN32
HANDLE m_handle;
#endif
};
/**
Map a file to memory.
\param filename Filename to map.
\param readOnly Must be true at this time.
\param pos Starting position of file to map.
\param size Number of bytes in file to map.
\return MapContext. addr() gets the mapped address. what() gets
any error message. addr() returns nullptr on error.
*/
MapContext mapFile(const std::string& filename, bool readOnly,
size_t pos, size_t size);

/**
Unmap a previously mapped file.
\param ctx Previously returned MapContext
\return MapContext indicating current state of the file mapping.
*/
MapContext unmapFile(MapContext ctx);

} // namespace untwine

0 comments on commit b592708

Please sign in to comment.
You can’t perform that action at this time.