Skip to content

Commit

Permalink
Implement raster source feature to read data from third-party sources…
Browse files Browse the repository at this point in the history 
…, to be used in lua profiles.

* Adds a data structure, RasterSource, to store parsed + queryable data
* Adds bindings for that and relevant data structures as well as source_function and segment_function
* Adds relevant unit tests and cucumber tests
* Bring-your-own-data feature
  • Loading branch information
Lauren Budorick committed Sep 4, 2015
1 parent 6cbbd1e commit bac6703
Show file tree
Hide file tree
Showing 19 changed files with 744 additions and 12 deletions.
5 changes: 3 additions & 2 deletions CMakeLists.txt
View file
Expand Up @@ -51,10 +51,11 @@ configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/util/git_sha.cpp
)
file(GLOB ExtractorGlob extractor/*.cpp)
file(GLOB ImporterGlob data_structures/import_edge.cpp data_structures/external_memory_node.cpp)
file(GLOB ImporterGlob data_structures/import_edge.cpp data_structures/external_memory_node.cpp data_structures/raster_source.cpp)
add_library(IMPORT OBJECT ${ImporterGlob})
add_library(LOGGER OBJECT util/simple_logger.cpp)
add_library(PHANTOMNODE OBJECT data_structures/phantom_node.cpp)
add_library(RASTERSOURCE OBJECT data_structures/raster_source.cpp)
add_library(EXCEPTION OBJECT util/osrm_exception.cpp)
add_library(MERCATOR OBJECT util/mercator.cpp)
add_library(ANGLE OBJECT util/compute_angle.cpp)
Expand Down Expand Up @@ -102,7 +103,7 @@ add_executable(osrm-routed routed.cpp ${ServerGlob} $<TARGET_OBJECTS:EXCEPTION>)
add_executable(osrm-datastore datastore.cpp $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:FINGERPRINT> $<TARGET_OBJECTS:GITDESCRIPTION> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR>)

# Unit tests
add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR> $<TARGET_OBJECTS:COMPRESSEDEDGE> $<TARGET_OBJECTS:GRAPHCOMPRESSOR> $<TARGET_OBJECTS:RESTRICTION>)
add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:MERCATOR> $<TARGET_OBJECTS:COMPRESSEDEDGE> $<TARGET_OBJECTS:GRAPHCOMPRESSOR> $<TARGET_OBJECTS:RESTRICTION> $<TARGET_OBJECTS:RASTERSOURCE>)
add_executable(algorithm-tests EXCLUDE_FROM_ALL unit_tests/algorithm_tests.cpp ${AlgorithmTestsGlob} $<TARGET_OBJECTS:COORDINATE> $<TARGET_OBJECTS:LOGGER> $<TARGET_OBJECTS:PHANTOMNODE> $<TARGET_OBJECTS:EXCEPTION> $<TARGET_OBJECTS:RESTRICTION> $<TARGET_OBJECTS:COMPRESSEDEDGE>)

# Benchmarks
Expand Down
6 changes: 4 additions & 2 deletions appveyor-build.bat
View file
Expand Up @@ -65,12 +65,14 @@ IF %ERRORLEVEL% NEQ 0 GOTO ERROR

SET PATH=c:\projects\osrm\osrm-deps\libs\bin;%PATH%

CD ..
ECHO running datastructure-tests.exe ...
datastructure-tests.exe
%Configuration%\datastructure-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
ECHO running algorithm-tests.exe ...
algorithm-tests.exe
%Configuration%\algorithm-tests.exe
IF %ERRORLEVEL% NEQ 0 GOTO ERROR
cd %Configuration%

IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE
ECHO ========= CREATING PACKAGES ==========
Expand Down
178 changes: 178 additions & 0 deletions data_structures/raster_source.cpp
View file
@@ -0,0 +1,178 @@
/*
Copyright (c) 2015, Project OSRM contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "raster_source.hpp"

#include "../util/simple_logger.hpp"
#include "../util/timing_util.hpp"

#include <osrm/coordinate.hpp>

#include <cmath>

RasterSource::RasterSource(RasterGrid _raster_data,
std::size_t _width,
std::size_t _height,
int _xmin,
int _xmax,
int _ymin,
int _ymax)
: xstep(calcSize(_xmin, _xmax, _width)), ystep(calcSize(_ymin, _ymax, _height)),
raster_data(_raster_data), width(_width), height(_height), xmin(_xmin), xmax(_xmax),
ymin(_ymin), ymax(_ymax)
{
BOOST_ASSERT(xstep != 0);
BOOST_ASSERT(ystep != 0);
}

float RasterSource::calcSize(int min, int max, std::size_t count) const
{
BOOST_ASSERT(count > 0);
return (max - min) / (static_cast<float>(count) - 1);
}

// Query raster source for nearest data point
RasterDatum RasterSource::getRasterData(const int lon, const int lat) const
{
if (lon < xmin || lon > xmax || lat < ymin || lat > ymax)
{
return {};
}

const std::size_t xth = static_cast<std::size_t>(round((lon - xmin) / xstep));
const std::size_t yth = static_cast<std::size_t>(round((ymax - lat) / ystep));

return {raster_data(xth, yth)};
}

// Query raster source using bilinear interpolation
RasterDatum RasterSource::getRasterInterpolate(const int lon, const int lat) const
{
if (lon < xmin || lon > xmax || lat < ymin || lat > ymax)
{
return {};
}

const auto xthP = (lon - xmin) / xstep;
const auto ythP = (ymax - lat) / ystep;

const std::size_t top = static_cast<std::size_t>(fmax(floor(ythP), 0));
const std::size_t bottom = static_cast<std::size_t>(fmin(ceil(ythP), height - 1));
const std::size_t left = static_cast<std::size_t>(fmax(floor(xthP), 0));
const std::size_t right = static_cast<std::size_t>(fmin(ceil(xthP), width - 1));

// Calculate distances from corners for bilinear interpolation
const float fromLeft = (lon - left * xstep + xmin) / xstep;
const float fromTop = (ymax - top * ystep - lat) / ystep;
const float fromRight = 1 - fromLeft;
const float fromBottom = 1 - fromTop;

return {static_cast<std::int32_t>(raster_data(left, top) * (fromRight * fromBottom) +
raster_data(right, top) * (fromLeft * fromBottom) +
raster_data(left, bottom) * (fromRight * fromTop) +
raster_data(right, bottom) * (fromLeft * fromTop))};
}

// Load raster source into memory
int SourceContainer::loadRasterSource(const std::string &path_string,
double xmin,
double xmax,
double ymin,
double ymax,
std::size_t nrows,
std::size_t ncols)
{
const auto _xmin = static_cast<int>(xmin * COORDINATE_PRECISION);
const auto _xmax = static_cast<int>(xmax * COORDINATE_PRECISION);
const auto _ymin = static_cast<int>(ymin * COORDINATE_PRECISION);
const auto _ymax = static_cast<int>(ymax * COORDINATE_PRECISION);

const auto itr = LoadedSourcePaths.find(path_string);
if (itr != LoadedSourcePaths.end())
{
SimpleLogger().Write() << "[source loader] Already loaded source '" << path_string
<< "' at source_id " << itr->second;
return itr->second;
}

int source_id = static_cast<int>(LoadedSources.size());

SimpleLogger().Write() << "[source loader] Loading from " << path_string << " ... ";
TIMER_START(loading_source);

boost::filesystem::path filepath(path_string);
if (!boost::filesystem::exists(filepath))
{
throw osrm::exception("error reading: no such path");
}

RasterGrid rasterData{filepath, ncols, nrows};

RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax};
TIMER_STOP(loading_source);
LoadedSourcePaths.emplace(path_string, source_id);
LoadedSources.push_back(std::move(source));

SimpleLogger().Write() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";

return source_id;
}

// External function for looking up nearest data point from a specified source
RasterDatum SourceContainer::getRasterDataFromSource(unsigned int source_id, int lon, int lat)
{
if (LoadedSources.size() < source_id + 1)
{
throw osrm::exception("error reading: no such loaded source");
}

BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION));
BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION));
BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION));
BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION));

const auto &found = LoadedSources[source_id];
return found.getRasterData(lon, lat);
}

// External function for looking up interpolated data from a specified source
RasterDatum
SourceContainer::getRasterInterpolateFromSource(unsigned int source_id, int lon, int lat)
{
if (LoadedSources.size() < source_id + 1)
{
throw osrm::exception("error reading: no such loaded source");
}

BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION));
BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION));
BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION));
BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION));

const auto &found = LoadedSources[source_id];
return found.getRasterInterpolate(lon, lat);
}
175 changes: 175 additions & 0 deletions data_structures/raster_source.hpp
View file
@@ -0,0 +1,175 @@
/*
Copyright (c) 2015, Project OSRM contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef RASTER_SOURCE_HPP
#define RASTER_SOURCE_HPP

#include "../util/osrm_exception.hpp"

#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/spirit/include/qi_int.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/assert.hpp>

#include <unordered_map>
#include <iterator>

/**
\brief Small wrapper around raster source queries to optionally provide results
gracefully, depending on source bounds
*/
struct RasterDatum
{
static std::int32_t get_invalid() { return std::numeric_limits<std::int32_t>::max(); }

std::int32_t datum = get_invalid();

RasterDatum() = default;

RasterDatum(std::int32_t _datum) : datum(_datum) {}
};

class RasterGrid
{
public:
RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim)
{
xdim = _xdim;
ydim = _ydim;
_data.reserve(ydim * xdim);

boost::filesystem::ifstream stream(filepath);
if (!stream)
{
throw osrm::exception("Unable to open raster file.");
}

stream.seekg(0, std::ios_base::end);
std::string buffer;
buffer.resize(static_cast<std::size_t>(stream.tellg()));

stream.seekg(0, std::ios_base::beg);

BOOST_ASSERT(buffer.size() > 1);
stream.read(&buffer[0], static_cast<std::streamsize>(buffer.size()));

boost::algorithm::trim(buffer);

auto itr = buffer.begin();
auto end = buffer.end();

bool r = false;
try
{
r = boost::spirit::qi::parse(
itr, end, +boost::spirit::qi::int_ % +boost::spirit::qi::space, _data);
}
catch (std::exception const &ex)
{
throw osrm::exception(
std::string("Failed to read from raster source with exception: ") + ex.what());
}

if (!r || itr != end)
{
throw osrm::exception("Failed to parse raster source correctly.");
}
}

RasterGrid(const RasterGrid &) = default;
RasterGrid &operator=(const RasterGrid &) = default;

RasterGrid(RasterGrid &&) = default;
RasterGrid &operator=(RasterGrid &&) = default;

std::int32_t operator()(std::size_t x, std::size_t y) { return _data[y * xdim + x]; }
std::int32_t operator()(std::size_t x, std::size_t y) const { return _data[(y)*xdim + (x)]; }

private:
std::vector<std::int32_t> _data;
std::size_t xdim, ydim;
};

/**
\brief Stores raster source data in memory and provides lookup functions.
*/
class RasterSource
{
private:
const float xstep;
const float ystep;

float calcSize(int min, int max, std::size_t count) const;

public:
RasterGrid raster_data;

const std::size_t width;
const std::size_t height;
const int xmin;
const int xmax;
const int ymin;
const int ymax;

RasterDatum getRasterData(const int lon, const int lat) const;

RasterDatum getRasterInterpolate(const int lon, const int lat) const;

RasterSource(RasterGrid _raster_data,
std::size_t width,
std::size_t height,
int _xmin,
int _xmax,
int _ymin,
int _ymax);
};

class SourceContainer
{
public:
SourceContainer() = default;

int loadRasterSource(const std::string &path_string,
double xmin,
double xmax,
double ymin,
double ymax,
std::size_t nrows,
std::size_t ncols);

RasterDatum getRasterDataFromSource(unsigned int source_id, int lon, int lat);

RasterDatum getRasterInterpolateFromSource(unsigned int source_id, int lon, int lat);

private:
std::vector<RasterSource> LoadedSources;
std::unordered_map<std::string, int> LoadedSourcePaths;
};

#endif /* RASTER_SOURCE_HPP */

0 comments on commit bac6703

Please sign in to comment.