Skip to content

Commit

Permalink
Merge pull request #2 from Turbo87/osmium
Browse files Browse the repository at this point in the history 
Osmium input reader
  • Loading branch information
@minad
minad committed Dec 11, 2011
2 parents 881ca70 + c550a3f commit 58e68c5
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 267 deletions.
16 changes: 10 additions & 6 deletions Makefile
View file
@@ -1,21 +1,25 @@
CPP=g++
CC=gcc
CFLAGS=-O2 -Wall
CPP = g++

CXXFLAGS = -ggdb -Wall -Wredundant-decls
CXXFLAGS += -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64

all: osm2shp

LIB_GEOS = $(shell geos-config --libs)
LIB_SHAPE = -lshp $(LIB_GEOS)
LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf

FILES = \
xml.o \
osm2shp.o \
osm/shapefile.o \
osm/handler.o \
osm/point_database.o

osm2shp: $(FILES)
$(CPP) $(CFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams -o $@
$(CPP) $(CXXFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams $(LIB_PROTOBUF) $(LIB_SHAPE) -o $@

%.o: %.cc
$(CPP) $(CFLAGS) -c $< -o $@
$(CPP) $(CXXFLAGS) -c $< -o $@

clean:
rm -f $(FILES) osm2shp
96 changes: 28 additions & 68 deletions osm/handler.cc
View file
@@ -1,6 +1,5 @@
#include "handler.hpp"
#include "shapefile.hpp"
#include "../xml.hpp"

#include <shapefil.h>
#include <sys/stat.h>
Expand All @@ -11,22 +10,23 @@

namespace osm {

template<typename T>
inline bool has_key(const T& map, const typename T::key_type& key) {
return map.find(key) != map.end();
template<typename T, class K>
inline bool has_key(const T& map, const K& key) {
const char *v = map.get_tag_by_key(key);
return v;
}

template<typename T, class K, class V>
inline bool has_key_value(const T& map, const K& key, const V& value) {
typename T::const_iterator i = map.find(key);
return i != map.end() && i->second == value;
const char *v = map.get_tag_by_key(key);
return v && !strcmp(v, value);
}

handler::handler(const std::string& base)
: tmp_nodes_(boost::str(boost::format("tmpnodes-%1%.sqlite") % getpid())),
processed_nodes_(0), processed_ways_(0),
exported_nodes_(0), exported_ways_(0),
base_path_(base), taggable_(false) {
base_path_(base) {

mkdir(base.c_str(), 0755);

Expand Down Expand Up @@ -63,24 +63,6 @@ handler::~handler() {
delete value.second;
}

void handler::start_element(const xml::string& name, const xml::attributes& attr) {
if (name == "node")
start_node(attr);
else if (name == "way")
start_way(attr);
else if (name == "nd")
start_nd(attr);
else if (taggable_ && name == "tag")
start_tag(attr);
}

void handler::end_element(const xml::string& name) {
if (name == "node")
end_node();
else if (name == "way")
end_way();
}

void handler::add_shape(const std::string& name, int type) {
shapes_[name] = new shape_file(base_path_ + "/" + name, type);
if (type == SHPT_POINT)
Expand All @@ -93,30 +75,10 @@ void handler::add_layer(const std::string& name, const std::string& type, const
layers_.push_back(layer(shape, type, subtype));
}

void handler::start_node(const xml::attributes& attr) {
taggable_ = true;
tags_.clear();
id_ = attr.as_int64("id");
x_ = attr.as_double("lon");
y_ = attr.as_double("lat");
}

void handler::start_way(const xml::attributes& attr) {
taggable_ = true;
tags_.clear();
nodes_.clear();
}

void handler::start_nd(const xml::attributes& attr) {
nodes_.push_back(attr.as_int64("ref"));
}

void handler::start_tag(const xml::attributes& attr) {
tags_[attr["k"]] = attr["v"];
}

void handler::end_node() {
taggable_ = false;
void handler::node(const shared_ptr<Osmium::OSM::Node const>& node) {
int64_t id_ = node->id();
double x_ = node->position().lon();
double y_ = node->position().lat();

if (++processed_nodes_ % 100000 == 0)
std::cout << processed_nodes_ << " nodes processed, " << exported_nodes_ << " nodes exported" << std::endl;
Expand All @@ -126,49 +88,47 @@ void handler::end_node() {

tmp_nodes_.set(id_, x_, y_);

tag_map::const_iterator i = tags_.find("name");
if (i == tags_.end())
const char* name = node->tags().get_tag_by_key("name");
if (!name)
return;

foreach (const layer& lay, layers_) {
if (lay.shape()->type() == SHPT_POINT &&
has_key_value(tags_, lay.type(), lay.subtype())) {
has_key_value(node->tags(), lay.type().c_str(), lay.subtype().c_str())) {
lay.shape()->point(x_, y_);
lay.shape()->add_attribute(0, i->second);
lay.shape()->add_attribute(0, name);
++exported_nodes_;
break;
}
}
}

void handler::end_way() {
taggable_ = false;

void handler::way(const shared_ptr<Osmium::OSM::Way>& way) {
if (++processed_ways_ % 10000 == 0)
std::cout << processed_ways_ << " ways processed, " << exported_ways_ << " ways exported" << std::endl;

int type = is_area() ? SHPT_POLYGON : SHPT_ARC;
if ((type == SHPT_POLYGON && nodes_.size() < 3) || nodes_.size() < 2)
int type = is_area(way) ? SHPT_POLYGON : SHPT_ARC;
if ((type == SHPT_POLYGON && way->nodes().size() < 3) || way->nodes().size() < 2)
return;

foreach (const layer& lay, layers_) {
if (lay.shape()->type() == type && has_key_value(tags_, lay.type(), lay.subtype())) {
double x[nodes_.size()], y[nodes_.size()];
if (tmp_nodes_.get(nodes_, x, y)) {
lay.shape()->multipoint(type, nodes_.size(), x, y);
if (lay.shape()->type() == type && has_key_value(way->tags(), lay.type().c_str(), lay.subtype().c_str())) {
double x[way->nodes().size()], y[way->nodes().size()];
if (tmp_nodes_.get(way->nodes(), x, y)) {
lay.shape()->multipoint(type, way->nodes().size(), x, y);
++exported_ways_;
}
break;
}
}
}

bool handler::is_area() {
return has_key_value(tags_, "area", "yes") ||
has_key(tags_, "landuse") ||
has_key_value(tags_, "natural", "land") ||
has_key_value(tags_, "natural", "water") ||
has_key_value(tags_, "natural", "woord");
bool handler::is_area(const shared_ptr<Osmium::OSM::Way>& way) {
return has_key_value(way->tags(), "area", "yes") ||
has_key(way->tags(), "landuse") ||
has_key_value(way->tags(), "natural", "land") ||
has_key_value(way->tags(), "natural", "water") ||
has_key_value(way->tags(), "natural", "woord");
}

}
28 changes: 7 additions & 21 deletions osm/handler.hpp
View file
Expand Up @@ -7,18 +7,15 @@
#include <map>
#include <vector>

namespace xml {

class string;
class attributes;

}
#include <osmium/osm/way.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/handler.hpp>

namespace osm {

class shape_file;

class handler {
class handler: public Osmium::Handler::Base {
typedef std::map<std::string, shape_file*> shape_map;
typedef std::map<std::string, std::string> tag_map;

Expand All @@ -27,33 +24,22 @@ class handler {
handler(const std::string& base);
~handler();

void start_element(const xml::string& name, const xml::attributes& attr);
void end_element(const xml::string& name);
void node(const shared_ptr<Osmium::OSM::Node const>& node);
void way(const shared_ptr<Osmium::OSM::Way>& way);

private:

void add_shape(const std::string& name, int type);
void add_layer(const std::string& name, const std::string& type, const std::string& subtype);
void start_node(const xml::attributes& attr);
void start_way(const xml::attributes& attr);
void start_nd(const xml::attributes& attr);
void start_tag(const xml::attributes& attr);
void end_node();
void end_way();
bool is_area();
bool is_area(const shared_ptr<Osmium::OSM::Way>& way);

point_database tmp_nodes_;
int64_t processed_nodes_;
int64_t processed_ways_;
int64_t exported_nodes_;
int64_t exported_ways_;
std::vector<layer> layers_;
std::vector<int64_t> nodes_;
std::string base_path_;
bool taggable_;
tag_map tags_;
int64_t id_;
double x_, y_;
shape_map shapes_;
};

Expand Down
16 changes: 9 additions & 7 deletions osm/point_database.cc
View file
Expand Up @@ -45,7 +45,7 @@ void point_database::set(int64_t id, double x, double y) {
db_error("step failed");
}

bool point_database::get(const std::vector<int64_t>& ids, double* x_result, double* y_result) {
bool point_database::get(const Osmium::OSM::WayNodeList& ids, double* x_result, double* y_result) {
const int block_size = 128;

int points = ids.size();
Expand All @@ -56,7 +56,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
stmt_wrapper block_stmt, rest_stmt;

int todo = points;
std::vector<int64_t>::const_iterator i = ids.begin();
std::vector<Osmium::OSM::WayNode>::const_iterator i = ids.begin();
while (todo > 0) {
int step_size;
sqlite3_stmt* stmt;
Expand All @@ -74,17 +74,19 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
stmt = rest_stmt.stmt = build_fetch_stmt(todo);
}

for (int n = 1; n <= step_size; ++n)
sqlite3_bind_int64(stmt, n, *i++);
for (int n = 1; n <= step_size; ++n) {
sqlite3_bind_int64(stmt, n, i->ref());
i++;
}

int ret;
while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
sqlite3_int64 found = sqlite3_column_int64(stmt, 0);
double x = sqlite3_column_double(stmt, 1);
double y = sqlite3_column_double(stmt, 2);
int n = 0;
foreach (int64_t id, ids) {
if (id == found) {
foreach (const Osmium::OSM::WayNode &node, ids) {
if (node.ref() == found) {
x_result[n] = x;
y_result[n] = y;
resolved[n] = true;
Expand All @@ -101,7 +103,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub

for (int n = 0; n < points; ++n) {
if (!resolved[n]) {
std::cerr << "unresolved node " << ids.at(n) << std::endl;
std::cerr << "unresolved node " << ids[n].ref() << std::endl;
return false;
}
}
Expand Down
4 changes: 3 additions & 1 deletion osm/point_database.hpp
View file
Expand Up @@ -6,6 +6,8 @@
#include <boost/utility.hpp>
#include <stdexcept>

#include <osmium/osm/way_node_list.hpp>

namespace osm {

struct point_database : boost::noncopyable {
Expand All @@ -17,7 +19,7 @@ struct point_database : boost::noncopyable {
}

void set(int64_t id, double x, double y);
bool get(const std::vector<int64_t>& ids, double* x_result, double* y_result);
bool get(const Osmium::OSM::WayNodeList& way_node_list, double* x_result, double* y_result);

private:

Expand Down
10 changes: 7 additions & 3 deletions osm2shp.cc
View file
@@ -1,15 +1,19 @@
#include "osm/handler.hpp"
#include "xml.hpp"

#include <osmium.hpp>

int main(int argc, char* argv[]) {
try {
if (argc != 3) {
std::cerr << "usage: " << argv[0] << " planet.osm(.gz|.bz2) base-path" << std::endl;
return 1;
}

Osmium::init(true);

Osmium::OSMFile infile(argv[1]);
osm::handler handler(argv[2]);
xml::parser<osm::handler> parser(handler);
parser.parse(argv[1]);
infile.read(handler);
return 0;
} catch (const std::exception& ex) {
std::cerr << ex.what() << std::endl;
Expand Down
21 changes: 0 additions & 21 deletions xml.cc
View file

This file was deleted.

0 comments on commit 58e68c5

Please sign in to comment.