Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

osm2shp: Use Osmium as input handler instead of our own XML/expat imp…

…lementation

This enables us to read also PBF files
  • Loading branch information...
commit fe3b92ca1dfc5c1ab315c358aeff859c7388c2b1 1 parent 713dea2
@Turbo87 Turbo87 authored
View
9 Makefile
@@ -1,18 +1,21 @@
CPP=g++
CC=gcc
-CFLAGS=-O2 -Wall
+CFLAGS=-O2 -Wall -Wredundant-decls
all: osm2shp
+LIB_GEOS = $(shell geos-config --libs)
+LIB_SHAPE = -lshp $(LIB_GEOS)
+LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf
+
FILES = \
- xml.o \
osm2shp.o \
osm/shapefile.o \
osm/handler.o \
osm/point_database.o
osm2shp: $(FILES)
- $(CPP) $(CFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams -o $@
+ $(CPP) $(CFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams $(LIB_PROTOBUF) $(LIB_SHAPE) -o $@
%.o: %.cc
$(CPP) $(CFLAGS) -c $< -o $@
View
96 osm/handler.cc
@@ -1,6 +1,5 @@
#include "handler.hpp"
#include "shapefile.hpp"
-#include "../xml.hpp"
#include <shapefil.h>
#include <sys/stat.h>
@@ -11,22 +10,23 @@
namespace osm {
-template<typename T>
-inline bool has_key(const T& map, const typename T::key_type& key) {
- return map.find(key) != map.end();
+template<typename T, class K>
+inline bool has_key(const T& map, const K& key) {
+ const char *v = map.get_tag_by_key(key);
+ return v;
}
template<typename T, class K, class V>
inline bool has_key_value(const T& map, const K& key, const V& value) {
- typename T::const_iterator i = map.find(key);
- return i != map.end() && i->second == value;
+ const char *v = map.get_tag_by_key(key);
+ return v && !strcmp(v, value);
}
handler::handler(const std::string& base)
: tmp_nodes_(boost::str(boost::format("tmpnodes-%1%.sqlite") % getpid())),
processed_nodes_(0), processed_ways_(0),
exported_nodes_(0), exported_ways_(0),
- base_path_(base), taggable_(false) {
+ base_path_(base) {
mkdir(base.c_str(), 0755);
@@ -63,24 +63,6 @@ handler::~handler() {
delete value.second;
}
-void handler::start_element(const xml::string& name, const xml::attributes& attr) {
- if (name == "node")
- start_node(attr);
- else if (name == "way")
- start_way(attr);
- else if (name == "nd")
- start_nd(attr);
- else if (taggable_ && name == "tag")
- start_tag(attr);
-}
-
-void handler::end_element(const xml::string& name) {
- if (name == "node")
- end_node();
- else if (name == "way")
- end_way();
-}
-
void handler::add_shape(const std::string& name, int type) {
shapes_[name] = new shape_file(base_path_ + "/" + name, type);
if (type == SHPT_POINT)
@@ -93,30 +75,10 @@ void handler::add_layer(const std::string& name, const std::string& type, const
layers_.push_back(layer(shape, type, subtype));
}
-void handler::start_node(const xml::attributes& attr) {
- taggable_ = true;
- tags_.clear();
- id_ = attr.as_int64("id");
- x_ = attr.as_double("lon");
- y_ = attr.as_double("lat");
-}
-
-void handler::start_way(const xml::attributes& attr) {
- taggable_ = true;
- tags_.clear();
- nodes_.clear();
-}
-
-void handler::start_nd(const xml::attributes& attr) {
- nodes_.push_back(attr.as_int64("ref"));
-}
-
-void handler::start_tag(const xml::attributes& attr) {
- tags_[attr["k"]] = attr["v"];
-}
-
-void handler::end_node() {
- taggable_ = false;
+void handler::node(const shared_ptr<Osmium::OSM::Node const>& node) {
+ int64_t id_ = node->id();
+ double x_ = node->position().lon();
+ double y_ = node->position().lat();
if (++processed_nodes_ % 100000 == 0)
std::cout << processed_nodes_ << " nodes processed, " << exported_nodes_ << " nodes exported" << std::endl;
@@ -126,36 +88,34 @@ void handler::end_node() {
tmp_nodes_.set(id_, x_, y_);
- tag_map::const_iterator i = tags_.find("name");
- if (i == tags_.end())
+ const char* name = node->tags().get_tag_by_key("name");
+ if (!name)
return;
foreach (const layer& lay, layers_) {
if (lay.shape()->type() == SHPT_POINT &&
- has_key_value(tags_, lay.type(), lay.subtype())) {
+ has_key_value(node->tags(), lay.type().c_str(), lay.subtype().c_str())) {
lay.shape()->point(x_, y_);
- lay.shape()->add_attribute(0, i->second);
+ lay.shape()->add_attribute(0, name);
++exported_nodes_;
break;
}
}
}
-void handler::end_way() {
- taggable_ = false;
-
+void handler::way(const shared_ptr<Osmium::OSM::Way>& way) {
if (++processed_ways_ % 10000 == 0)
std::cout << processed_ways_ << " ways processed, " << exported_ways_ << " ways exported" << std::endl;
- int type = is_area() ? SHPT_POLYGON : SHPT_ARC;
- if ((type == SHPT_POLYGON && nodes_.size() < 3) || nodes_.size() < 2)
+ int type = is_area(way) ? SHPT_POLYGON : SHPT_ARC;
+ if ((type == SHPT_POLYGON && way->nodes().size() < 3) || way->nodes().size() < 2)
return;
foreach (const layer& lay, layers_) {
- if (lay.shape()->type() == type && has_key_value(tags_, lay.type(), lay.subtype())) {
- double x[nodes_.size()], y[nodes_.size()];
- if (tmp_nodes_.get(nodes_, x, y)) {
- lay.shape()->multipoint(type, nodes_.size(), x, y);
+ if (lay.shape()->type() == type && has_key_value(way->tags(), lay.type().c_str(), lay.subtype().c_str())) {
+ double x[way->nodes().size()], y[way->nodes().size()];
+ if (tmp_nodes_.get(way->nodes(), x, y)) {
+ lay.shape()->multipoint(type, way->nodes().size(), x, y);
++exported_ways_;
}
break;
@@ -163,12 +123,12 @@ void handler::end_way() {
}
}
-bool handler::is_area() {
- return has_key_value(tags_, "area", "yes") ||
- has_key(tags_, "landuse") ||
- has_key_value(tags_, "natural", "land") ||
- has_key_value(tags_, "natural", "water") ||
- has_key_value(tags_, "natural", "woord");
+bool handler::is_area(const shared_ptr<Osmium::OSM::Way>& way) {
+ return has_key_value(way->tags(), "area", "yes") ||
+ has_key(way->tags(), "landuse") ||
+ has_key_value(way->tags(), "natural", "land") ||
+ has_key_value(way->tags(), "natural", "water") ||
+ has_key_value(way->tags(), "natural", "woord");
}
}
View
28 osm/handler.hpp
@@ -7,18 +7,15 @@
#include <map>
#include <vector>
-namespace xml {
-
-class string;
-class attributes;
-
-}
+#include <osmium/osm/way.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/handler.hpp>
namespace osm {
class shape_file;
-class handler {
+class handler: public Osmium::Handler::Base {
typedef std::map<std::string, shape_file*> shape_map;
typedef std::map<std::string, std::string> tag_map;
@@ -27,20 +24,14 @@ class handler {
handler(const std::string& base);
~handler();
- void start_element(const xml::string& name, const xml::attributes& attr);
- void end_element(const xml::string& name);
+ void node(const shared_ptr<Osmium::OSM::Node const>& node);
+ void way(const shared_ptr<Osmium::OSM::Way>& way);
private:
void add_shape(const std::string& name, int type);
void add_layer(const std::string& name, const std::string& type, const std::string& subtype);
- void start_node(const xml::attributes& attr);
- void start_way(const xml::attributes& attr);
- void start_nd(const xml::attributes& attr);
- void start_tag(const xml::attributes& attr);
- void end_node();
- void end_way();
- bool is_area();
+ bool is_area(const shared_ptr<Osmium::OSM::Way>& way);
point_database tmp_nodes_;
int64_t processed_nodes_;
@@ -48,12 +39,7 @@ class handler {
int64_t exported_nodes_;
int64_t exported_ways_;
std::vector<layer> layers_;
- std::vector<int64_t> nodes_;
std::string base_path_;
- bool taggable_;
- tag_map tags_;
- int64_t id_;
- double x_, y_;
shape_map shapes_;
};
View
16 osm/point_database.cc
@@ -45,7 +45,7 @@ void point_database::set(int64_t id, double x, double y) {
db_error("step failed");
}
-bool point_database::get(const std::vector<int64_t>& ids, double* x_result, double* y_result) {
+bool point_database::get(const Osmium::OSM::WayNodeList& ids, double* x_result, double* y_result) {
const int block_size = 128;
int points = ids.size();
@@ -56,7 +56,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
stmt_wrapper block_stmt, rest_stmt;
int todo = points;
- std::vector<int64_t>::const_iterator i = ids.begin();
+ std::vector<Osmium::OSM::WayNode>::const_iterator i = ids.begin();
while (todo > 0) {
int step_size;
sqlite3_stmt* stmt;
@@ -74,8 +74,10 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
stmt = rest_stmt.stmt = build_fetch_stmt(todo);
}
- for (int n = 1; n <= step_size; ++n)
- sqlite3_bind_int64(stmt, n, *i++);
+ for (int n = 1; n <= step_size; ++n) {
+ sqlite3_bind_int64(stmt, n, i->ref());
+ i++;
+ }
int ret;
while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
@@ -83,8 +85,8 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
double x = sqlite3_column_double(stmt, 1);
double y = sqlite3_column_double(stmt, 2);
int n = 0;
- foreach (int64_t id, ids) {
- if (id == found) {
+ foreach (const Osmium::OSM::WayNode &node, ids) {
+ if (node.ref() == found) {
x_result[n] = x;
y_result[n] = y;
resolved[n] = true;
@@ -101,7 +103,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
for (int n = 0; n < points; ++n) {
if (!resolved[n]) {
- std::cerr << "unresolved node " << ids.at(n) << std::endl;
+ std::cerr << "unresolved node " << ids[n].ref() << std::endl;
return false;
}
}
View
4 osm/point_database.hpp
@@ -6,6 +6,8 @@
#include <boost/utility.hpp>
#include <stdexcept>
+#include <osmium/osm/way_node_list.hpp>
+
namespace osm {
struct point_database : boost::noncopyable {
@@ -17,7 +19,7 @@ struct point_database : boost::noncopyable {
}
void set(int64_t id, double x, double y);
- bool get(const std::vector<int64_t>& ids, double* x_result, double* y_result);
+ bool get(const Osmium::OSM::WayNodeList& way_node_list, double* x_result, double* y_result);
private:
View
10 osm2shp.cc
@@ -1,5 +1,6 @@
#include "osm/handler.hpp"
-#include "xml.hpp"
+
+#include <osmium.hpp>
int main(int argc, char* argv[]) {
try {
@@ -7,9 +8,12 @@ int main(int argc, char* argv[]) {
std::cerr << "usage: " << argv[0] << " planet.osm(.gz|.bz2) base-path" << std::endl;
return 1;
}
+
+ Osmium::init(true);
+
+ Osmium::OSMFile infile(argv[1]);
osm::handler handler(argv[2]);
- xml::parser<osm::handler> parser(handler);
- parser.parse(argv[1]);
+ infile.read(handler);
return 0;
} catch (const std::exception& ex) {
std::cerr << ex.what() << std::endl;
View
21 xml.cc
@@ -1,21 +0,0 @@
-#include "xml.hpp"
-
-namespace xml {
-
-const char* attributes::get(const xml::string& key) const {
- for (int i = 0; attr_[i]; i += 2) {
- if (key == attr_[i])
- return attr_[i + 1];
- }
- throw std::runtime_error(std::string(key.c_str()) + " not found");
-}
-
-int64_t attributes::as_int64(const xml::string& key) const {
- return atoll(get(key));
-}
-
-double attributes::as_double(const xml::string& key) const {
- return atof(get(key));
-}
-
-} // namespace xml
View
140 xml.hpp
@@ -1,140 +0,0 @@
-#ifndef OSM2SHP_XML_HPP
-#define OSM2SHP_XML_HPP
-
-#include <expat.h>
-#include <fstream>
-#include <boost/iostreams/filter/gzip.hpp>
-#include <boost/iostreams/filter/bzip2.hpp>
-#include <boost/iostreams/filtering_stream.hpp>
-
-namespace xml {
-
-class parse_error: public std::runtime_error {
-public:
- parse_error(const std::string& what, int where)
- : std::runtime_error(what), where_(where) {
- }
-
- int where() const {
- return where_;
- }
-
-private:
- int where_;
-};
-
-class string {
-public:
-
- string(const char* str)
- : str_(str) {
- }
-
- bool operator==(const char* s) const {
- return !strcmp(str_, s);
- }
-
- const char* c_str() const {
- return str_;
- }
-
-private:
- const char* str_;
-};
-
-class attributes {
-public:
-
- explicit attributes(const char** attr)
- : attr_(attr) {
- }
-
- const char* operator[](const xml::string& key) const {
- return get(key);
- }
-
- const char* get(const xml::string& key) const;
- int64_t as_int64(const xml::string& key) const;
- double as_double(const xml::string& key) const;
-
-private:
- const char** attr_;
-};
-
-template<class Handler>
-class parser : public boost::noncopyable {
-public:
- explicit parser(Handler& handler);
- ~parser();
-
- void parse(std::istream& in);
- void parse(const std::string& name);
-
-private:
-
- static void start_element(void* data, const char* name, const char** attr);
- static void end_element(void* data, const char* name);
-
- XML_Parser parser_;
-};
-
-template<class Handler>
-parser<Handler>::parser(Handler& handler)
- : parser_(XML_ParserCreate(0)) {
- if (!parser_)
- throw std::runtime_error("Could not allocate parser");
- XML_SetUserData(parser_, &handler);
- XML_SetElementHandler(parser_, start_element, end_element);
-}
-
-template<class Handler>
-parser<Handler>::~parser() {
- XML_ParserFree(parser_);
-}
-
-template<class Handler>
-void parser<Handler>::parse(std::istream& in) {
- char buf[64*1024];
- while (!in.eof()) {
- in.read(buf, sizeof (buf));
- if (!XML_Parse(parser_, buf, in.gcount(), 0)) {
- throw parse_error(XML_ErrorString(XML_GetErrorCode(parser_)),
- XML_GetCurrentLineNumber(parser_));
- }
- }
- XML_Parse(parser_, 0, 0, 1);
-}
-
-template<class Handler>
-void parser<Handler>::parse(const std::string& name) {
- std::ifstream file(name.c_str(), std::ios::in | std::ios::binary);
- if (!file.good())
- throw std::runtime_error("failed to open file " + name);
- if (name.rfind(".gz") == name.size() - 3) {
- boost::iostreams::filtering_stream<boost::iostreams::input> in;
- in.push(boost::iostreams::gzip_decompressor());
- in.push(file);
- parse(in);
- } else if (name.rfind(".bz2") == name.size() - 4) {
- boost::iostreams::filtering_stream<boost::iostreams::input> in;
- in.push(boost::iostreams::bzip2_decompressor());
- in.push(file);
- parse(in);
- } else {
- parse(file);
- }
-}
-
-template<class Handler>
-void parser<Handler>::start_element(void* data, const char* name, const char** attr) {
- static_cast<Handler*>(data)->start_element(string(name), attributes(attr));
-}
-
-template<class Handler>
-void parser<Handler>::end_element(void* data, const char* name) {
- static_cast<Handler*>(data)->end_element(string(name));
-}
-
-} // namespace xml
-
-#endif

0 comments on commit fe3b92c

Please sign in to comment.
Something went wrong with that request. Please try again.