Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

osm2shp: Use Osmium as input handler instead of our own XML/expat imp…

…lementation

This enables us to read also PBF files
  • Loading branch information...
commit fe3b92ca1dfc5c1ab315c358aeff859c7388c2b1 1 parent 713dea2
Tobias Bieniek Turbo87 authored
9 Makefile
... ... @@ -1,18 +1,21 @@
1 1 CPP=g++
2 2 CC=gcc
3   -CFLAGS=-O2 -Wall
  3 +CFLAGS=-O2 -Wall -Wredundant-decls
4 4
5 5 all: osm2shp
6 6
  7 +LIB_GEOS = $(shell geos-config --libs)
  8 +LIB_SHAPE = -lshp $(LIB_GEOS)
  9 +LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf
  10 +
7 11 FILES = \
8   - xml.o \
9 12 osm2shp.o \
10 13 osm/shapefile.o \
11 14 osm/handler.o \
12 15 osm/point_database.o
13 16
14 17 osm2shp: $(FILES)
15   - $(CPP) $(CFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams -o $@
  18 + $(CPP) $(CFLAGS) $+ -lexpat -lsqlite3 -lshp -lboost_iostreams $(LIB_PROTOBUF) $(LIB_SHAPE) -o $@
16 19
17 20 %.o: %.cc
18 21 $(CPP) $(CFLAGS) -c $< -o $@
96 osm/handler.cc
... ... @@ -1,6 +1,5 @@
1 1 #include "handler.hpp"
2 2 #include "shapefile.hpp"
3   -#include "../xml.hpp"
4 3
5 4 #include <shapefil.h>
6 5 #include <sys/stat.h>
@@ -11,22 +10,23 @@
11 10
12 11 namespace osm {
13 12
14   -template<typename T>
15   -inline bool has_key(const T& map, const typename T::key_type& key) {
16   - return map.find(key) != map.end();
  13 +template<typename T, class K>
  14 +inline bool has_key(const T& map, const K& key) {
  15 + const char *v = map.get_tag_by_key(key);
  16 + return v;
17 17 }
18 18
19 19 template<typename T, class K, class V>
20 20 inline bool has_key_value(const T& map, const K& key, const V& value) {
21   - typename T::const_iterator i = map.find(key);
22   - return i != map.end() && i->second == value;
  21 + const char *v = map.get_tag_by_key(key);
  22 + return v && !strcmp(v, value);
23 23 }
24 24
25 25 handler::handler(const std::string& base)
26 26 : tmp_nodes_(boost::str(boost::format("tmpnodes-%1%.sqlite") % getpid())),
27 27 processed_nodes_(0), processed_ways_(0),
28 28 exported_nodes_(0), exported_ways_(0),
29   - base_path_(base), taggable_(false) {
  29 + base_path_(base) {
30 30
31 31 mkdir(base.c_str(), 0755);
32 32
@@ -63,24 +63,6 @@ handler::~handler() {
63 63 delete value.second;
64 64 }
65 65
66   -void handler::start_element(const xml::string& name, const xml::attributes& attr) {
67   - if (name == "node")
68   - start_node(attr);
69   - else if (name == "way")
70   - start_way(attr);
71   - else if (name == "nd")
72   - start_nd(attr);
73   - else if (taggable_ && name == "tag")
74   - start_tag(attr);
75   -}
76   -
77   -void handler::end_element(const xml::string& name) {
78   - if (name == "node")
79   - end_node();
80   - else if (name == "way")
81   - end_way();
82   -}
83   -
84 66 void handler::add_shape(const std::string& name, int type) {
85 67 shapes_[name] = new shape_file(base_path_ + "/" + name, type);
86 68 if (type == SHPT_POINT)
@@ -93,30 +75,10 @@ void handler::add_layer(const std::string& name, const std::string& type, const
93 75 layers_.push_back(layer(shape, type, subtype));
94 76 }
95 77
96   -void handler::start_node(const xml::attributes& attr) {
97   - taggable_ = true;
98   - tags_.clear();
99   - id_ = attr.as_int64("id");
100   - x_ = attr.as_double("lon");
101   - y_ = attr.as_double("lat");
102   -}
103   -
104   -void handler::start_way(const xml::attributes& attr) {
105   - taggable_ = true;
106   - tags_.clear();
107   - nodes_.clear();
108   -}
109   -
110   -void handler::start_nd(const xml::attributes& attr) {
111   - nodes_.push_back(attr.as_int64("ref"));
112   -}
113   -
114   -void handler::start_tag(const xml::attributes& attr) {
115   - tags_[attr["k"]] = attr["v"];
116   -}
117   -
118   -void handler::end_node() {
119   - taggable_ = false;
  78 +void handler::node(const shared_ptr<Osmium::OSM::Node const>& node) {
  79 + int64_t id_ = node->id();
  80 + double x_ = node->position().lon();
  81 + double y_ = node->position().lat();
120 82
121 83 if (++processed_nodes_ % 100000 == 0)
122 84 std::cout << processed_nodes_ << " nodes processed, " << exported_nodes_ << " nodes exported" << std::endl;
@@ -126,36 +88,34 @@ void handler::end_node() {
126 88
127 89 tmp_nodes_.set(id_, x_, y_);
128 90
129   - tag_map::const_iterator i = tags_.find("name");
130   - if (i == tags_.end())
  91 + const char* name = node->tags().get_tag_by_key("name");
  92 + if (!name)
131 93 return;
132 94
133 95 foreach (const layer& lay, layers_) {
134 96 if (lay.shape()->type() == SHPT_POINT &&
135   - has_key_value(tags_, lay.type(), lay.subtype())) {
  97 + has_key_value(node->tags(), lay.type().c_str(), lay.subtype().c_str())) {
136 98 lay.shape()->point(x_, y_);
137   - lay.shape()->add_attribute(0, i->second);
  99 + lay.shape()->add_attribute(0, name);
138 100 ++exported_nodes_;
139 101 break;
140 102 }
141 103 }
142 104 }
143 105
144   -void handler::end_way() {
145   - taggable_ = false;
146   -
  106 +void handler::way(const shared_ptr<Osmium::OSM::Way>& way) {
147 107 if (++processed_ways_ % 10000 == 0)
148 108 std::cout << processed_ways_ << " ways processed, " << exported_ways_ << " ways exported" << std::endl;
149 109
150   - int type = is_area() ? SHPT_POLYGON : SHPT_ARC;
151   - if ((type == SHPT_POLYGON && nodes_.size() < 3) || nodes_.size() < 2)
  110 + int type = is_area(way) ? SHPT_POLYGON : SHPT_ARC;
  111 + if ((type == SHPT_POLYGON && way->nodes().size() < 3) || way->nodes().size() < 2)
152 112 return;
153 113
154 114 foreach (const layer& lay, layers_) {
155   - if (lay.shape()->type() == type && has_key_value(tags_, lay.type(), lay.subtype())) {
156   - double x[nodes_.size()], y[nodes_.size()];
157   - if (tmp_nodes_.get(nodes_, x, y)) {
158   - lay.shape()->multipoint(type, nodes_.size(), x, y);
  115 + if (lay.shape()->type() == type && has_key_value(way->tags(), lay.type().c_str(), lay.subtype().c_str())) {
  116 + double x[way->nodes().size()], y[way->nodes().size()];
  117 + if (tmp_nodes_.get(way->nodes(), x, y)) {
  118 + lay.shape()->multipoint(type, way->nodes().size(), x, y);
159 119 ++exported_ways_;
160 120 }
161 121 break;
@@ -163,12 +123,12 @@ void handler::end_way() {
163 123 }
164 124 }
165 125
166   -bool handler::is_area() {
167   - return has_key_value(tags_, "area", "yes") ||
168   - has_key(tags_, "landuse") ||
169   - has_key_value(tags_, "natural", "land") ||
170   - has_key_value(tags_, "natural", "water") ||
171   - has_key_value(tags_, "natural", "woord");
  126 +bool handler::is_area(const shared_ptr<Osmium::OSM::Way>& way) {
  127 + return has_key_value(way->tags(), "area", "yes") ||
  128 + has_key(way->tags(), "landuse") ||
  129 + has_key_value(way->tags(), "natural", "land") ||
  130 + has_key_value(way->tags(), "natural", "water") ||
  131 + has_key_value(way->tags(), "natural", "woord");
172 132 }
173 133
174 134 }
28 osm/handler.hpp
@@ -7,18 +7,15 @@
7 7 #include <map>
8 8 #include <vector>
9 9
10   -namespace xml {
11   -
12   -class string;
13   -class attributes;
14   -
15   -}
  10 +#include <osmium/osm/way.hpp>
  11 +#include <osmium/osm/node.hpp>
  12 +#include <osmium/handler.hpp>
16 13
17 14 namespace osm {
18 15
19 16 class shape_file;
20 17
21   -class handler {
  18 +class handler: public Osmium::Handler::Base {
22 19 typedef std::map<std::string, shape_file*> shape_map;
23 20 typedef std::map<std::string, std::string> tag_map;
24 21
@@ -27,20 +24,14 @@ class handler {
27 24 handler(const std::string& base);
28 25 ~handler();
29 26
30   - void start_element(const xml::string& name, const xml::attributes& attr);
31   - void end_element(const xml::string& name);
  27 + void node(const shared_ptr<Osmium::OSM::Node const>& node);
  28 + void way(const shared_ptr<Osmium::OSM::Way>& way);
32 29
33 30 private:
34 31
35 32 void add_shape(const std::string& name, int type);
36 33 void add_layer(const std::string& name, const std::string& type, const std::string& subtype);
37   - void start_node(const xml::attributes& attr);
38   - void start_way(const xml::attributes& attr);
39   - void start_nd(const xml::attributes& attr);
40   - void start_tag(const xml::attributes& attr);
41   - void end_node();
42   - void end_way();
43   - bool is_area();
  34 + bool is_area(const shared_ptr<Osmium::OSM::Way>& way);
44 35
45 36 point_database tmp_nodes_;
46 37 int64_t processed_nodes_;
@@ -48,12 +39,7 @@ class handler {
48 39 int64_t exported_nodes_;
49 40 int64_t exported_ways_;
50 41 std::vector<layer> layers_;
51   - std::vector<int64_t> nodes_;
52 42 std::string base_path_;
53   - bool taggable_;
54   - tag_map tags_;
55   - int64_t id_;
56   - double x_, y_;
57 43 shape_map shapes_;
58 44 };
59 45
16 osm/point_database.cc
@@ -45,7 +45,7 @@ void point_database::set(int64_t id, double x, double y) {
45 45 db_error("step failed");
46 46 }
47 47
48   -bool point_database::get(const std::vector<int64_t>& ids, double* x_result, double* y_result) {
  48 +bool point_database::get(const Osmium::OSM::WayNodeList& ids, double* x_result, double* y_result) {
49 49 const int block_size = 128;
50 50
51 51 int points = ids.size();
@@ -56,7 +56,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
56 56 stmt_wrapper block_stmt, rest_stmt;
57 57
58 58 int todo = points;
59   - std::vector<int64_t>::const_iterator i = ids.begin();
  59 + std::vector<Osmium::OSM::WayNode>::const_iterator i = ids.begin();
60 60 while (todo > 0) {
61 61 int step_size;
62 62 sqlite3_stmt* stmt;
@@ -74,8 +74,10 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
74 74 stmt = rest_stmt.stmt = build_fetch_stmt(todo);
75 75 }
76 76
77   - for (int n = 1; n <= step_size; ++n)
78   - sqlite3_bind_int64(stmt, n, *i++);
  77 + for (int n = 1; n <= step_size; ++n) {
  78 + sqlite3_bind_int64(stmt, n, i->ref());
  79 + i++;
  80 + }
79 81
80 82 int ret;
81 83 while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
@@ -83,8 +85,8 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
83 85 double x = sqlite3_column_double(stmt, 1);
84 86 double y = sqlite3_column_double(stmt, 2);
85 87 int n = 0;
86   - foreach (int64_t id, ids) {
87   - if (id == found) {
  88 + foreach (const Osmium::OSM::WayNode &node, ids) {
  89 + if (node.ref() == found) {
88 90 x_result[n] = x;
89 91 y_result[n] = y;
90 92 resolved[n] = true;
@@ -101,7 +103,7 @@ bool point_database::get(const std::vector<int64_t>& ids, double* x_result, doub
101 103
102 104 for (int n = 0; n < points; ++n) {
103 105 if (!resolved[n]) {
104   - std::cerr << "unresolved node " << ids.at(n) << std::endl;
  106 + std::cerr << "unresolved node " << ids[n].ref() << std::endl;
105 107 return false;
106 108 }
107 109 }
4 osm/point_database.hpp
@@ -6,6 +6,8 @@
6 6 #include <boost/utility.hpp>
7 7 #include <stdexcept>
8 8
  9 +#include <osmium/osm/way_node_list.hpp>
  10 +
9 11 namespace osm {
10 12
11 13 struct point_database : boost::noncopyable {
@@ -17,7 +19,7 @@ struct point_database : boost::noncopyable {
17 19 }
18 20
19 21 void set(int64_t id, double x, double y);
20   - bool get(const std::vector<int64_t>& ids, double* x_result, double* y_result);
  22 + bool get(const Osmium::OSM::WayNodeList& way_node_list, double* x_result, double* y_result);
21 23
22 24 private:
23 25
10 osm2shp.cc
... ... @@ -1,5 +1,6 @@
1 1 #include "osm/handler.hpp"
2   -#include "xml.hpp"
  2 +
  3 +#include <osmium.hpp>
3 4
4 5 int main(int argc, char* argv[]) {
5 6 try {
@@ -7,9 +8,12 @@ int main(int argc, char* argv[]) {
7 8 std::cerr << "usage: " << argv[0] << " planet.osm(.gz|.bz2) base-path" << std::endl;
8 9 return 1;
9 10 }
  11 +
  12 + Osmium::init(true);
  13 +
  14 + Osmium::OSMFile infile(argv[1]);
10 15 osm::handler handler(argv[2]);
11   - xml::parser<osm::handler> parser(handler);
12   - parser.parse(argv[1]);
  16 + infile.read(handler);
13 17 return 0;
14 18 } catch (const std::exception& ex) {
15 19 std::cerr << ex.what() << std::endl;
21 xml.cc
... ... @@ -1,21 +0,0 @@
1   -#include "xml.hpp"
2   -
3   -namespace xml {
4   -
5   -const char* attributes::get(const xml::string& key) const {
6   - for (int i = 0; attr_[i]; i += 2) {
7   - if (key == attr_[i])
8   - return attr_[i + 1];
9   - }
10   - throw std::runtime_error(std::string(key.c_str()) + " not found");
11   -}
12   -
13   -int64_t attributes::as_int64(const xml::string& key) const {
14   - return atoll(get(key));
15   -}
16   -
17   -double attributes::as_double(const xml::string& key) const {
18   - return atof(get(key));
19   -}
20   -
21   -} // namespace xml
140 xml.hpp
... ... @@ -1,140 +0,0 @@
1   -#ifndef OSM2SHP_XML_HPP
2   -#define OSM2SHP_XML_HPP
3   -
4   -#include <expat.h>
5   -#include <fstream>
6   -#include <boost/iostreams/filter/gzip.hpp>
7   -#include <boost/iostreams/filter/bzip2.hpp>
8   -#include <boost/iostreams/filtering_stream.hpp>
9   -
10   -namespace xml {
11   -
12   -class parse_error: public std::runtime_error {
13   -public:
14   - parse_error(const std::string& what, int where)
15   - : std::runtime_error(what), where_(where) {
16   - }
17   -
18   - int where() const {
19   - return where_;
20   - }
21   -
22   -private:
23   - int where_;
24   -};
25   -
26   -class string {
27   -public:
28   -
29   - string(const char* str)
30   - : str_(str) {
31   - }
32   -
33   - bool operator==(const char* s) const {
34   - return !strcmp(str_, s);
35   - }
36   -
37   - const char* c_str() const {
38   - return str_;
39   - }
40   -
41   -private:
42   - const char* str_;
43   -};
44   -
45   -class attributes {
46   -public:
47   -
48   - explicit attributes(const char** attr)
49   - : attr_(attr) {
50   - }
51   -
52   - const char* operator[](const xml::string& key) const {
53   - return get(key);
54   - }
55   -
56   - const char* get(const xml::string& key) const;
57   - int64_t as_int64(const xml::string& key) const;
58   - double as_double(const xml::string& key) const;
59   -
60   -private:
61   - const char** attr_;
62   -};
63   -
64   -template<class Handler>
65   -class parser : public boost::noncopyable {
66   -public:
67   - explicit parser(Handler& handler);
68   - ~parser();
69   -
70   - void parse(std::istream& in);
71   - void parse(const std::string& name);
72   -
73   -private:
74   -
75   - static void start_element(void* data, const char* name, const char** attr);
76   - static void end_element(void* data, const char* name);
77   -
78   - XML_Parser parser_;
79   -};
80   -
81   -template<class Handler>
82   -parser<Handler>::parser(Handler& handler)
83   - : parser_(XML_ParserCreate(0)) {
84   - if (!parser_)
85   - throw std::runtime_error("Could not allocate parser");
86   - XML_SetUserData(parser_, &handler);
87   - XML_SetElementHandler(parser_, start_element, end_element);
88   -}
89   -
90   -template<class Handler>
91   -parser<Handler>::~parser() {
92   - XML_ParserFree(parser_);
93   -}
94   -
95   -template<class Handler>
96   -void parser<Handler>::parse(std::istream& in) {
97   - char buf[64*1024];
98   - while (!in.eof()) {
99   - in.read(buf, sizeof (buf));
100   - if (!XML_Parse(parser_, buf, in.gcount(), 0)) {
101   - throw parse_error(XML_ErrorString(XML_GetErrorCode(parser_)),
102   - XML_GetCurrentLineNumber(parser_));
103   - }
104   - }
105   - XML_Parse(parser_, 0, 0, 1);
106   -}
107   -
108   -template<class Handler>
109   -void parser<Handler>::parse(const std::string& name) {
110   - std::ifstream file(name.c_str(), std::ios::in | std::ios::binary);
111   - if (!file.good())
112   - throw std::runtime_error("failed to open file " + name);
113   - if (name.rfind(".gz") == name.size() - 3) {
114   - boost::iostreams::filtering_stream<boost::iostreams::input> in;
115   - in.push(boost::iostreams::gzip_decompressor());
116   - in.push(file);
117   - parse(in);
118   - } else if (name.rfind(".bz2") == name.size() - 4) {
119   - boost::iostreams::filtering_stream<boost::iostreams::input> in;
120   - in.push(boost::iostreams::bzip2_decompressor());
121   - in.push(file);
122   - parse(in);
123   - } else {
124   - parse(file);
125   - }
126   -}
127   -
128   -template<class Handler>
129   -void parser<Handler>::start_element(void* data, const char* name, const char** attr) {
130   - static_cast<Handler*>(data)->start_element(string(name), attributes(attr));
131   -}
132   -
133   -template<class Handler>
134   -void parser<Handler>::end_element(void* data, const char* name) {
135   - static_cast<Handler*>(data)->end_element(string(name));
136   -}
137   -
138   -} // namespace xml
139   -
140   -#endif

0 comments on commit fe3b92c

Please sign in to comment.
Something went wrong with that request. Please try again.