(Non-multipolygon) relation support (#360)

docs/RELATIONS.md

@@ -0,0 +1,66 @@
+## Relations
+
+Tilemaker has (as yet not complete) support for reading relations in the Lua process scripts. This means you can support route and boundary relations when creating your vector tiles.
+
+Note that relation support is in its early stages and behaviour may change between point versions.
+
+
+### Multipolygon relations
+
+Multipolygon relations are supported natively by tilemaker; you do not need to write special Lua code for them. When a multipolygon is read, tilemaker constructs the geometry as normal, and passes the tags to `way_function` just as it would a simple area.
+
+
+### Reading relation memberships
+
+You can set your Lua script so that `way_function` is able to access the relations that the way is a member of. You would use this, for example, if you wanted to read road numbers from a relation, or to colour roads that are part of a bus route differently.
+
+This is a two-stage process: first, when reading relations, indicate that these should be considered ("accepted"); then, when reading ways in `way_function`, you can access the relation tags.
+
+#### Stage 1: accepting relations
+
+To define which relations should be accepted, add a `relation_scan_function`:
+
+    function relation_scan_function(relation)
+      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
+        local network = relation:Find("network")
+        if network=="ncn" then relation:Accept() end
+      end
+    end
+
+This function takes the relation as its sole argument. Examine the tags using `relation:Find(key)` as normal. (You can also use `relation:Holds(key)` and `relation:Id()`.) If you want to use this relation, call `relation:Accept()`.
+
+#### Stage 2: accessing relations from ways
+
+Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`way:NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `way:FindInRelation(key)`. For example:
+
+    while true do
+      local rel = way:NextRelation()
+      if not rel then break end
+      print ("Part of route "..way:FindInRelation("ref"))
+    end
+
+
+### Writing relation geometries
+
+You can also construct complete multi-linestring geometries from relations. Use this if, for example, you want a geometry for a bike or bus route that you can show at lower zoom levels, or draw with a continuous pattern.
+
+First, make sure that you have accepted the relations using `relation_scan_function` as above.
+
+Then write a `relation_function`, which works in the same way as `way_function` would:
+
+    function relation_function(relation)
+      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
+        relation:Layer("bike_routes", false)
+        relation:Attribute("class", relation:Find("network"))
+        relation:Attribute("ref", relation:Find("ref"))
+      end
+    end
+
+
+### Not supported
+
+Tilemaker does not yet support:
+
+- relation roles
+- nested relations
+- nodes in relations

include/osm_lua_processing.h

@@ -54,6 +54,10 @@ class OsmLuaProcessing {
 
 	// Has this object been assigned to any layers?
 	bool empty();
+
+	// Do we have Lua routines for non-MP relations?
+	bool canReadRelations();
+	bool canWriteRelations();
 
 	// Shapefile tag remapping
 	bool canRemapShapefiles();
@@ -64,6 +68,9 @@ class OsmLuaProcessing {
 
 	using tag_map_t = boost::container::flat_map<std::string, std::string>;
 
+	// Scan non-MP relation
+	bool scanRelation(WayID id, const tag_map_t &tags);
+
 	/// \brief We are now processing a significant node
 	void setNode(NodeID id, LatpLon node, const tag_map_t &tags);
 
@@ -74,7 +81,7 @@ class OsmLuaProcessing {
 	 * (note that we store relations as ways with artificial IDs, and that
 	 *  we use decrementing positive IDs to give a bit more space for way IDs)
 	 */
-	void setRelation(int64_t relationId, WayVec const &outerWayVec, WayVec const &innerWayVec, const tag_map_t &tags);
+	void setRelation(int64_t relationId, WayVec const &outerWayVec, WayVec const &innerWayVec, const tag_map_t &tags, bool isNativeMP);
 
 	// ----	Metadata queries called from Lua
 
@@ -149,6 +156,11 @@ class OsmLuaProcessing {
 	void AttributeBooleanWithMinZoom(const std::string &key, const bool val, const char minzoom);
 	void MinZoom(const double z);
 	void ZOrder(const double z);
+
+	// Relation scan support
+	kaguya::optional<int> NextRelation();
+	std::string FindInRelation(const std::string &key);
+	void Accept();
 
 	// Write error if in verbose mode
 	void ProcessingError(const std::string &errStr) {
@@ -167,6 +179,8 @@ class OsmLuaProcessing {
 
 	const Polygon &polygonCached();
 
+	const MultiLinestring &multiLinestringCached();
+
 	const MultiPolygon &multiPolygonCached();
 
 	inline AttributeStore &getAttributeStore() { return attributeStore; }
@@ -179,8 +193,11 @@ class OsmLuaProcessing {
 		outerWayVecPtr = nullptr;
 		innerWayVecPtr = nullptr;
 		linestringInited = false;
+		multiLinestringInited = false;
 		polygonInited = false;
 		multiPolygonInited = false;
+		relationAccepted = false;
+		relationSubscript = -1;
 	}
 
 	const inline Point getPoint() {
@@ -191,6 +208,8 @@ class OsmLuaProcessing {
 
 	kaguya::State luaState;
 	bool supportsRemappingShapefiles;
+	bool supportsReadingRelations;
+	bool supportsWritingRelations;
 	const class ShpMemTiles &shpMemTiles;
 	class OsmMemTiles &osmMemTiles;
 	AttributeStore &attributeStore;			// key/value store
@@ -199,6 +218,10 @@ class OsmLuaProcessing {
 	int64_t originalOsmID;					///< Original OSM object ID
 	bool isWay, isRelation, isClosed;		///< Way, node, relation?
 
+	bool relationAccepted;					// in scanRelation, whether we're using a non-MP relation
+	std::vector<WayID> relationList;		// in processWay, list of relations this way is in
+	int relationSubscript = -1;				// in processWay, position in the relation list
+
 	int32_t lon,latp;						///< Node coordinates
 	NodeVec const *nodeVecPtr;
 	WayVec const *outerWayVecPtr;
@@ -208,6 +231,8 @@ class OsmLuaProcessing {
 	bool linestringInited;
 	Polygon polygonCache;
 	bool polygonInited;
+	MultiLinestring multiLinestringCache;
+	bool multiLinestringInited;
 	MultiPolygon multiPolygonCache;
 	bool multiPolygonInited;
 

include/osm_store.h

@@ -9,6 +9,9 @@
 #include <vector>
 #include <mutex>
 #include <unordered_set>
+#include <boost/container/flat_map.hpp>
+
+extern bool verbose;
 
 class void_mmap_allocator
 {
@@ -235,6 +238,43 @@ class UsedWays {
 	}
 };
 
+// scanned relations store
+class RelationScanStore {
+
+private:
+	using tag_map_t = boost::container::flat_map<std::string, std::string>;
+	std::map<WayID, std::vector<WayID>> relationsForWays;
+	std::map<WayID, tag_map_t> relationTags;
+
+public:
+	void relation_contains_way(WayID relid, WayID wayid) {
+		if (relationsForWays.find(wayid) != relationsForWays.end()) {
+			relationsForWays[wayid] = {};
+		}
+		relationsForWays[wayid].emplace_back(relid);
+	}
+	void store_relation_tags(WayID relid, const tag_map_t &tags) {
+		relationTags[relid] = tags;
+	}
+	bool way_in_any_relations(WayID wayid) {
+		return relationsForWays.find(wayid) != relationsForWays.end();
+	}
+	std::vector<WayID> relations_for_way(WayID wayid) {
+		return relationsForWays[wayid];
+	}
+	std::string get_relation_tag(WayID relid, const std::string &key) {
+		auto it = relationTags.find(relid);
+		if (it==relationTags.end()) return "";
+		auto jt = it->second.find(key);
+		if (jt==it->second.end()) return "";
+		return jt->second;
+	}
+	void clear() {
+		relationsForWays.clear();
+		relationTags.clear();
+	}
+};
+
 // way store
 class WayStore {
 
@@ -360,6 +400,9 @@ class OSMStore
 	using linestring_t = boost::geometry::model::linestring<Point, std::vector, mmap_allocator>;
 	using linestring_store_t = std::deque<std::pair<NodeID, linestring_t>>;
 
+	using multi_linestring_t = boost::geometry::model::multi_linestring<linestring_t, std::vector, mmap_allocator>;
+	using multi_linestring_store_t = std::deque<std::pair<NodeID, multi_linestring_t>>;
+
 	using polygon_t = boost::geometry::model::polygon<Point, true, true, std::vector, std::vector, mmap_allocator, mmap_allocator>;
 	using multi_polygon_t = boost::geometry::model::multi_polygon<polygon_t, std::vector, mmap_allocator>;
 	using multi_polygon_store_t = std::deque<std::pair<NodeID, multi_polygon_t>>;
@@ -373,6 +416,9 @@ class OSMStore
 
 		std::mutex multi_polygon_store_mutex;
 		std::unique_ptr<multi_polygon_store_t> multi_polygon_store;
+
+		std::mutex multi_linestring_store_mutex;
+		std::unique_ptr<multi_linestring_store_t> multi_linestring_store;
 	};
 
 protected:	
@@ -384,6 +430,7 @@ class OSMStore
 	WayStore ways;
 	RelationStore relations;
 	UsedWays used_ways;
+	RelationScanStore scanned_relations;
 
 	generated osm_generated;
 	generated shp_generated;
@@ -397,6 +444,7 @@ class OSMStore
 		osm_generated.points_store = std::make_unique<point_store_t>();
 		osm_generated.linestring_store = std::make_unique<linestring_store_t>();
 		osm_generated.multi_polygon_store = std::make_unique<multi_polygon_store_t>();
+		osm_generated.multi_linestring_store = std::make_unique<multi_linestring_store_t>();
 
 		shp_generated.points_store = std::make_unique<point_store_t>();
 		shp_generated.linestring_store = std::make_unique<linestring_store_t>();
@@ -435,7 +483,7 @@ class OSMStore
 		return use_compact_nodes ? compact_nodes.size() : nodes.size();
 	}
 
-  LatpLon nodes_at(NodeID i) const { 
+	LatpLon nodes_at(NodeID i) const { 
 		return use_compact_nodes ? compact_nodes.at(i) : nodes.at(i);
 	}
 
@@ -454,6 +502,13 @@ class OSMStore
 	void ensure_used_ways_inited() {
 		if (!used_ways.inited) used_ways.reserve(use_compact_nodes, nodes_size());
 	}
+
+	using tag_map_t = boost::container::flat_map<std::string, std::string>;
+	void relation_contains_way(WayID relid, WayID wayid) { scanned_relations.relation_contains_way(relid,wayid); }
+	void store_relation_tags(WayID relid, const tag_map_t &tags) { scanned_relations.store_relation_tags(relid,tags); }
+	bool way_in_any_relations(WayID wayid) { return scanned_relations.way_in_any_relations(wayid); }
+	std::vector<WayID> relations_for_way(WayID wayid) { return scanned_relations.relations_for_way(wayid); }
+	std::string get_relation_tag(WayID relid, const std::string &key) { return scanned_relations.get_relation_tag(relid, key); }
 
 	generated &osm() { return osm_generated; }
 	generated const &osm() const { return osm_generated; }
@@ -498,6 +553,30 @@ class OSMStore
 
 		return iter->second;
 	}
+
+	template<typename Input>
+	void store_multi_linestring(generated &store, NodeID id, Input const &src)
+	{
+		multi_linestring_t dst;
+		dst.resize(src.size());
+		for (std::size_t i=0; i<src.size(); ++i) {
+			boost::geometry::assign(dst[i], src[i]);
+		}
+
+		std::lock_guard<std::mutex> lock(store.multi_linestring_store_mutex);
+		store.multi_linestring_store->emplace_back(id, std::move(dst));
+	}
+
+	multi_linestring_t const &retrieve_multi_linestring(generated const &store, NodeID id) const {
+		auto iter = std::lower_bound(store.multi_linestring_store->begin(), store.multi_linestring_store->end(), id, [](auto const &e, auto id) { 
+			return e.first < id; 
+		});
+
+		if(iter == store.multi_linestring_store->end() || iter->first != id)
+			throw std::out_of_range("Could not find generated multi-linestring with id " + std::to_string(id));
+
+		return iter->second;
+	}
 
 	template<typename Input>
 	void store_multi_polygon(generated &store, NodeID id, Input const &src)
@@ -541,8 +620,9 @@ class OSMStore
 	void reportStoreSize(std::ostringstream &str);
 	void reportSize() const;
 
-	// Relation -> MultiPolygon
+	// Relation -> MultiPolygon or MultiLinestring
 	MultiPolygon wayListMultiPolygon(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd, WayVec::const_iterator innerBegin, WayVec::const_iterator innerEnd) const;
+	MultiLinestring wayListMultiLinestring(WayVec::const_iterator outerBegin, WayVec::const_iterator outerEnd) const;
 	void mergeMultiPolygonWays(std::vector<NodeDeque> &results, std::map<WayID,bool> &done, WayVec::const_iterator itBegin, WayVec::const_iterator itEnd) const;
 
 	///It is not really meaningful to try using a relation as a linestring. Not normally used but included

include/output_object.h

@@ -15,7 +15,7 @@
 #include "osmformat.pb.h"
 #include "vector_tile.pb.h"
 
-enum OutputGeometryType { POINT_, LINESTRING_, POLYGON_ };
+enum OutputGeometryType { POINT_, LINESTRING_, MULTILINESTRING_, POLYGON_ };
 
 #define OSMID_TYPE_OFFSET	40
 #define OSMID_MASK 		((1L<<OSMID_TYPE_OFFSET)-1)
@@ -103,6 +103,17 @@ class OutputObjectOsmStoreLinestring : public OutputObject
 	}
 };
 
+class OutputObjectOsmStoreMultiLinestring : public OutputObject
+{
+public:
+	OutputObjectOsmStoreMultiLinestring(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeStoreRef attributes, uint minzoom)
+		: OutputObject(type, l, id, attributes, minzoom)
+	{ 
+		assert(type == MULTILINESTRING_);
+	}
+};
+
+
 class OutputObjectOsmStoreMultiPolygon : public OutputObject
 {
 public:

include/read_pbf.h

@@ -33,6 +33,17 @@ class PbfReader
 			pbfreader_generate_stream const &generate_stream,
 			pbfreader_generate_output const &generate_output);
 
+	// Read tags into a map from a way/node/relation
+	using tag_map_t = boost::container::flat_map<std::string, std::string>;
+	template<typename T>
+	void readTags(T &pbfObject, PrimitiveBlock const &pb, tag_map_t &tags) {
+		auto keysPtr = pbfObject.mutable_keys();
+		auto valsPtr = pbfObject.mutable_vals();
+		for (uint n=0; n < pbfObject.keys_size(); n++) {
+			tags[pb.stringtable().s(keysPtr->Get(n))] = pb.stringtable().s(valsPtr->Get(n));
+		}
+	}
+
 private:
 	bool ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::pair<std::size_t, std::size_t> progress, std::size_t datasize, std::unordered_set<std::string> const &nodeKeys, ReadPhase phase = ReadPhase::All);
 	bool ReadNodes(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb, const std::unordered_set<int> &nodeKeyPositions);
@@ -43,7 +54,7 @@ class PbfReader
 
 	/// Find a string in the dictionary
 	static int findStringPosition(PrimitiveBlock const &pb, char const *str);
-
+	
 	OSMStore &osmStore;
 };