feat: Denormalize ways & relations

Cargo.toml

@@ -9,3 +9,5 @@ edition = "2018"
 [dependencies]
 vadeen_osm = "0.4.1"
 osmpbf = "0.2.2"
+hex = "0.4.2"
+lru = "0.6.1"

src/denormalize.rs

@@ -1,59 +1,102 @@
+use hex;
+use lru::LruCache;
 use std::fs;
-use vadeen_osm::osm_io::write;
-use vadeen_osm::OsmBuilder;
+use std::path::{Path, PathBuf};
+use vadeen_osm::osm_io;
+use vadeen_osm::Osm;
 
 pub struct Writer {
-    nodes: String,
-    ways: String,
-    relations: String,
+    output: String,
+    cache: LruCache<String, bool>,
 }
 
 fn create_directory(path: &str) {
     match fs::create_dir(path) {
         Ok(_) => {
-            println!("Created directory {}", path);
+            //Created directory!("{}", path);
+        }
+        Err(_e) => {
+            //eprintln!("{}", _e);
         }
-        Err(_) => {}
     }
 }
 
 impl Writer {
     pub fn new(output: &str) -> Writer {
-        let nodes = format!("{}/{}", output, "nodes");
-        let ways = format!("{}/{}", output, "ways");
-        let relations = format!("{}/{}", output, "relations");
+        let out = Path::new(output);
+        let nodes = out.join("nodes");
+        let ways = out.join("ways");
+        let relations = out.join("relations");
         create_directory(output);
-        create_directory(&nodes);
-        create_directory(&ways);
-        create_directory(&relations);
+
+        let cache = LruCache::new(1000);
+        let nodes = nodes.to_str().unwrap();
+        let ways = ways.to_str().unwrap();
+        let relations = relations.to_str().unwrap();
+        create_directory(nodes);
+        create_directory(ways);
+        create_directory(relations);
         return Writer {
-            nodes,
-            ways,
-            relations,
+            cache,
+            output: output.to_string(),
         };
     }
 
-    pub fn add_relation(&self, relation: osmpbf::elements::Relation) -> u64 {
-        return 0;
-    }
+    fn write(&mut self, dir: &str, id: i64, osm: &Osm) -> u64 {
+        let bytes = id.to_be_bytes();
+        let mut i = 0;
 
-    pub fn add_way(&self, relation: osmpbf::elements::Way) -> u64 {
-        return 0;
-    }
+        let mut writable_dir = PathBuf::new();
+        writable_dir.push(&self.output);
+        writable_dir.push(&dir);
+        while i < bytes.len() {
+            let pre = hex::encode(&bytes[i..i + 1]);
+            i += 1;
+            writable_dir.push(&pre);
+            match self.cache.get(&pre) {
+                Some(_) => {}
+                None => {
+                    let written = writable_dir.to_str().unwrap();
+                    create_directory(written);
+                    self.cache.put(written.to_string(), true);
+                    println!("{}", written);
+                }
+            }
+        }
 
-    pub fn add_node(&self, id: i64, point: (f64, f64), tags: Vec<(&str, &str)>) -> u64 {
-        let mut builder = OsmBuilder::default();
-        builder.add_point(point, tags);
-        let osm = builder.build();
-        let writing = &format!("{}/{}.o5m", self.nodes, id);
-        println!("Writing {}", writing);
-        match write(writing, &osm) {
+        let rest = hex::encode(&bytes[i - 1..bytes.len()]);
+        match osm_io::write(
+            &format!("{}/{}.o5m", writable_dir.to_str().unwrap(), rest),
+            osm,
+        ) {
             Ok(_) => {
                 return 1;
             }
-            Err(_) => {
+            Err(e) => {
+                eprintln!("{}", e);
                 return 0;
             }
         }
     }
+
+    pub fn add_relation(&mut self, relation: vadeen_osm::Relation) -> u64 {
+        let mut osm = Osm::default();
+        let id = relation.id;
+        osm.add_relation(relation);
+        return self.write("relations", id, &osm);
+    }
+
+    pub fn add_way(&mut self, way: vadeen_osm::Way) -> u64 {
+        let mut osm = Osm::default();
+        let id = way.id;
+        osm.add_way(way);
+        return self.write("ways", id, &osm);
+    }
+
+    pub fn add_node(&mut self, node: vadeen_osm::Node) -> u64 {
+        let mut osm = Osm::default();
+        let id = node.id;
+        osm.add_node(node);
+        return self.write("nodes", id, &osm);
+    }
 }

src/main.rs

@@ -1,50 +1,128 @@
 //use georender_pack::encode;
+use std::sync::{Arc, Mutex};
+use std::vec::Vec;
 mod denormalize;
 use osmpbf::{Element, ElementReader};
 use std::env;
-use std::result::Result;
 use vadeen_osm::osm_io::error::Error;
+use vadeen_osm::{geo::Coordinate, Node, Relation, Tag, Way};
 
 fn main() -> std::result::Result<(), Error> {
     let args: Vec<String> = env::args().collect();
     let pbf = &args[1];
     let output = &args[2];
 
-    read(pbf, output);
+    write_denormalized_data(pbf, output)?;
     return Ok(());
 }
 
-fn read(pbf: &str, output: &str) -> std::result::Result<bool, Error> {
-    let reader = ElementReader::from_path(pbf).unwrap();
+fn get_meta(_tags: osmpbf::TagIter, info: osmpbf::Info) -> vadeen_osm::Meta {
+    let author = vadeen_osm::AuthorInformation {
+        change_set: info.changeset().unwrap() as u64,
+        uid: info.uid().unwrap() as u64,
+        user: info.user().unwrap().unwrap().to_string(),
+        created: info.milli_timestamp().unwrap(),
+    };
+
+    let mut tags = Vec::with_capacity(_tags.len());
+    _tags.for_each(|t| {
+        tags.push(Tag {
+            key: t.0.to_string(),
+            value: t.1.to_string(),
+        })
+    });
+
+    return vadeen_osm::Meta {
+        version: Some(info.version().unwrap() as u32),
+        tags: tags,
+        author: Some(author),
+    };
+}
 
-    let writer = denormalize::Writer::new(output);
+fn get_dense_meta(_tags: osmpbf::DenseTagIter, info: osmpbf::DenseNodeInfo) -> vadeen_osm::Meta {
+    let author = vadeen_osm::AuthorInformation {
+        change_set: info.changeset() as u64,
+        uid: info.uid() as u64,
+        user: info.user().unwrap().to_string(),
+        created: info.milli_timestamp(),
+    };
+
+    let mut tags = Vec::with_capacity(_tags.len());
+    _tags.for_each(|t| {
+        tags.push(Tag {
+            key: t.0.to_string(),
+            value: t.1.to_string(),
+        })
+    });
+    return vadeen_osm::Meta {
+        version: Some(info.version() as u32),
+        tags: tags,
+        author: Some(author),
+    };
+}
+
+fn write_denormalized_data(pbf: &str, output: &str) -> std::result::Result<bool, Error> {
+    let reader = ElementReader::from_path(pbf).unwrap();
+    let writer = Arc::new(Mutex::new(denormalize::Writer::new(output)));
 
     let total = reader
         .par_map_reduce(
             |element| match element {
                 Element::Node(node) => {
-                    return writer.add_node(
-                        node.id(),
-                        (node.lon(), node.lat()),
-                        node.tags().into_iter().collect(),
-                    );
+                    let node = Node {
+                        id: node.id(),
+                        coordinate: Coordinate::new(node.lat(), node.lon()),
+                        meta: get_meta(node.tags(), node.info()),
+                    };
+                    let count = writer.lock().unwrap().add_node(node);
+                    return count;
                 }
                 Element::DenseNode(node) => {
-                    return writer.add_node(
-                        node.id,
-                        (node.lon(), node.lat()),
-                        node.tags().into_iter().collect(),
-                    );
+                    let count = writer.lock().unwrap().add_node(vadeen_osm::Node {
+                        id: node.id,
+                        coordinate: Coordinate::new(node.lon(), node.lat()),
+                        meta: get_dense_meta(node.tags(), node.info().unwrap().clone()),
+                    });
+                    return count;
                 }
                 Element::Relation(rel) => {
-                    return writer.add_relation(rel);
+                    let mut members = Vec::new();
+                    for member in rel.members() {
+                        let var_name = match member.member_type {
+                            osmpbf::RelMemberType::Node => vadeen_osm::RelationMember::Node(
+                                member.member_id,
+                                "node".to_string(),
+                            ),
+                            osmpbf::RelMemberType::Way => {
+                                vadeen_osm::RelationMember::Way(member.member_id, "way".to_string())
+                            }
+                            osmpbf::RelMemberType::Relation => {
+                                vadeen_osm::RelationMember::Relation(
+                                    member.member_id,
+                                    "relation".to_string(),
+                                )
+                            }
+                        };
+                        members.push(var_name);
+                    }
+                    let count = writer.lock().unwrap().add_relation(Relation {
+                        id: rel.id(),
+                        members: members,
+                        meta: get_meta(rel.tags(), rel.info()),
+                    });
+                    return count;
                 }
                 Element::Way(way) => {
-                    return writer.add_way(way);
+                    let count = writer.lock().unwrap().add_way(Way {
+                        id: way.id(),
+                        refs: way.refs().collect(),
+                        meta: get_meta(way.tags(), way.info()),
+                    });
+                    return count;
                 }
             },
-            || 0_u64,     // Zero is the identity value for addition
-            |a, b| a + b, // Sum the partial results
+            || 0_u64,
+            |a, b| a + b,
         )
         .unwrap();