Added Feature Classification functionality

History.txt

@@ -1,7 +1,8 @@
 === 0.0.1 / 2012-06-26
 
 * 1 major enhancement
-  * Birthday!
 * 2 published gem
+* 3 added command line app
+* 4 added filter osm map features
 
 

Manifest.txt

@@ -1,4 +1,8 @@
 .autotest
+.yardoc/checksums
+.yardoc/object_types
+.yardoc/objects/root.dat
+.yardoc/proxy_types
 History.txt
 Manifest.txt
 README.txt
@@ -10,6 +14,7 @@ lib/db/abstract_interface.rb
 lib/db/dbhandler.rb
 lib/db/mongohandler.rb
 lib/osm2mongo.rb
+osm2mongo.gemspec
 test/profile.rb
 test/test_callbacks.rb
 test/test_mongo.rb

README.txt

@@ -24,9 +24,13 @@ Import Openstreetmap .osm(xml) data format into MongoDB written in Ruby.
 
   From command line:
   =============
-
+  >>>Load data into Mongo<<<
   ruby osm2mongo.rb -H localhost -p 27017 -d osm -l 200 -f /tmp/map.osm
-  ruby osm2mongo.rb -H localhost -p 27017 -d osm -l 200 -U http://people.inf.elte.hu/ntb/map.osm.bz2
+  ruby osm2mongo.rb -H localhost -p 27017 -d osm -l 200 -U http://someaddress.com/map.osm.bz2
+
+  >>Filter data into new collection by map feature type<<<
+  ruby osm2mongo.rb -F highway -d osm -H localhost -p 27017 -l 300
+  ruby osm2mongo.rb -F waterway -S riverbank -d osm -H localhost -p 27017 -l 300
 
 
 == REQUIREMENTS:

bin/osm2mongo

@@ -41,6 +41,7 @@ require 'date'
 
 require_relative 'common'
 require_relative 'callbacks'
+require_relative 'db/classifier'
 
 
 class App
@@ -72,8 +73,23 @@ class App
       callback = Osm2Mongo::Callbacks.new(@todos['db'], collections, Integer(@todos['limit']), common, @todos['host'], @todos['port'])
       file_path = ''
       beginning_time = Time.now
+
+      if @todos.has_key?("feature") or @todos.has_key?("subfeature")
+          filter = DB::Classifier.new(@todos['db'], @todos['limit'], @todos['host'], @todos['port'])
+          result = 0
+
+          if @todos.has_key?("subfeature")
+              result = filter.classify_subtype(@todos['feature'], @todos['subfeature'])
+          else
+              result = filter.classify_all(@todos['feature'])
+          end
+          end_time = Time.now
+          puts "\n Processed Objects :(#{result}) >>> Elapsed: #{(end_time - beginning_time)}"
+          return 0
+      end
+
       unless @todos.has_key?("file")
-          status = common.download("http://download.geofabrik.de/osm/europe/hungary.osm.bz2", "/tmp/")
+          status = common.download(@todos['url'], "/tmp/")
           status = common.decompress(status['path'])
           file_path = status['path']
       else
@@ -105,15 +121,18 @@ class App
       opts.on('-d', '--database')   { @options.db = true}
       opts.on('-l', '--limit')      { @options.limit = true}
 
-      opts.parse!(@arguments) rescue return false
+      opts.on('-F', '--feature')    { @options.feature = true}
+      opts.on('-S', '--sfeature')   { @options.subfeature = true}
 
+      opts.parse!(@arguments) rescue return false
+
       true      
     end
 
 
     # True if required arguments were provided
     def arguments_valid?
-      true if @arguments.length == 5
+      true if @arguments.length >= 5
     end
 
     # Setup the arguments
@@ -129,6 +148,10 @@ class App
 
     end
 
+    def output_usage
+        puts "Error parsing arguments"
+    end
+
     def output_version
       puts "Osm2Mongo Utility version #{VERSION}"
     end

lib/db/classifier.rb

@@ -0,0 +1,105 @@
+require_relative 'mongohandler'
+
+# Classifying OSM Features
+# 
+# Purpose:
+#  Assuming you have already imported data into nodes, ways and rels collections.
+#
+# This class implements:
+# - Dbhandler interface.
+#
+module DB
+    class Classifier < DB::Dbhandler
+
+        attr_accessor :connection
+
+        COLL_WAYS = "ways"
+
+        #
+        # Constructor.
+        # ==== Attributes
+        # * +database_name+ database name
+        # * +collection_name+ collection name
+        # * +bulk_limit+ Array limit for bulk insert.
+        # * +host+ host name (default: localhost)
+        # * +port+ port number (default: 27017)    
+        def initialize(database_name, bulk_limit, host, port)
+            @host = host
+            @port = port
+            @dbname = database_name
+            @bulk_limit = bulk_limit
+            @data_array = Array.new
+            connect()
+        end
+
+        #
+        # Connecting to database
+        #
+        # Connecting to database with given:
+        # ==== Attributes
+        # * +host+ Host name
+        # * +port+ Port number
+        # * +pool_size+ optional
+        #
+        def connect
+            @connection = Mongo::Connection.new(@host, @port, :pool_size => 5)
+            @collection = (@connection[@dbname])[COLL_WAYS]
+        end
+
+        #
+        # Load all features into feature_name collection.
+        # ==== Attributes
+        # * +feature_name+ OSM Map Feature Name
+        #
+        def classify_all(feature_name)
+            feature_con = DB::Mongohandler.new(@dbname, feature_name, @bulk_limit, @host, @port)
+            feature_con.use_connection(@connection)
+
+            @collection.find("tags." << feature_name => {"$exists" => true}).each do |doc|
+                feature_con.bulk_insert(doc)
+            end
+            return feature_con.collection.size
+        end
+
+        #
+        # Load all features into feature_name collection with subfeature filtering.
+        # ==== Attributes
+        # * +feature_name+ OSM Map Feature name
+        # * +sub_feature+ OSM Map Sub feature name
+        #
+
+        def classify_subtype(main_feature, sub_feature)
+            # First argument identifies Main Feature, the rest sub-features
+            feature_con = DB::Mongohandler.new(@dbname, main_feature, @bulk_limit, @host, @port)
+            feature_con.use_connection(@connection)
+
+            @collection.find({"tags." << main_feature => sub_feature}).each do |doc|
+                feature_con.bulk_insert(doc)
+            end
+            return feature_con.collection.size
+        end
+
+        #
+        # Abstract method
+        #
+        # Insert data:
+        # ==== Attributes
+        # * +data+ Hash or Array
+        #    
+        def insert(*args)
+
+        end
+
+        #
+        # Abstract method
+        #
+        # Insert many data at once.
+        # ==== Attributes
+        # * +data+ Array of Hashes
+        #    
+        def bulk_insert(*args)
+
+        end
+
+    end
+end

lib/osm2mongo.rb

@@ -41,6 +41,7 @@
 
 require_relative 'common'
 require_relative 'callbacks'
+require_relative 'db/classifier'
 
 
 class App
@@ -72,8 +73,23 @@ def run
       callback = Osm2Mongo::Callbacks.new(@todos['db'], collections, Integer(@todos['limit']), common, @todos['host'], @todos['port'])
       file_path = ''
       beginning_time = Time.now
+
+      if @todos.has_key?("feature") or @todos.has_key?("subfeature")
+          filter = DB::Classifier.new(@todos['db'], @todos['limit'], @todos['host'], @todos['port'])
+          result = 0
+
+          if @todos.has_key?("subfeature")
+              result = filter.classify_subtype(@todos['feature'], @todos['subfeature'])
+          else
+              result = filter.classify_all(@todos['feature'])
+          end
+          end_time = Time.now
+          puts "\n Processed Objects :(#{result}) >>> Elapsed: #{(end_time - beginning_time)}"
+          return 0
+      end
+
       unless @todos.has_key?("file")
-          status = common.download("http://download.geofabrik.de/osm/europe/hungary.osm.bz2", "/tmp/")
+          status = common.download(@todos['url'], "/tmp/")
           status = common.decompress(status['path'])
           file_path = status['path']
       else
@@ -105,15 +121,18 @@ def parsed_options?
       opts.on('-d', '--database')   { @options.db = true}
       opts.on('-l', '--limit')      { @options.limit = true}
 
-      opts.parse!(@arguments) rescue return false
+      opts.on('-F', '--feature')    { @options.feature = true}
+      opts.on('-S', '--sfeature')   { @options.subfeature = true}
 
+      opts.parse!(@arguments) rescue return false
+
       true      
     end
 
 
     # True if required arguments were provided
     def arguments_valid?
-      true if @arguments.length == 5
+      true if @arguments.length >= 5
     end
 
     # Setup the arguments
@@ -129,6 +148,10 @@ def process_arguments
 
     end
 
+    def output_usage
+        puts "Error parsing arguments"
+    end
+
     def output_version
       puts "Osm2Mongo Utility version #{VERSION}"
     end

test/test_classifier.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH << '../lib/db'
+require 'classifier.rb'
+
+
+describe DB::Classifier, "#connect" do
+    it "should returns connection status active" do
+        mongo = DB::Classifier.new("osm", 100, 'localhost', '27017')
+        (mongo.connection.connected?).should be(true)
+    end
+end
+
+
+describe DB::Classifier do
+    before do
+        @handler = DB::Classifier.new("osm", 100, 'localhost', '27017')
+    end
+
+    it "should insert documents by given feature type" do
+        result = @handler.classify_all("highway")
+        result.should be > 0
+    end
+
+    it "should insert documents by given feature-sub-feature type" do
+        result = @handler.classify_subtype("waterway", "riverbank")
+        result.should be > 0
+    end
+
+
+    after do
+        #@handler.collection.drop()
+    end
+end