1989 guessing of namedplaces

services/importer/lib/importer/content_guesser.rb

@@ -41,6 +41,14 @@ def country_column
         nil
       end
 
+      def namedplace_column
+        return nil if not enabled?
+        columns.each do |column|
+          return column[:column_name] if is_namedplace_column? column
+        end
+        nil
+      end
+
       def ip_column
         return nil if not enabled?
         columns.each do |column|
@@ -73,10 +81,30 @@ def is_country_column?(column)
         end
       end
 
+      def is_namedplace_column?(column)
+        return false unless is_text_type? column
+        entropy = metric_entropy(column, country_name_normalizer) # TODO: optimize
+        if entropy < minimum_entropy
+          false
+        else
+          proportion = namedplace_proportion(column)
+          if proportion < threshold
+            false
+          else
+            log_namedplace_guessing_match_metrics(proportion)
+            true
+          end
+        end
+      end
+
       def log_country_guessing_match_metrics(proportion)
         @importer_stats.gauge('country_proportion', proportion)
       end
 
+      def log_namedplace_guessing_match_metrics(proportion)
+        @importer_stats.gauge('namedplace_proportion', proportion)
+      end
+
       def log_ip_guessing_match_metrics(proportion)
         @importer_stats.gauge('ip_proportion', proportion)
       end
@@ -137,6 +165,21 @@ def country_proportion(column)
         country_proportion
       end
 
+      def namedplace_proportion(column)
+        column_name_sym = column[:column_name].to_sym
+        matches = count_namedplaces(sample, column_name_sym)
+        country_proportion = matches.to_f / sample.count
+        log "namedplace_proportion(#{column[:column_name]}) = #{country_proportion}"
+        country_proportion
+      end
+
+      def count_namedplaces(sample, column_name_sym)
+        sql_array = sample.map{|row| "'" + row[column_name_sym] + "'"}.join(',')
+        query = "WITH geo_function as (SELECT (geocode_namedplace(Array[#{sql_array}])).*) select count(success) FROM geo_function where success = TRUE"
+        ret = geocoder_sql_api.fetch(query)
+        ret.first['count']
+      end
+
       def log(msg)
         @job.log msg if @job
       end

services/importer/lib/importer/georeferencer.rb

@@ -47,6 +47,7 @@ def run
           create_the_geom_from_latlon           ||
           create_the_geom_from_ip_guessing      ||
           create_the_geom_from_country_guessing ||
+          create_the_geom_from_namedplaces_guessing ||
           create_the_geom_in(table_name)
 
         enable_autovacuum
@@ -151,6 +152,31 @@ def create_the_geom_from_country_guessing
         return false
       end
 
+      def create_the_geom_from_namedplaces_guessing
+        return false if not @content_guesser.enabled?
+        job.log 'Trying namedplaces guessing...'
+        begin
+          namedplace_column_name = nil
+          @importer_stats.timing('guessing') do
+            @tracker.call('guessing')
+            namedplace_column_name = @content_guesser.namedplace_column
+            @tracker.call('importing')
+          end
+          if namedplace_column_name
+            job.log "Found namedplace column: #{namedplace_column_name}"
+            create_the_geom_in table_name
+            return geocode_namedplaces namedplace_column_name
+          end
+        rescue Exception => ex
+          message = "create_the_geom_from_namedplaces_guessing failed: #{ex.message}"
+          Rollbar.report_message(message,
+                                 'warning',
+                                 {user_id: @job.logger.user_id, backtrace: ex.backtrace})
+          job.log "WARNING: #{message}"
+        end
+        return false
+      end
+
       def create_the_geom_from_ip_guessing
         return false if not @content_guesser.enabled?
         job.log 'Trying ip guessing...'
@@ -180,6 +206,11 @@ def geocode_countries country_column_name
         geocode(country_column_name, 'polygon', 'admin0')
       end
 
+      def geocode_namedplaces namedplace_column_name
+        job.log "Geocoding namedplaces..."
+        geocode(namedplace_column_name, 'point', 'namedplace')
+      end
+
       def geocode_ips ip_column_name
         job.log "Geocoding ips..."
         geocode(ip_column_name, 'point', 'ipaddress')

services/table-geocoder/lib/internal-geocoder/abstract_query_generator.rb

@@ -27,7 +27,7 @@ def copy_results_to_table_query
 
       def country
         country = @internal_geocoder.countries
-        country == %Q{'world'} ? 'null' : country
+        (country == %Q{'world'} || country.blank?) ? 'null' : country
       end
 
       def dest_table