est.rb @@ -1,3 +1,4 @@ +.DS_Store public/text/*.pdf .svn solr/**/* .gitignore @@ -1,25 +1,56 @@ require 'lib/query_support.rb' -require 'json/pure' +#require 'json/pure' +#require 'json/add/rails' class IndexController < ApplicationController - + # - # The client side is javascript so just deliver code to the client first - # and javascript will take over + # The client side is javascript so this does very litle # def index + # strive to supply session state of previous question if any to the map for json refresh + @map.question = nil + @map.question = session[:q] = params[:q] if params[:q] + @map.question = session[:q] if !params[:q] + # strive to supply a hint to the map regarding where to center + @map.south = @map.west = @map.north = @map.east = 0.0 + begin + # attempt to fetch map location from parameters + @map.south = session[:s] = params[:s].to_f if params[:s] + @map.west = session[:w] = params[:w].to_f if params[:w] + @map.north = session[:n] = params[:n].to_f if params[:n] + @map.east = session[:e] = params[:e].to_f if params[:e] + # otherwise fetch them from session state if present (or set to nil) + @map.south = session[:s].to_f if !params[:s] + @map.west = session[:w].to_f if !params[:w] + @map.north = session[:n].to_f if !params[:n] + @map.east = session[:e].to_f if !params[:e] + rescue + end end # # Our first pass at an API - handle place, person and subject queries - # and return json # def json - question = params[:q] + + # pull user question and location of question; ignore session state + @q = nil + @q = session[:q] = params[:q].to_s if params[:q] + @s = @w = @n = @e = 0.0 + begin + @s = session[:s] = params[:s].to_f if params[:s] + @w = session[:w] = params[:w].to_f if params[:w] + @n = session[:n] = params[:n].to_f if params[:n] + @e = session[:e] = params[:e].to_f if params[:e] + rescue + end + + # pull user search term if supplied synchronous = false synchronous = true if params[:synchronous] && params[:synchronous] ==true - results = QuerySupport::query(question,synchronous) + results = QuerySupport::query(@q,@s,@w,@n,@e,synchronous) render :json => results.to_json end app/controllers/index_controller.rb @@ -57,15 +57,15 @@ Rails::Initializer.run do |config| # If you change this key, all old sessions will become invalid! # Make sure the secret is at least 30 characters and all random, # no regular words or you'll be exposed to dictionary attacks. - #config.action_controller.session = { - # :session_key => '_authgasm_example_session', - # :secret => '94b9c594695e69bdef6b1d4be037af5853be976b39a52a02f260fca0d0a36a8f913572bfdb631f55971a3b10b8dd9a875f9776ca61371741544e6ccc064dd41e' - #} + config.action_controller.session = { + :session_key => 'makerlab_angel', + :secret => '94b9c594695e69bdef6b1d4be037af5853be976b39a52a02f260fca0d0a36a8f913572bfdb631f55971a3b10b8dd9a875f9776ca61371741544e6ccc064dd41e' + } # Use the database for sessions instead of the cookie-based default, # which shouldn't be used to store highly confidential information # (create the session table with "rake db:sessions:create") - config.action_controller.session_store = :active_record_store + # config.action_controller.session_store = :active_record_store # Use SQL instead of Active Record's schema dumper when creating the test database. # This is necessary if your schema can't be completely dumped by the schema dumper, config/environment.rb @@ -1,6 +1,10 @@ class CreateNotes < ActiveRecord::Migration def self.up + # + # Core of our basic subject matter definition + # See relations for extended metadata (below) + # create_table :notes do |t| t.string :type @@ -30,6 +34,13 @@ class CreateNotes < ActiveRecord::Migration t.timestamps end + # + # Notes can contain arbitrary meta-data in an RDF like manner. + # + # One consideration: + # If two notes contain the same meta-data that data is not conserved; it exists in duplicate for each node. + # This may make it slightly harder to find if two notes share the same attribute... + # create_table :relations do |t| t.string :type t.string :kind db/migrate/003_create_notes.rb @@ -34,7 +34,7 @@ while($running) do # # go ahead and fetch any new content related to query string # - QuerySupport::query(question,synchronous) + QuerySupport::query(question,0,0,0,synchronous) # # sleep for 60 minutes lib/daemons/poll_twitter.rb @@ -34,8 +34,11 @@ class Dynamapper attr_accessor :apikey attr_accessor :map_cover_all_points - attr_accessor :lat - attr_accessor :lon + attr_accessor :question + attr_accessor :south + attr_accessor :west + attr_accessor :north + attr_accessor :east attr_accessor :width attr_accessor :height attr_accessor :zoom @@ -47,6 +50,7 @@ class Dynamapper # initialize() # def initialize(args = {}) + @south = @west = @east = @north = 0.0 @apikey = args[:apikey] @map_cover_all_points = true @lat = args[:latitude] || 45.516510 @@ -553,17 +557,32 @@ function mapper_page_paint(blob) { // function mapper_page_paint_request(recenter) { - var q = document.getElementById("q"); if(map == null) return; - var center = map.getCenter(); - var lat = 0; - var lng = 0; - if(center != null) { - lat = center.lat(); - lng = center.lng(); + var url = "/json?"; + + // tack on the search phrase + var q = document.getElementById("q"); + if(q != null) { + q = q.value; + if(q != null && q.length < 1) q = null; + } + if(q != null) { + url = url + "q="+q+"&"; } - var url = "/json?q="+q+"&lat="+lat+"&lng="+lng; + // send the bounds upward to server as well + var sw = map.getBounds().getSouthWest(); + var ne = map.getBounds().getNorthEast(); + if(sw == null || ne == null) { + return; + } + var s = sw.lat(); + var w = sw.lng(); + var n = ne.lat(); + var e = ne.lng(); + url = url + "s="+s+"&w="+w+"&n="+n+"&e="+e; + + // fetch the map; indicating work in progress by changing border color map_div.style.border = "1px solid yellow"; new Ajax.Request(url, { method:'get', @@ -602,6 +621,20 @@ function mapper_initialize() { // setup custom icon support mapper_icons(); // map.removeMapType(G_HYBRID_MAP); + // try to respect supplied map boundaries for the very first refresh before user does any actions + var did_not_center_yet = true; + map.south = #{@south}; + map.west = #{@west}; + map.north = #{@north}; + map.east = #{@east}; + if(map.north < 0.0 || map.north > 0.0 || map.south < 0.0 || map.south > 0.0) { + did_not_center_yet = false; + var bounds = new GLatLngBounds( new GLatLng(map.south,map.west,false), new GLatLng(map.north,map.east,false) ); + var center = bounds.getCenter(); + var zoom = map.getBoundsZoomLevel(bounds); + if(zoom < 2 ) zoom = 2; + map.setCenter(center,zoom); + } // capture map location whenever the map is moved and go ahead and ask for a view of that areas markers from our own server if(map_location_callback) { GEvent.addListener(map, "moveend", function() { @@ -615,8 +648,10 @@ function mapper_initialize() { // add features from a statically cached list if any [ this can help make first page display a bit faster ] mapper_inject(map_markers_raw); // center on any data we have already if any [ slight tension here with dynamic updates so can be disabled ] - if(#{@map_cover_all_points}) { - mapper_center(); + if( did_not_center_yet ) { + if(#{@map_cover_all_points}) { + mapper_center(); + } } // ask to add features from a remote connection dynamically [ and will center on them ] mapper_page_paint_request(true); lib/dynamapper/dynamapper.rb @@ -1,6 +1,6 @@ require 'dynamapper/geolocate.rb' -require 'twitter_support/twitter_aggregate.rb' +require 'lib/twitter_support/twitter_aggregate.rb' ############################################################################################ # @@ -60,98 +60,130 @@ class QuerySupport # # get query location - # i want to keep this logic at the controller level for now # - def self.query_locate(q) + def self.query_locate(q,map_s,map_w,map_n,map_e) - # get time - # TODO implement + # TODO deal with time also begins = nil ends = nil - # compute get bounds if any - lat,lon,rad = 0,0,0 - lat,lon,rad = Dynamapper.geolocate(q[:placenames].join(' ')) if q[:placenames].length - rad = 5 # TODO hack remove - q[:lat],q[:lon],q[:rad] = lat,lon,rad + # first we have no location + q[:s] = q[:w] = q[:n] = q[:e] = 0.0 + + # look for bounds information in a supplied string using brute force geocoder + lat,lon,rad = 0.0, 0.0, 0.0 + if q[:placenames].length + lat,lon,rad = Dynamapper.geolocate(q[:placenames].join(' ')) + rad = 5.0 # TODO hack remove + end + if lat < 0.0 || lat > 0.0 || lon < 0.0 || lon > 0.0 + # did we get boundaries from user query as ascii? "pizza near pdx" for example. + q[:bounds_from_text] = "true" + q[:s] = lat - rad + q[:w] = lon - rad + q[:n] = lat + rad + q[:e] = lon + rad + else + # otherwise try get boundaries from supplied params - this is the more conventional case + q[:bounds_from_text] = "false" + if ( map_s < 0.0 || map_s > 0.0 || map_w < 0.0 || map_w > 0.0 || map_n > 0.0 || map_n < 0.0 || map_e < 0.0 || map_e > 0.0 ) + q[:s] = map_s.to_f; + q[:w] = map_w.to_f; + q[:n] = map_n.to_f; + q[:e] = map_e.to_f; + end + end ActionController::Base.logger.info "query: located the query #{q[:placenames].join(' ')} to #{lat} #{lon} #{rad}" return q - end - def self.query(phrase,synchronous=false) + def self.query(question,map_s,map_w,map_n,map_e,synchronous=false) # basic string parsing - q = QuerySupport::query_parse(phrase) - QuerySupport::query_locate(q) + q = QuerySupport::query_parse(question) + QuerySupport::query_locate(q,map_s,map_w,map_n,map_e) - # optionally pull in fresh content (only supports twitter for now) - TwitterSupport::aggregate_memoize(q,synchronous) + # Aggregate? + TwitterSupport::aggregate_memoize(q,true) if synchronous # look at our internal database and return best results results_length = 0 results = [] - search,lat,lon,rad = q[:search],q[:lat],q[:lon],q[:rad] + words = q[:words] + s,w,n,e = q[:s],q[:w],q[:n],q[:e] - ActionController::Base.logger.info "Query: now looking for: #{search} at location #{lat} #{lon} #{rad}" + ActionController::Base.logger.info "Query: now looking for: #{words} at location #{s} #{w} #{n} #{e}" - # TODO need to figure out how to do tsearch AND ordinary search + conditions = [] + condition_arguments = [] - if ( lat < 0 || lat > 0 || lon < 0 || lon > 0 ) - results_length = Note.count(:conditions => ["lat >= ? AND lat <= ? AND lon >= ? AND lon <= ?", lat-rad,lat+rad,lon-rad,lon+rad ] ) - results = Note.all(:conditions => ["lat >= ? AND lat <= ? AND lon >= ? AND lon <= ?", lat-rad,lat+rad,lon-rad,lon+rad ] ) - else - # TODO combine not eor - results = Note.find_by_tsearch(search) if search.length > 0 - results_length = 0 # fix TODO - results.each do - results_length = results_length + 1 - end + # if there are search terms then add them to the search boundary + # are we totally cleaning words to disallow garbage? TODO + if(words.length > 0 ) + conditions << "description @@ to_tsquery(?)" + condition_arguments << words.join('&') + conditions << "title @@ to_tsquery(?)" + condition_arguments << words.join('&') end - results_length = 0 - results = [] - Note.all(:conditions => [ "kind = ?", Note::KIND_POST ], :limit => 50, :order => "id desc" ).each do |note| - results << note - results_length = results_length + 1 - end - Note.all(:conditions => [ "kind = ?", Note::KIND_USER ], :limit => 50, :order => "id desc" ).each do |note| - results << note - results_length = results_length + 1 - end + # also add lat long constraints + # TODO deal with wrap around the planet + if ( s < 0 || s > 0 || w < 0 || w > 0 || n > 0 || n < 0 || e < 0 || e > 0 ) + conditions << "lat >= ? AND lat <= ? AND lon >= ? AND lon <= ?" + condition_arguments << s; + condition_arguments << n; + condition_arguments << w; + condition_arguments << e; + end - ActionController::Base.logger.info "Query: got results #{results} #{results_length}" + # i am actually only interested in posts - not people right now + if true + conditions << "kind = ?" + condition_arguments << Note::KIND_POST + end - q[:results_length] = results_length - q[:results] = results + # + # collect a big old pile of posts + # + results_length = 0 + results = [] - return q - end +ActionController::Base.logger.info "ABOUT TO QUERY #{conditions} and #{condition_arguments.join(' *** ' ) } " +# ABOUT TO QUERY description @@ to_tsquery(?) AND title @@ to_tsquery(?) AND kind = ? nullnullKIND_POST + + conditions = [ conditions.join(' AND ') ] + condition_arguments + + Note.all(:conditions => conditions , :limit => 255, :order => "id desc" ).each do |note| + results << note + results_length = results_length + 1 + end + +ActionController::Base.logger.info "GOT #{results_length} posts " + # + # lets go ahead and inject in only the people who were associated with the posts we found (so the user can see them) + # TODO this could be cleaned up massively using a bit of smarter SQL that finds uniques only + # -=begin - # i found solr to be very slow so this is not used now - def query_with_solr(phrase) - if ( lat < 0 || lat > 0 || lon < 0 || lon > 0 ) - search << "lat:[#{lat-rad} TO #{lat+rad}]" - search << "lon:[#{lon-rad-rad} TO #{lon+rad+rad}]" + people = {} + results.each do |post| + person = Note.find(:first,:conditions => { :id => post.owner_id } ) + people[post.owner_id] = person if person != nil end - search_phrase = search.join(" AND ") - ActionController::Base.logger.info "Query: solr now looking for: #{search_phrase}" - if search.length - results = Note.find_by_solr(search_phrase, - :offset => 0, - :limit => 50 - #:order => "id desc", - #:operator => "and" - ) - total_hits = 0 - total_hits = results.total_hits if results - results = results.docs if results - results = [] if !results + +ActionController::Base.logger.info "GOT #{results_length} people " + + people.each do |key,value| + results << value + results_length = results_length + 1 end - ActionController::Base.logger.info "Query: solr now done" + + ActionController::Base.logger.info "Query: got results #{results} #{results_length}" + + q[:results_length] = results_length + q[:results] = results + + return q end -=end end lib/query_support.rb @@ -27,7 +27,10 @@ class TwitterSupport ActionController::Base.logger.info "Query: aggregation of a query starting #{Time.now}" - lat,lon,rad = q[:lat],q[:lon],q[:rad] + map_s = q[:s] + map_w = q[:w] + map_n = q[:n] + map_e = q[:e] # did the user supply some people as the anchor of a search? # refresh them and get their friends ( this is cheap and can be done synchronously ) @@ -61,8 +64,8 @@ class TwitterSupport end else # if there are no people to anchor the search then just let twitter do the search - ActionController::Base.logger.info "query: using a general search strategy looking for #{q[:words].join(' ')} near #{lat} #{lon} #{rad}" - self.twitter_search(q[:words],lat,lon,rad) if q[:words] && q[:words].length > 0 + ActionController::Base.logger.info "query: using a general search strategy looking for #{q[:words].join(' ')} near #{map_s} #{map_w} #{map_n} #{map_e}" + self.twitter_search(q[:words],map_s,map_n,map_w,map_e) if q[:words] && q[:words].length > 0 end # TODO idea lib/twitter_support/twitter_aggregate.rb @@ -1,14 +1,21 @@ - +###################################################################################### +# # Twitter centric Analytics # -# Arguable that this may be general and can be pushed out of twitter +# This is currently a work in progress. It consists of: # -# Hash tags are broken out -# Urls are broken out -# Parties are broken out -# NLP is used to find interesting objects using named entity extraction -# Duplicate or similar posts are clustered +# Geolocation of post using Yahoo PlaceMaker [ already done on post save ] +# Pluck out hashtags [ already done on post save ] +# Pluck out urls [ already done on post save ] +# Pluck out entities using third party named entity extraction [ TODO ] +# Pluck out sentiment using NLP [ TODO ] +# Objective scoring [ TODO ] +# Subjective scoring [ TODO ] +# Clustering based on content, time and location [ TODO ] +# Duplicate removal [ TODO ] +# +###################################################################################### class TwitterSupport @@ -27,7 +34,6 @@ class TwitterSupport # - ... end - ########################################################################################### # metadata extraction ########################################################################################### @@ -36,4 +42,44 @@ class TwitterSupport def self.metadata_all end + ########################################################################################### + # scoring - both objective and subjective + ########################################################################################### + + # + # Score ( not done ) + # objective people score is a function of distance from golden members + # objective post score is a function of distance from golden members + # objective post score is increased if replied to + # objective post score is increased if new + # objective post score is increased for magic keywords + # subjective post score is a sum of distances from anchors + # + + def self.score + # + # if we can do scoring in the sql query then do that... otherwise... + # + # for each post + # + # score = 0 to 1 based on newness ranging over say a month + # score = 0 to 1 based on graph distance + # score = 0 to 1 based on geographic distance + # score = 0 to 1 based on who is bookmarking this thing; its velocity? + # + # TODO fold duplicates and increase their score... + end + + # score friends + def self.score_friends + end + + # score posts + def self.score_posts + end + + # score everything + def self.score_all + end + end lib/twitter_support/twitter_analytics.rb @@ -174,7 +174,7 @@ class TwitterSupport else # TODO note that for some reason :updated_at is not set here - and we rely on this behavior but it is implicit. party.update_attributes(:title => title, :description => description ); - if lat < 0 || lat > 0 || lon < 0 || lon > 0 + if lat < 0.0 || lat > 0.0 || lon < 0.0 || lon > 0.0 party.update_attributes(:lat => lat, :lon => lon ) end ActionController::Base.logger.info "Updated a party with title #{title}" @@ -240,7 +240,7 @@ class TwitterSupport if note # test -if lat || lon +if lat > 0.0 || lat < 0.0 || lon > 0.0 || lon < 0.0 note.update_attributes( :lat => lat, :lon => lon, :rad => rad ) end # test end @@ -272,13 +272,30 @@ end ) note.save - # build a relationship to the owner - not really needed except for CNG traversals + # build a relationship to the owner - not really needed except for CNG traversals ( off for now ) # note.relation_add(Note::RELATION_OWNER,party.id) - # build a relationship to hash tags - # TODO verify that this works - args[:title].scan(/#[a-zA-Z]+/).each do |tag| - note.relation_add(Note::RELATION_TAG,tag[1..-1]) + # build a relationship to tags + #args[:title].scan(/#[a-zA-Z]+/).each do |tag| + # note.relation_add(Note::RELATION_TAG,tag[1..-1]) + #end + + begin + tags = {} + args[:title].gsub(/ ?(#\w+)/) { |tag| tag = tag.strip[1..-1].downcase; tags[tag] = tag } + tags.each do |key,tag| + note.relation_add(Note::RELATION_TAG,tag) + end + rescue + end + + # build a relationship to urls + # TODO expand all URLS and try to get to a duplicate URL consistency; perhaps fingerprint the target page??? + begin + args[:title].split.grep(/http[s]?:\/\/\w/).each do |url| + note.relation_add(Note::RELATION_URL,url) + end + rescue end ActionController::Base.logger.info "Saved a new post from #{party.title} ... #{title}" @@ -290,7 +307,6 @@ end return note end - ########################################################################################################## # twitter get a set of people objects from their names ########################################################################################################## lib/twitter_support/twitter_base.rb @@ -11,9 +11,9 @@ class TwitterSupport # TODO slightly worried that a bad rad might be passed in ########################################################################################################## - def self.twitter_search(terms,lat,lon,rad) + def self.twitter_search(terms,map_s,map_w,map_n,map_e) - ActionController::Base.logger.info "Searching for #{terms.join(' ')} near #{lat} #{lon} #{rad}" + ActionController::Base.logger.info "Searching for #{terms.join(' ')} near #{map_s} #{map_w} #{map_n} #{map_e}" if self.twitter_get_remaining_hits < 1 # TODO what should we do? @@ -26,16 +26,26 @@ class TwitterSupport posts = [] parties = [] blob = [] - if lat < 0 || lat > 0 || lon < 0 || lon > 0 - rad = "25mi" - blob = Twitter::Search.new(terms.join(' ')).geocode(lat,lon,rad) # TODO try conflating this verbosity - ActionController::Base.logger.debug("twitter_search with #{terms} and lat lon #{lat} #{lon}") - else - blob = Twitter::Search.new(terms.join(' ')) - ActionController::Base.logger.debug("twitter_search with #{terms} without lat or lon") + # turn into ordinary center and radius + # TODO radius is wrong + lat = map_n - map_s / 2 + map_s + lon = map_w - map_e / 2 + map_e + rad = 5 + twitter_rad = "25mi" + + # do a general purpose search with location if any else just do a search + begin + if map_s < 0.0 || map_s > 0.0 || map_n < 0.0 || map_n > 0.0 || map_e < 0.0 || map_e > 0.0 || map_w < 0.0 || map_w > 0.0 + blob = Twitter::Search.new(terms.join(' ')).geocode(lat,lon,twitter_rad) # TODO try conflating this verbosity + ActionController::Base.logger.debug("twitter_search with #{terms} and lat lon #{lat} #{lon}") + else + blob = Twitter::Search.new(terms.join(' ')) + ActionController::Base.logger.debug("twitter_search with #{terms} without lat or lon") + end + rescue end - if !blob + if !blob || blob.length < 1 ActionController::Base.logger.debug("did not find any twitter search results?") return [],[] end lib/twitter_support/twitter_collect.rb lib/twitter_support/twitter_score.rb f1cabb5c6f47e65ad6a55bd64ddef4dc3a69fa4b anselm anselm@hook.org http://github.com/anselm/angel/commit/d30e6237128c527934749b9229e1a7b38b367354 d30e6237128c527934749b9229e1a7b38b367354 2009-08-19T23:06:35-07:00 2009-08-19T23:06:35-07:00 switched to expressing full bounding region rather than lat,lon,center client side delivers location to server side from user map focus client side map focus tries to remain persistent between user sessions server side uses map boundaries as a hint for returning results results now focus on posts and only show users related to those posts trying to continue to work around a json/pure bug - unsure still switched to cookie based session state for now dd78385ae10e857d25e94a8462f2f9f904a74f43 anselm anselm@hook.org