test.rb @@ -1,33 +1,76 @@ - +require 'net/http' +require 'uri' +require 'open-uri' require 'lib/query_support.rb' +require 'note.rb' + #require 'json/pure' #require 'json/add/rails' class IndexController < ApplicationController + def expand_url(url) + begin + timeout(60) do + uri = URI.parse(url) + http = Net::HTTP.new(uri.host) + http.open_timeout = 30 + http.read_timeout = 60 + if uri.host == "ping.fm" || uri.host == "www.ping.fm" + logger.info "ping.fm requires us to fetch the body" + response = http.get(uri.path) + return nil + else + response = http.head(uri.path) # get2(uri.path,{ 'Range' => 'bytes=0-1' }) + if response.class == Net::HTTPRedirection || response.class == Net::HTTPMovedPermanently + logger.info "expand_url #{response} looking at relationship #{r.value} to become #{response['location']}" + return response['location'] + end + end + end + rescue => exception + logger.info "expand_url inner rescue error #{exception} while fetching #{url}" + return nil + end + return nil + end + + def fix_relations + # fix the broken links + Relation.find(:all, :conditions => { :kind => Note::RELATION_URL } ).each do |r| + ActionController::Base.logger.info "looking at relationship #{r.value}" + fixed = expand_url(r.value) + if fixed + r.value = fixed + r.save + end + end + end + # # The client side is javascript so this does very litle # def index + # strive to supply session state of previous question if any to the map for json refresh - @map.question = nil + @map.question = nil @map.question = session[:q] = params[:q] if params[:q] @map.question = session[:q] if !params[:q] - # strive to supply a hint to the map regarding where to center - @map.south = @map.west = @map.north = @map.east = 0.0 - begin - # attempt to fetch map location from parameters - @map.south = session[:s] = params[:s].to_f if params[:s] - @map.west = session[:w] = params[:w].to_f if params[:w] - @map.north = session[:n] = params[:n].to_f if params[:n] - @map.east = session[:e] = params[:e].to_f if params[:e] - # otherwise fetch them from session state if present (or set to nil) - @map.south = session[:s].to_f if !params[:s] - @map.west = session[:w].to_f if !params[:w] - @map.north = session[:n].to_f if !params[:n] - @map.east = session[:e].to_f if !params[:e] - rescue - end + # strive to supply a hint to the map regarding where to center + @map.south = @map.west = @map.north = @map.east = 0.0 + begin + # attempt to fetch map location from parameters + @map.south = session[:s] = params[:s].to_f if params[:s] + @map.west = session[:w] = params[:w].to_f if params[:w] + @map.north = session[:n] = params[:n].to_f if params[:n] + @map.east = session[:e] = params[:e].to_f if params[:e] + # otherwise fetch them from session state if present (or set to nil) + @map.south = session[:s].to_f if !params[:s] + @map.west = session[:w].to_f if !params[:w] + @map.north = session[:n].to_f if !params[:n] + @map.east = session[:e].to_f if !params[:e] + rescue + end end # app/controllers/index_controller.rb @@ -32,26 +32,73 @@ class Note < ActiveRecord::Base # acts_as_solr :fields => [:title, {:lat=>:range_float}, {:lon=>:range_float}] # slower than heck - using tsearch instead acts_as_tsearch :fields => ["title","description"] + # + # The database architecture consists of nodes and edges + # + # Edges may relate two nodes + # Edges may just be about one node + # Edges may be labelled + # + # There is a ongoing design question regarding if some attributes should be first class nodes or just edges + # RELATION_URL is being treated as both an edge and as a first class node for example. + # + +=begin RELATIONS = %w{ - RELATION_TAG + RELATION_TAG # a tag or hashtag + RELATION_CATEGORY # a formal category object similar to a tag but from a named set + RELATION_ENTITY # a citation of an entity [ also see first class entities in notes ] + RELATION_RELATION # an unlabelled relation between two notes + RELATION_FRIEND # a friendship relation between two notes + RELATION_URL # a citation of an url [ see also first class treatment of urls ] + RELATION_IMAGE # a citation of an image + RELATION_AUDIO # a citation of an audio + RELATION_MOVIE # a citation of a movie + } + + KINDS = %w{ + KIND_USER # a stand in for a local user account - a user makes posts + KIND_FEED # a feed - a feed makes posts + KIND_REPORTER # a reporter person - makes posts + KIND_REPORT # a kind of post (unused) + KIND_POST # a user post + KIND_GROUP # a grouping of posts + KIND_EVENT # an event that groups posts + KIND_PLACE # a place that groups posts + KIND_MAP # a map that groups posts + KIND_FILTER # a filter that groups posts + KIND_ENTITY # an entity of some kind + KIND_URL # a treatment of an url edge as a first class object [ used for client views ] + KIND_TAG # a treatment of an edge as a first class object [ just an idea ] + } +=end + + RELATIONS = %w{ + RELATION_TAG RELATION_CATEGORY - RELATION_ENTITY + RELATION_ENTITY RELATION_RELATION - RELATION_FRIEND - RELATION_URL + RELATION_FRIEND + RELATION_URL + RELATION_IMAGE + RELATION_AUDIO + RELATION_MOVIE } - KINDS = %w{ - KIND_USER - KIND_POST + KINDS = %w{ + KIND_USER KIND_FEED KIND_REPORTER KIND_REPORT + KIND_POST KIND_GROUP KIND_EVENT KIND_PLACE - KIND_MAP - KIND_FILTER + KIND_MAP + KIND_FILTER + KIND_ENTITY + KIND_URL + KIND_TAG } STATEBITS_RESERVED = 0 @@ -110,6 +157,13 @@ class Note return r end + def relations_all(kind = nil, sibling_id = nil) + query = { :note_id => self.id } + query[:kind] = kind if kind + query[:sibling_id] = sibling_id if sibling_id + return Relation.find(:all,:conditions=>query) + end + # TODO rate limit # TODO use a join def relation_as_notes(kind = nil, sibling_id = nil) app/models/note.rb @@ -1 +1,9 @@ <%= @map.body() %> +<table width="100%"> +<tr> +<td width="33%" valign=top><div id="people_box">People</div></td> +<td width="33%" valign=top><div id="posts_box">Posts</div></td> +<td width="33%" valign=top><div id="urls_box">Urls</div></td> +</tr> +</table> + app/views/index/index.html.erb @@ -500,12 +500,32 @@ function mapper_page_paint(blob) { feature["iconWindowAnchor"] = [ 9, 2 ]; mapper_inject_feature(feature); - // a list to draw text to - var map_list = document.getElementById("map_list"); + var glyph_url = "/dynamapper/icons/emblem-important.png"; + var feature = {}; + feature["kind"] = "icon"; + feature["image"] = glyph_url; + feature["iconSize"] = [ 32, 32 ]; + feature["iconAnchor"] = [ 9, 34 ]; + feature["iconWindowAnchor"] = [ 9, 2 ]; + mapper_inject_feature(feature); // mark all objects as stale mapper_mark_all_stale(); + // a list to draw text to + var people_box = document.getElementById("people_box"); + var posts_box = document.getElementById("posts_box"); + var urls_box = document.getElementById("urls_box"); + if ( people_box.hasChildNodes() ) { + while ( people_box.childNodes.length >= 1 ) { people_box.removeChild( people_box.firstChild ); } + } + if ( posts_box.hasChildNodes() ) { + while ( posts_box.childNodes.length >= 1 ) { posts_box.removeChild( posts_box.firstChild ); } + } + if ( urls_box.hasChildNodes() ) { + while ( urls_box.childNodes.length >= 1 ) { urls_box.removeChild( urls_box.firstChild ); } + } + // visit all the markers and add them for (var i=0; i<markers.length; i++) { @@ -543,19 +563,32 @@ function mapper_page_paint(blob) { feature["lat"] = lat; feature["lon"] = lon; feature["glyph"] = glyph_post; - if( kind != "KIND_POST" ) feature["glyph"] = glyph_person; + if( kind == "KIND_USER" ) feature["glyph"] = glyph_person; + if( kind == "KIND_URL" ) feature["glyph"] = glyph_url; mapper_inject_feature(feature); // Draw a list of features as well - if(map_list) { + if(true) { var div = document.createElement('div'); + // build the children if(div) { - div.innerHTML = "<img src='"+feature["glyph"]+"'></img> "+title; + div.style.border = "1px solid green"; - map_list.appendChild(div); + div.style.width = "100%"; + if(kind == "KIND_URL") { + div.innerHTML = "<img src='"+feature["glyph"]+"'></img> <a href='"+title+"'>"+title+"</a>"; + people_box.appendChild(div); + } + if(kind == "KIND_USER") { + div.innerHTML = "<img src='"+feature["glyph"]+"'></img> <a href='http://twitter.com/"+title+"'>"+title+"</a>"; + posts_box.appendChild(div); + } + if(kind == "KIND_POST") { + div.innerHTML = "<img src='"+feature["glyph"]+"'></img> "+title; + urls_box.appendChild(div); + } } } - } // sweep the ones that are not part of this display @@ -621,6 +654,7 @@ function mapper_page_paint_request(recenter) { /// start mapping engine (invoked once only) function mapper_initialize() { + if(map_div) return; map_div = document.getElementById("map"); if(!map_div) return; lib/dynamapper/dynamapper.rb @@ -135,8 +135,13 @@ class QuerySupport condition_arguments << w; condition_arguments << e; end + + # always disallow features with 0,0 as a location + if true + conditions << "lat <> 0 AND lon <> 0" + end - # i am actually only interested in posts - not people right now + # filter for posts; we'll collect only people related to those posts later if true conditions << "kind = ?" condition_arguments << Note::KIND_POST @@ -149,7 +154,6 @@ class QuerySupport results = [] ActionController::Base.logger.info "ABOUT TO QUERY #{conditions} and #{condition_arguments.join(' *** ' ) } " -# ABOUT TO QUERY description @@ to_tsquery(?) AND title @@ to_tsquery(?) AND kind = ? nullnullKIND_POST conditions = [ conditions.join(' AND ') ] + condition_arguments @@ -162,7 +166,7 @@ ActionController::Base.logger.info "GOT #{results_length} posts " # # lets go ahead and inject in only the people who were associated with the posts we found (so the user can see them) - # TODO this could be cleaned up massively using a bit of smarter SQL that finds uniques only + # TODO this could be cleaned up massively using a bit of smarter SQL that finds uniques only or at least a HASH join # people = {} @@ -170,10 +174,36 @@ ActionController::Base.logger.info "GOT #{results_length} posts " person = Note.find(:first,:conditions => { :id => post.owner_id } ) people[post.owner_id] = person if person != nil end + people.each do |key,value| + results << value + results_length = results_length + 1 + end -ActionController::Base.logger.info "GOT #{results_length} people " + # + # we also have a specific interest in 'entities' such as hashtags, urls, places and clusters + # here i am injecting 'fake' entities into the stream for now. + # later the thought is to promote some of the current edges to be first class entities + # annoyingly since these are not real notes they do not have unique ids. + # TODO should i promote urls and like concepts to be first class objects? [ probably ] + # - people.each do |key,value| + entities = {} + results.each do |post| + next if post.kind != Note::KIND_POST + post.relations_all(Note::RELATION_URL).each do |relation| + url = relation.value + next if entities[url] + note = Note.new + note.title = relation.value + note.id = post.id * 1000000 + relation.id # TODO such a hack ugh. + note.lat = post.lat + note.lon = post.lon + note.owner_id = post.owner_id + note.kind = Note::KIND_URL + entities[url] = note + end + end + entities.each do |key,value| results << value results_length = results_length + 1 end lib/query_support.rb @@ -293,6 +293,11 @@ end # TODO expand all URLS and try to get to a duplicate URL consistency; perhaps fingerprint the target page??? begin args[:title].split.grep(/http[s]?:\/\/\w/).each do |url| + response = Net::HTTP.get_response(URI.parse(uri_str)) + case response + when Net::HTTPRedirection then puts response['location'] + when Net::HTTPOK then puts uri_str + end note.relation_add(Note::RELATION_URL,url) end rescue lib/twitter_support/twitter_base.rb 44db8159f283e72ce26ef7229f02e7a685d81cdb anselm anselm@hook.org http://github.com/anselm/angel/commit/42c341959ba18eeed9a0c28395e7e65a8aa81520 42c341959ba18eeed9a0c28395e7e65a8aa81520 2009-08-22T13:43:39-07:00 2009-08-22T13:43:39-07:00 showing people posts and urls urls are implememented as pseudo objects expanding urls from a variety of services although ping.fm doesn't send a redirect annoyingly d724a363b36024710039c7276bb3e87e52af5316 anselm anselm@hook.org