@@ -1,3 +1,4 @@ .DS_Store coverage -pkg \ No newline at end of file +pkg +spec/fixtures/calais.yml .gitignore @@ -11,90 +11,90 @@ module Calais :instances => 'type/sys/InstanceInfo', :relevances => 'type/sys/RelevanceInfo', } - + attr_accessor :hashes, :entities, :relations, :geographies - + def initialize(rdf_string) @raw_response = rdf_string - + @hashes = [] @entities = [] @relations = [] @geographies = [] - + @relevances = {} # key = hash, val = relevance + extract_data + + process_relevances process_entities process_relations process_geographies end - + class Entity - attr_accessor :hash, :type, :attributes + attr_accessor :hash, :type, :attributes, :relevance end - + class Relation attr_accessor :hash, :type, :attributes end - + class Geography attr_accessor :name, :hash, :attributes end - + class CalaisHash attr_accessor :value - + def self.find_or_create(hash, hashes) - selected = hashes.select {|h| h.value } - - if selected.empty? - new_hash = self.new - new_hash.value = hash - hashes << new_hash - new_hash - else - selected.first + if !selected = hashes.select {|h| h.value == hash }.first + selected = self.new + selected.value = hash + hashes << selected end + + selected end end - + private def extract_data doc = XML::Parser.string(@raw_response).parse - + @nodes = {} @nodes[:docinfo] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:docinfo]}')]/..") @nodes[:docinfo].each { |node| node.remove! } - + @nodes[:docinfometa] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:docinfometa]}')]/..") @nodes[:docinfometa].each { |node| node.remove! } - + @nodes[:defaultlangid] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:defaultlangid]}')]/..") @nodes[:defaultlangid].each { |node| node.remove! } - + @nodes[:doccat] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:doccat]}')]/..") @nodes[:doccat].each { |node| node.remove! } - + @nodes[:entities] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:entities]}')]/..") @nodes[:entities].each { |node| node.remove! } - + @nodes[:relations] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:relations]}')]/..") @nodes[:relations].each { |node| node.remove! } - + @nodes[:geographies] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:geographies]}')]/..") @nodes[:geographies].each { |node| node.remove! } - + @nodes[:instances] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:instances]}')]/..") @nodes[:instances].each { |node| node.remove! } - + @nodes[:relevances] = doc.root.find("rdf:Description/rdf:type[contains(@rdf:resource, '#{MATCHERS[:relevances]}')]/..") @nodes[:relevances].each { |node| node.remove! } - + @nodes[:others] = doc.root.find("./*") @nodes[:others].each { |node| node.remove! } - + return end - + def extract_attributes(nodes) nodes.inject({}) do |hsh, node| value = if node['resource'] @@ -106,44 +106,57 @@ module Calais hsh.merge(node.name => value) end end - + + def process_relevances + @nodes[:relevances].each do |node| + subject_hash = node.find_first("c:subject")[:resource].split('/')[-1] + + @relevances[subject_hash] = node.find_first("c:relevance").content.to_f + end + + @relevances + end + def process_entities @entities = @nodes[:entities].map do |node| extracted_hash = node['about'].split('/')[-1] rescue nil - + entity = Entity.new entity.hash = CalaisHash.find_or_create(extracted_hash, @hashes) entity.type = node.find("*[name()='rdf:type']")[0]['resource'].split('/')[-1] rescue nil entity.attributes = extract_attributes(node.find("*[contains(name(), 'c:')]")) - + + relevance = @relevances[extracted_hash] + entity.relevance = relevance if relevance + entity end end - + def process_relations @relations = @nodes[:relations].map do |node| extracted_hash = node['about'].split('/')[-1] rescue nil - + relation = Relation.new relation.hash = CalaisHash.find_or_create(extracted_hash, @hashes) relation.type = node.find("*[name()='rdf:type']")[0]['resource'].split('/')[-1] rescue nil relation.attributes = extract_attributes(node.find("*[contains(name(), 'c:')]")) - + relation end end - + def process_geographies @geographies = @nodes[:geographies].map do |node| attributes = extract_attributes(node.find("*[contains(name(), 'c:')]")) - + geography = Geography.new geography.name = attributes.delete('name') geography.hash = attributes.delete('subject') geography.attributes = attributes - + geography end end end -end \ No newline at end of file +end lib/calais/response.rb @@ -11,19 +11,44 @@ describe Calais::Response, :new do before :all do @response = Calais::Response.new(SAMPLE_RESPONSE) end - + it 'should extract entities' do entities = @response.entities entities.map { |e| e.type }.sort.uniq.should == %w[City Continent Country IndustryTerm Organization Person ProvinceOrState] end - + it 'should extract relations' do relations = @response.relations relations.map { |e| e.type }.sort.uniq.should == %w[GenericRelations PersonAttributes PersonProfessional Quotation] end - + it 'should extract geographies' do geographies = @response.geographies geographies.map { |e| e.name }.sort.uniq.should == %w[Australia Hobart,Tasmania,Australia Tasmania,Australia] end -end \ No newline at end of file + + it 'should extract relevances' do + @response.instance_variable_get("@relevances").size.should == 10 + end + + it 'should assign a floating-point relevance to each entity' do + @response.entities.each {|e| e.relevance.class.should == Float } + end + + it 'should assign the correct relevance to each entity' do + correct_relevances = { + "84a34c48-25ac-327f-a805-7b81fd570f7d" => 0.725, + "9853f11e-5efa-3efc-90b9-0d0450f7d673" => 0.396, + "9fa3fb8a-f517-32c7-8a46-3c1506ea3a70" => 0.156, + "ed0e83f9-87e8-3da6-ab46-cd6be116357c" => 0.291, + "e05f3d33-1622-3172-836c-b48637a156d3" => 0.316, + "d0ca04b6-9cf5-3595-ad4b-7758a0b57997" => 0.156, + "0bb9cdb4-3cb7-342a-9901-6d1f12b32f6a" => 0.31, + "3979e581-0823-3e84-9257-1ca36db4665e" => 0.228, + "0c3d5340-106f-390e-92d3-a4aa18004fb8" => 0.158, + "3bcf2655-ff2a-3a80-8de4-558b9626ad21" => 0.644 + } + @response.entities.each {|e| correct_relevances[e.hash.value].should == e.relevance } + end + +end spec/calais/response_spec.rb 69a89bdc26a29ee848cb99f8f9ece4fb6fc55573 Paul Legato pjlegato@gmail.com http://github.com/abhay/calais/commit/f65b067e4063de23c766ff88f18cbdee58e4e187 f65b067e4063de23c766ff88f18cbdee58e4e187 2008-11-16T15:28:47-08:00 2008-11-16T15:28:47-08:00 Add entity relevance processing and associated spec. Add spec/fixtures/calais.yml to .gitignore (contains the API key.) 15b75f80e68837f66cd9af0b9a11d228b6ca5761 Paul Legato pjlegato@gmail.com