diff --git a/dbpedialite.rb b/dbpedialite.rb index d5d8bf8..c1cdd6a 100644 --- a/dbpedialite.rb +++ b/dbpedialite.rb @@ -193,13 +193,6 @@ def format_iso8061(datetime) redirect_from_title($2) elsif params[:url] =~ %r{^https?://(www\.)?wikidata.org/(wiki|entity)/(Q\d+)$} redirect_from_wikidata($3) - elsif params[:url] =~ %r{^https?://www.freebase.com/(view|inspect|edit/topic)(/.+)$} - begin - data = FreebaseApi.lookup_by_id($2) - redirect "/things/#{data['key']['value']}", 301 - rescue FreebaseApi::NotFound - not_found("No Wikipedia page id found for Freebase topic") - end elsif params[:url] =~ %r{^https?://([\w\.\-\:]+)/(things|categories)/(\d+)(\#\w*)?$} begin data = WikipediaApi.page_info(:pageids => $3) diff --git a/lib/freebase_api.rb b/lib/freebase_api.rb deleted file mode 100644 index 268a57c..0000000 --- a/lib/freebase_api.rb +++ /dev/null @@ -1,83 +0,0 @@ -require 'net/https' -require 'uri' -require 'cgi' - -module FreebaseApi - USER_AGENT = 'DbpediaLite/1' - MQLREAD_URI = URI.parse('https://www.googleapis.com/freebase/v1/mqlread') - RDF_BASE_URI = URI.parse('http://rdf.freebase.com/rdf/') - GOOGLE_API_KEY = ENV['GOOGLE_API_KEY'] - HTTP_TIMEOUT = 2 - - class Exception < Exception - end - - class NotFound < FreebaseApi::Exception - end - - def self.mqlread(query) - uri = MQLREAD_URI.clone - uri.query = 'query='+CGI::escape(JSON.dump(query)) - - unless GOOGLE_API_KEY.nil? - uri.query += "&key=#{GOOGLE_API_KEY}" - end - - http = Net::HTTP.new(uri.host, uri.port) - http.open_timeout = HTTP_TIMEOUT - http.read_timeout = HTTP_TIMEOUT - - if uri.scheme == 'https' - http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - end - - res = http.get(uri.request_uri, {'User-Agent' => USER_AGENT}) - - # Throw an exception if HTTP request was unsuccessful - res.value - - # Throw and exception if the JSON response was unsuccessful - data = JSON.parse(res.body) - if data.has_key?('error') - raise FreebaseApi::Exception.new("Freebase query failed: #{data['error']['message']}") - end - - if data['result'].nil? - raise FreebaseApi::NotFound.new("Freebase query returned no results") - end - - data['result'] - end - - def self.lookup_wikipedia_pageid(pageid, language='en') - mqlread({ - 'key' => { - 'namespace' => "/wikipedia/#{language}_id", - 'value' => pageid.to_s, - 'limit' => 0 - }, - 'id' => nil, - 'name' => nil, - 'mid' => nil, - 'guid' => nil, - 'limit' => 1 - }) - end - - def self.lookup_by_id(identifier, language='en') - mqlread({ - 'id' => identifier, - 'key' => { - 'namespace' => "/wikipedia/#{language}_id", - 'value' => nil, - 'limit' => 1 - }, - 'name' => nil, - 'mid' => nil, - 'guid' => nil, - 'limit' => 1 - }) - end - -end diff --git a/lib/thing.rb b/lib/thing.rb index c8f1dd7..ebb4c90 100644 --- a/lib/thing.rb +++ b/lib/thing.rb @@ -1,6 +1,5 @@ require 'wikipedia_api' require 'wikidata_api' -require 'freebase_api' require 'base_model' @@ -12,8 +11,6 @@ class Thing < BaseModel has :latitude, :kind => Float, :default => nil has :externallinks, :kind => Array, :default => [] has :updated_at, :kind => DateTime, :default => nil - has :freebase_guid, :kind => String, :default => nil - has :freebase_mid, :kind => String, :default => nil has :wikidata_id, :kind => String, :default => nil has :wikidata_label, :kind => String, :default => nil has :wikidata_description, :kind => String, :default => nil @@ -34,49 +31,6 @@ def load #end end - def freebase_guid - fetch_freebase_uris - @freebase_guid - end - - def freebase_mid - fetch_freebase_uris - @freebase_mid - end - - def freebase_guid_uri - fetch_freebase_uris - if freebase_guid - @freebase_guid_uri ||= RDF::URI.parse("http://rdf.freebase.com/ns/"+freebase_guid.sub('#','guid.')) - end - end - - def freebase_mid_uri - fetch_freebase_uris - if freebase_mid - @freebase_mid_uri ||= RDF::URI.parse("http://rdf.freebase.com/ns/"+freebase_mid.sub('/m/','m.')) - end - end - - def fetch_freebase_uris - # Only make call to freebase once - unless @called_freebase - @called_freebase = true - # Attempt to match to Freebase, but silently fail on error - begin - data = FreebaseApi.lookup_wikipedia_pageid(pageid) - self.freebase_mid = data['mid'] - self.freebase_guid = data['guid'] - rescue Timeout::Error => e - $stderr.puts "Timed out while reading from Freebase: #{e.message}" - rescue FreebaseApi::Exception => e - $stderr.puts "Error while reading from Freebase: #{e.message}" - rescue Net::HTTPServerException => e - $stderr.puts "Error while making HTTP request to Freebase: #{e.message}" - end - end - end - def wikidata_id fetch_wikidata @wikidata_id @@ -143,8 +97,6 @@ def to_rdf graph << [self.uri, RDF::RDFS.comment, abstract] graph << [self.uri, RDF::FOAF.isPrimaryTopicOf, wikipedia_uri] graph << [self.uri, RDF::OWL.sameAs, dbpedia_uri] - graph << [self.uri, RDF::OWL.sameAs, freebase_guid_uri] unless freebase_guid_uri.nil? - graph << [self.uri, RDF::OWL.sameAs, freebase_mid_uri] unless freebase_mid_uri.nil? graph << [self.uri, RDF::GEO.lat, latitude] unless latitude.nil? graph << [self.uri, RDF::GEO.long, longitude] unless longitude.nil? diff --git a/spec/dbpedialite_spec.rb b/spec/dbpedialite_spec.rb index 4e2ad2f..95be059 100644 --- a/spec/dbpedialite_spec.rb +++ b/spec/dbpedialite_spec.rb @@ -258,11 +258,6 @@ def app :body => fixture_data('wbgetentities-ceres.json'), :content_type => 'application/json' ) - FakeWeb.register_uri( - :get, %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-934787.json'), - :content_type => 'application/json' - ) end context "as an HTML document using content negotiation" do @@ -292,11 +287,6 @@ def app :body => fixture_data('parse-934787.json'), :content_type => 'application/json' ) - FakeWeb.register_uri( - :get, %r[http://api.freebase.com/api/service/mqlread], - :body => fixture_data('freebase-mqlread-934787.json'), - :content_type => 'application/json' - ) header "Accept", "text/plain" get '/things/934787' end @@ -449,11 +439,6 @@ def app :body => fixture_data('wbgetentities-imac.json'), :content_type => 'application/json' ) - FakeWeb.register_uri( - :get, %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-21492980.json'), - :content_type => 'application/json' - ) end context "as an HTML document" do @@ -518,7 +503,7 @@ def app end end - context "GETing an HTML thing for something that doesn't exist in Freebase or Wikidata" do + context "GETing an HTML thing for something that doesn't exist in Wikidata" do before :each do FakeWeb.register_uri( :get, 'https://en.wikipedia.org/w/api.php?action=parse&format=json&pageid=2008435&prop=text%7Cdisplaytitle', @@ -530,11 +515,6 @@ def app :body => fixture_data('wbgetentities-notfound.json'), :content_type => 'application/json' ) - FakeWeb.register_uri( - :get, %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-notfound.json'), - :content_type => 'application/json' - ) @stderr_buffer = StringIO.new previous_stderr, $stderr = $stderr, @stderr_buffer get '/things/2008435' @@ -740,26 +720,6 @@ def app end end - context "flipping from a Freebase page" do - before :each do - FakeWeb.register_uri( - :get, %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-en-new-york.json'), - :content_type => 'application/json' - ) - get '/flipr?url=http%3A%2F%2Fwww.freebase.com%2Fview%2Fen%2Fnew_york' - end - - it "should redirect to the coresponding Dbpedia lite thing page" do - last_response.status.should == 301 - last_response.location.should == 'http://example.org/things/645042' - end - - it "should be cachable" do - last_response.headers['Cache-Control'].should =~ /max-age=([1-9]+)/ - end - end - context "flipping from an unknown page" do before :each do get '/flipr?url=http%3A%2F%2Fwww.bbc.co.uk%2F' diff --git a/spec/fixtures/freebase-mqlread-21492980.json b/spec/fixtures/freebase-mqlread-21492980.json deleted file mode 100644 index 880e69f..0000000 --- a/spec/fixtures/freebase-mqlread-21492980.json +++ /dev/null @@ -1,9 +0,0 @@ -{ -"result": { -"mid": "/m/0mbxw", -"id": "/en/imac", -"guid": "#9202a8c04000641f800000000009ab9b", -"name": "iMac", -"key": null -} -} \ No newline at end of file diff --git a/spec/fixtures/freebase-mqlread-934787.json b/spec/fixtures/freebase-mqlread-934787.json deleted file mode 100644 index 256d65c..0000000 --- a/spec/fixtures/freebase-mqlread-934787.json +++ /dev/null @@ -1,9 +0,0 @@ -{ -"result": { -"name": "Ceres", -"key": null, -"guid": "#9202a8c04000641f80000000003bb45c", -"id": "/en/ceres_united_kingdom", -"mid": "/m/03rf2x" -} -} \ No newline at end of file diff --git a/spec/fixtures/freebase-mqlread-en-new-york.json b/spec/fixtures/freebase-mqlread-en-new-york.json deleted file mode 100644 index 43325fb..0000000 --- a/spec/fixtures/freebase-mqlread-en-new-york.json +++ /dev/null @@ -1,12 +0,0 @@ -{ -"result": { -"guid": "#9202a8c04000641f80000000002f8906", -"mid": "/m/02_286", -"key": { -"value": "645042", -"namespace": "/wikipedia/en_id" -}, -"name": "New York City", -"id": "/en/new_york" -} -} \ No newline at end of file diff --git a/spec/fixtures/freebase-mqlread-invalid.json b/spec/fixtures/freebase-mqlread-invalid.json deleted file mode 100644 index 9890fe2..0000000 --- a/spec/fixtures/freebase-mqlread-invalid.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "error": { - "errors": [ - { - "domain": "global", - "reason": "invalid", - "message": "Type /type/object does not have property foo" - } - ], - "code": 400, - "message": "Type /type/object does not have property foo" - } -} diff --git a/spec/fixtures/freebase-mqlread-notfound.json b/spec/fixtures/freebase-mqlread-notfound.json deleted file mode 100644 index 2a84edf..0000000 --- a/spec/fixtures/freebase-mqlread-notfound.json +++ /dev/null @@ -1,3 +0,0 @@ -{ -"result": null -} \ No newline at end of file diff --git a/spec/freebase_api_spec.rb b/spec/freebase_api_spec.rb deleted file mode 100644 index 3970ee0..0000000 --- a/spec/freebase_api_spec.rb +++ /dev/null @@ -1,128 +0,0 @@ -require 'spec_helper' -require 'freebase_api' - -describe FreebaseApi do - context "looking up the Freebase MID for a Wikipedia Pageid" do - before :each do - FakeWeb.register_uri(:get, - %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-934787.json') - ) - @data = FreebaseApi.lookup_wikipedia_pageid(934787) - end - - it "should return a Hash" do - @data.should be_a(Hash) - end - - it "should return the name of the topic" do - @data['name'].should == 'Ceres' - end - - it "should return the GUID for the topic" do - @data['guid'].should == '#9202a8c04000641f80000000003bb45c' - end - - it "should return the Machine ID for the topic" do - @data['mid'].should == '/m/03rf2x' - end - - it "should return the Freebase ID for the topic" do - @data['id'].should == '/en/ceres_united_kingdom' - end - end - - context "looking up the Wikipedia Pageid for a Freebase identifier" do - before :each do - FakeWeb.register_uri(:get, - %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-en-new-york.json') - ) - @data = FreebaseApi.lookup_by_id('/en/new_york') - end - - it "should return a Hash" do - @data.should be_a(Hash) - end - - it "should return the name of the topic" do - @data['name'].should == 'New York City' - end - - it "should return the GUID for the topic" do - @data['guid'].should == '#9202a8c04000641f80000000002f8906' - end - - it "should return the Machine ID for the topic" do - @data['mid'].should == '/m/02_286' - end - - it "should return the Freebase ID for the topic" do - @data['id'].should == '/en/new_york' - end - - it "should have a key for the English Wikipedia page id" do - @data['key']['namespace'].should == '/wikipedia/en_id' - end - - it "should return the Wikipedia page id" do - @data['key']['value'].should == '645042' - end - end - - context "making a request with a Google API key" do - before :each do - FreebaseApi.send(:remove_const, 'GOOGLE_API_KEY') - FreebaseApi.const_set('GOOGLE_API_KEY', 'mykey') - FakeWeb.register_uri(:get, - %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-en-new-york.json') - ) - @data = FreebaseApi.lookup_by_id('/en/new_york') - end - - after :each do - FreebaseApi.send(:remove_const, 'GOOGLE_API_KEY') - FreebaseApi.const_set('GOOGLE_API_KEY', nil) - end - - it "should put the Google API key in the query string" do - FakeWeb.last_request.path.should match(/key=mykey/) - end - - it "should return the name of the topic" do - @data['name'].should == 'New York City' - end - end - - context "lookup up a Wikipedia pageid that doesn't exist in Freebase" do - before :each do - FakeWeb.register_uri(:get, - %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-notfound.json') - ) - end - - it "should throw an exception" do - lambda do - FreebaseApi.lookup_wikipedia_pageid(4309010) - end.should raise_error(FreebaseApi::NotFound, 'Freebase query returned no results') - end - end - - context "making an invalid mqlread query" do - before :each do - FakeWeb.register_uri(:get, - %r[https://www.googleapis.com/freebase/v1/mqlread], - :body => fixture_data('freebase-mqlread-invalid.json') - ) - end - - it "should throw an exception" do - lambda do - FreebaseApi.mqlread(:foo => :bar) - end.should raise_error(FreebaseApi::Exception, 'Freebase query failed: Type /type/object does not have property foo') - end - end - -end diff --git a/spec/thing_spec.rb b/spec/thing_spec.rb index 56d0116..4a64821 100644 --- a/spec/thing_spec.rb +++ b/spec/thing_spec.rb @@ -149,71 +149,6 @@ @thing.abstract.should =~ /^Ceres is a village in Fife, Scotland/ end - context "when freebase responds with a parsable response" do - before :each do - freebase_data = { - 'guid' => '#9202a8c04000641f80000000003bb45c', - 'id' => '/en/ceres_united_kingdom', - 'mid' => '/m/03rf2x', - 'name' => 'Ceres', - } - allow(FreebaseApi).to receive(:lookup_wikipedia_pageid).and_return(freebase_data) - end - - it "should have a machine id property" do - @thing.freebase_mid.should == '/m/03rf2x' - end - - it "should have a GUID property" do - @thing.freebase_guid.should == '#9202a8c04000641f80000000003bb45c' - end - - it "should have a freebase URI based on the machine id" do - @thing.freebase_mid_uri.should == RDF::URI('http://rdf.freebase.com/ns/m.03rf2x') - end - - it "should have a freebase URI based on the guid" do - @thing.freebase_guid_uri.should == RDF::URI('http://rdf.freebase.com/ns/guid.9202a8c04000641f80000000003bb45c') - end - end - - context "when freebase times out" do - it "should send a message to stderr" do - allow(FreebaseApi).to receive(:lookup_wikipedia_pageid).and_raise(Timeout::Error) - previous_stderr, $stderr = $stderr, StringIO.new - - @thing.freebase_mid_uri - $stderr.string.should == "Timed out while reading from Freebase: Timeout::Error\n" - - $stderr = previous_stderr - end - end - - context "when FreebaseApi raises an a NotFound exception" do - it "should send a message to stderr" do - allow(FreebaseApi).to receive(:lookup_wikipedia_pageid).and_raise(FreebaseApi::NotFound) - previous_stderr, $stderr = $stderr, StringIO.new - - @thing.freebase_mid_uri - $stderr.string.should == "Error while reading from Freebase: FreebaseApi::NotFound\n" - - $stderr = previous_stderr - end - end - - context "when FreebaseApi raises an a HTTPServerException exception" do - it "should send a message to stderr" do - response = Net::HTTPForbidden.new("1.1", "403", "Forbidden") - allow(FreebaseApi).to receive(:lookup_wikipedia_pageid).and_raise(Net::HTTPServerException.new('403 "Forbidden"', response)) - previous_stderr, $stderr = $stderr, StringIO.new - - @thing.freebase_mid_uri - $stderr.string.should == "Error while making HTTP request to Freebase: 403 \"Forbidden\"\n" - - $stderr = previous_stderr - end - end - it "should have a single external like of type RDF::URI" do @thing.externallinks.should == [RDF::URI('http://www.fife.50megs.com/ceres-history.htm')] end @@ -230,24 +165,10 @@ 'There is no page with ID 999999' ) end - - it "should return raise a PageNotFound exception" do - expect(FreebaseApi).to_not receive(:lookup_wikipedia_pageid) - lambda {Thing.load(999999)}.should raise_error( - MediaWikiApi::NotFound, - 'There is no page with ID 999999' - ) - end end context "converting a thing to RDF" do before :each do - allow(FreebaseApi).to receive(:lookup_wikipedia_pageid).and_return({ - 'guid' => '#9202a8c04000641f8000000000066c8e', - 'id' => '/en/u2', - 'mid' => '/m/0dw4g', - 'name' => 'U2', - }) allow(WikidataApi).to receive(:find_by_title).and_return({ 'id' => 'q396', 'title'=> 'Q396', @@ -274,8 +195,8 @@ @graph.class.should == RDF::Graph end - it "should return a graph with 22 triples" do - @graph.count.should == 22 + it "should return a graph with 20 triples" do + @graph.count.should == 20 end it "should include an rdf:type triple for the thing" do @@ -326,22 +247,6 @@ ]) end - it "should include a owl:sameAs triple for the FreeBase Machine ID" do - @graph.should have_triple([ - RDF::URI("http://www.dbpedialite.org/things/52780#id"), - RDF::OWL.sameAs, - RDF::URI("http://rdf.freebase.com/ns/m.0dw4g") - ]) - end - - it "should include a owl:sameAs triple for the FreeBase GUID" do - @graph.should have_triple([ - RDF::URI("http://www.dbpedialite.org/things/52780#id"), - RDF::OWL.sameAs, - RDF::URI("http://rdf.freebase.com/ns/guid.9202a8c04000641f8000000000066c8e") - ]) - end - it "should include a rdf:type triple for the document" do @graph.should have_triple([ RDF::URI("http://www.dbpedialite.org/things/52780"),