Skip to content


Subversion checkout URL

You can clone with
Download ZIP
Branch: master
Fetching contributors…

Cannot retrieve contributors at this time

105 lines (92 sloc) 2.95 KB
import httplib, urllib
import json
import datetime
import time
import random
from discodb import DiscoDB
import os
from disco.error import CommError
import db
import util
keys = json.load(open('keys'))
def fetch(doi, delay=0.3):
before = time.time()
# conn = httplib.HTTPConnection('')
conn = httplib.HTTPConnection('')
path = '/metadata/json?%s' % urllib.urlencode({'q':'doi:'+doi, 'api_key':keys['metadata']})
conn.request('GET', path)
response = conn.getresponse()
status = response.status
data =
after = time.time()
time.sleep(max(0, delay + before - after))
if status == 200:
meta = util.encode(json.loads(data)) # !!! metadata encoding?
if meta['records']:
return meta
# sometimes get an empty response rather than a 404
raise db.NotFound('fetch:empty', doi)
elif status == 404:
raise db.NotFound('fetch:404', doi)
raise CommError(data, '' + path, code=status)
# for testing
def fake(doi):
def string():
return random.choice('qwertyuiop')
def keyed(key, gen):
return {key: gen()}
def value(gen):
return {'value':gen(), 'count':'1'}
def some(gen, args):
return [gen(*args) for i in xrange(0,int(random.expovariate(0.5)))]
def date():
year = 2010
month = random.randint(10,12)
day = random.randint(1,28)
return '%04d-%02d-%02d' % (year, month, day)
return {
'identifier':'doi:%s' % doi,
'title': string(),
'publicationDate': date(),
'creators': some(keyed, ['creator', string]),
'publicationName': string(),
'issn': string(),
'values':some(value, [string])
# raises db.NotFound
def get(doi):
return db.get('metadata', doi)
def features(doi, meta):
if not meta['records'][0].has_key('doi'):
yield 'doi:%s' % doi
for key, value in meta['records'][0].items():
if (type(value) is str):
yield '%s:%s' % (key, value)
elif type(value) is list:
for subvalue in value:
yield '%s:%s' % (key, subvalue.values()[0])
for facet in meta['facets']:
key = facet['name']
for value in facet['values']:
value = value['value']
yield '%s:%s' % (key, value)
#raises db.NotFound
def publication_date(doi):
# raises db.NotFound
def publication(doi):
record = get(doi)['records'][0]
if record.has_key('isbn'):
return 'isbn:' + record['isbn']
elif record.has_key('issn'):
return 'issn:' + record['issn']
raise NotFound('publication', doi)
Jump to Line
Something went wrong with that request. Please try again.