Skip to content

Commit

Permalink
changing handling of collections from frontend to get them working pr…
Browse files Browse the repository at this point in the history 
…operly again now that collections are stored as separate objects
  • Loading branch information
Mark MacGillivray committed Oct 2, 2011
1 parent 13b938b commit 937d3e2
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 86 deletions.
18 changes: 9 additions & 9 deletions bibserver/importer.py
View file
Expand Up @@ -30,37 +30,37 @@ def upload(self, fileobj, format_, collection=None):
'''
parser = Parser()
record_dicts = parser.parse(fileobj, format=format_)
collection_from_parser = None
if collection_from_parser:
collection = collection_from_parser
#collection_from_parser = None
#if collection_from_parser:
# collection = collection_from_parser
# TODO: check authz for write to this collection
collection['records'] = len(record_dicts)
return self.index(collection, record_dicts)

def upload_from_web(self, request):
'''
:param request_data: Flask request.values attribute.
'''
pkg = dict()
format = 'bibtex'
source = ''
if request.values.get("source"):
source = urllib2.unquote(request.values.get("source", ''))
fileobj = urllib2.urlopen(source)
pkg["source"] = source
format = self.findformat(source)
elif request.files.get('upfile'):
fileobj = request.files.get('upfile')
format = self.findformat(fileobj.filename)
elif request.values.get('data'):
fileobj = StringIO(request.values['data'])

if request.values.get('format'):
format = request.values.get('format')
pkg["format"] = format

if not 'collection' in request.values:
raise ValueError('You must provide a collection label')
collection = {
'label': request.values['collection']
'label': request.values['collection'],
'source': source,
'format': format
}
collection, records = self.upload(fileobj, format, collection)
return (collection, records)
Expand Down Expand Up @@ -128,7 +128,7 @@ def index(self, collection_dict, record_dicts):
bibserver.dao.Record.delete_by_query('collection.exact:"' +
collection.id + '"')
for rec in record_dicts:
rec['collection'] = collection.id

This comment has been minimized.

Sign in to view

Copy link
@rufuspollock

rufuspollock Oct 6, 2011

Contributor

Why have we change this from storing the collection id to the collection slug? The collection slug is not unique across collections (only the tuple (userid, slug) is). I am concerned this may cause bugs e.g. the delete of records from existing collection seems to just use the slug rather than slug plus userid. In addition good practice is to use the id for "foreign keys". Why violate this for collection references from records? If it is to do with doing display then IMO it would be better to get collection out when we are doing display rather than change what we store ...

Aside: would you mind dividing up commits a bit more. This commit not only changes this attribute on records but does stuff with graph.html (is it related). In addition it also seems to have changes that should have been part of a merge (I deleted "pkg" stuff back in my commits that you in theory had merged :-) ). Smaller, more granular commits mean easier understanding :-)

This comment has been minimized.

Sign in to view

Copy link
@markmacgillivray

markmacgillivray via email Oct 6, 2011

Contributor
rec['collection'] = collection["slug"]
records = bibserver.dao.Record.bulk_upsert(record_dicts)
return collection, records

Expand Down
21 changes: 10 additions & 11 deletions bibserver/iomanager.py
View file
Expand Up @@ -137,27 +137,26 @@ def get_str(self, result, field, raw=False):

def get_meta(self):
try:
if "collection/" in self.args['path']:
coll = self.results['hits']['hits'][0]["_source"]["collection"]
if isinstance(coll,list):
coll = coll[0]
res = bibserver.dao.Record.query(q='collection' + self.config.facet_field + ':"' + coll + '" AND type:collection')
if self.args['path'].startswith('collection'):
coll = self.args['path'].replace('collection/','')
print coll
res = bibserver.dao.Collection.query(q='slug:"' + coll + '"')
rec = res["hits"]["hits"][0]["_source"]
sizer = bibserver.dao.Record.query(q='collection' + self.config.facet_field + ':"' + coll + '"')
print rec

meta = '<p><a href="/'
meta += self.args['path'] + '.json?size=' + str(sizer["hits"]["total"])
meta += self.args['path'] + '.json?size=' + str(rec['records'])
meta += '">Download this collection</a><br />'
if "source" in rec:
meta += 'The source of this collection is <a href="'
meta += rec["source"] + '">' + rec["source"] + '</a>.<br /> '
if "received" in rec:
meta += 'This collection was last updated on ' + rec["received"] + '. '
if "modified" in rec:
meta += 'This collection was last updated on ' + rec["modified"] + '. '
if "source" in rec:
meta += '<br />If changes have been made to the source file since then, '
meta += '<a href="/upload?source=' + rec["source"] + '&collection=' + rec["collection"]
meta += '<a href="/upload?source=' + rec["source"] + '&collection=' + rec["slug"]
meta += '">refresh this collection</a>.'
meta += '<br /><a class="delete_link" href="/query?delete=true&q=collection.exact:%22' + rec["collection"] + '%22">Delete this collection</a></p>'
meta += '<br /><a class="delete_link" href="/query?delete=true&q=collection.exact:%22' + rec["slug"] + '%22">Delete this collection</a></p>'
return meta
else:
return ""
Expand Down
89 changes: 43 additions & 46 deletions bibserver/static/graph.html
View file
Expand Up @@ -4,10 +4,10 @@
<meta charset="UTF-8" />
<title>facet graphs</title>

<script src="jquery.js"></script>
<script type="text/javascript" src="/static/d3/d3.js"></script>
<script type="text/javascript" src="/static/d3/d3.geom.js?2.1.3"></script>
<script type="text/javascript" src="/static/d3/d3.layout.js?2.1.3"></script>
<script src="vendor/jquery.js"></script>
<script type="text/javascript" src="vendor/d3/d3.js"></script>
<script type="text/javascript" src="vendor/d3/d3.geom.js?2.1.3"></script>
<script type="text/javascript" src="vendor/d3/d3.layout.js?2.1.3"></script>

<style type="text/css">
body {
Expand Down Expand Up @@ -51,7 +51,7 @@
$('svg').remove();
load_data($(this).val());
}
$('#facet').bind('change',runit).bind('keyup',isenter);
$('#facet').bind('change',runit);//.bind('keyup',isenter);
$('#submit').bind('click',runit);

var load_data = function(facet) {
Expand Down Expand Up @@ -87,6 +87,44 @@
});
};

var bubble = function(json) {
data = {"children":[]};
var count = 0;
for (var item in json.facets.facet.terms) {
var arr = {
"className": json.facets.facet.terms[item].term,
"packageName": count += 1,
"value": json.facets.facet.terms[item].count
};
data["children"].push(arr);
};
var r = 960,
format = d3.format(",d"),
fill = d3.scale.category20c();
var bubble = d3.layout.pack()
.sort(null)
.size([r, r]);
var vis = d3.select("body").append("svg:svg")
.attr("width", r)
.attr("height", r)
.attr("class", "bubble");
var node = vis.selectAll("g.node")
.data(bubble(data)
.filter(function(d) { return !d.children; }))
.enter().append("svg:g")
.attr("class", "node")
.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
node.append("svg:title")
.text(function(d) { return d.data.className + ": " + format(d.value); });
node.append("svg:circle")
.attr("r", function(d) { return d.r; })
.style("fill", function(d) { return fill(d.data.packageName); });
node.append("svg:text")
.attr("text-anchor", "middle")
.attr("dy", ".3em")
.text(function(d) { return d.data.className + " (" + d.data.value + ")"; });
}

var barchart = function(json) {
var data = []
var titles = []
Expand Down Expand Up @@ -192,47 +230,6 @@
.text(function(d, i) { return data[i].label; }); //get the label from our original data array
}

var bubble = function(json) {
//data = {"children":[]};
var data = [];
var count = 0;
for (var item in json.facets.facet.terms) {
var arr = {
"className": json.facets.facet.terms[item].term,
"packageName": count += 1,
"value": json.facets.facet.terms[item].count
};
//data["children"].push(arr);
data.push(arr);
};
var r = 960,
format = d3.format(",d"),
fill = d3.scale.category20c();
var bubble = d3.layout.pack()
.sort(null)
.size([r, r]);
var vis = d3.select("body").append("svg:svg")
.attr("width", r)
.attr("height", r)
.attr("class", "bubble");
var node = vis.selectAll("g.node")
//.data(bubble(data)
//.filter(function(d) { return !d.children; }))
.data(bubble(data))
.enter().append("svg:g")
.attr("class", "node")
.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
node.append("svg:title")
.text(function(d) { return d.data.className + ": " + format(d.value); });
node.append("svg:circle")
.attr("r", function(d) { return d.r; })
.style("fill", function(d) { return fill(d.data.packageName); });
node.append("svg:text")
.attr("text-anchor", "middle")
.attr("dy", ".3em")
.text(function(d) { return d.data.className + " (" + d.data.value + ")"; });
}

</script>

</body>
Expand Down
27 changes: 7 additions & 20 deletions bibserver/web.py
View file
Expand Up @@ -45,9 +45,9 @@ def standard_authentication():
def home():
# get list of available collections
try:
result = bibserver.dao.Record.query(q="type.exact:collection",sort={"received.exact":{"order":"desc"}})
result = bibserver.dao.Collection.query(q="*",sort={"created":{"order":"desc"}})
if result["hits"]["total"] != 0:
colls = [i["_source"]["collection"] for i in result["hits"]["hits"]]
colls = [i["_source"]["label"] for i in result["hits"]["hits"]]
except:
colls = None
return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )
Expand Down Expand Up @@ -162,6 +162,7 @@ def search(path=''):

# get implicit facet
c = {'implicit_facet': {}}
results = None
if path != '' and not path.startswith("search"):
path = path.strip()
if path.endswith("/"):
Expand All @@ -171,26 +172,12 @@ def search(path=''):
# its an implicit facet
args['terms'][bits[0]+config["facet_field"]] = [bits[1]]
c['implicit_facet'][bits[0]] = bits[1]
elif len(bits) == 1:
# send request through as an implicit facet on type, if said type exists
qry = 'type' + config["facet_field"] + ':' + bits[0]
check = bibserver.dao.Record.query(q=qry,size=1)
if check["hits"]["total"] != 0:
c['implicit_facet']["type"] = bits[0]
args['terms']["type"+config["facet_field"]] = [bits[0]]
else:
# otherwise just show a listing of the facet values for that key
if 'q' in args:
qryval = args['q']
else:
qryval = "*:*"
result = bibserver.dao.Record.query(q=qryval,facet_fields=[bits[0]+config["facet_field"]])
vals = result["facets"][bits[0]+config["facet_field"]]["terms"]
#return render_template('search/listing.html', vals=vals)
elif len(bits) == 1 and bits[0] == "collection":
results = bibserver.dao.Collection.query(**args)


# get results and render
results = bibserver.dao.Record.query(**args)
if not results:
results = bibserver.dao.Record.query(**args)
args['path'] = path
c['io'] = bibserver.iomanager.IOManager(results, args)

Expand Down

0 comments on commit 937d3e2

Please sign in to comment.