Permalink
Browse files

lots of changes in the refactor process. combined setconfig and confi…

…g into config. combined urlmanager and resultmanager into iomanager. sorted the search so that text search and facet search are done together. added some layout changes, simplifying how params are passed back and fore. Have not yet actually changed the param structure, but in a much better state now to do so.
  • Loading branch information...
1 parent 64ea391 commit be8c74e7e4dfce01cc4a0939c86365291dad5e9b @markmacgillivray markmacgillivray committed Sep 7, 2011
View
131 bibserver/config.py
@@ -1,4 +1,7 @@
import json
+
+'''read the config.json file and make available as a config dict'''
+
def load_config(path):
fileobj = open(path)
c = ""
@@ -8,10 +11,136 @@ def load_config(path):
else:
c += line
out = json.loads(c)
+
+ out["facet_fields_display"] = {}
+ for item in out["facet_fields"]:
+ out["facet_fields_display"][item + out["facet_field"]] = item
+ out["facet_fields"] = [field + out["facet_field"] for field in out["facet_fields"]]
+
return out
-# TODO: make this less hacky and more configurable
config = load_config('config.json')
__all__ = ['config']
+
+''' wrap a config dict in a class and provide methods for displaying fields'''
+
+class Config(object):
+ def __init__(self,confdict):
+ '''Create Configuration object from a configuration dictionary.'''
+ self.cfg = confdict
+
+ def __getattr__(self, attr):
+ return self.cfg.get(attr, None)
+
+ def get_facet_display(self, facet_name):
+ if self.facet_fields_display.has_key(facet_name):
+ return self.facet_fields_display[facet_name]
+ else:
+ return facet_name
+
+ # run the desired method on a field content, to alter it for display
+ def get_field_display(self, field, value):
+ if self.display_value_functions.has_key(field):
+ d = self.display_value_functions[field]
+ func_name = d.keys()[0]
+ args = d[func_name]
+ args["field"] = field
+ func = globals()[func_name]
+ return func(str(value), args)
+ else:
+ return value
+
+
+# the following methods can be called by get_field_display
+# to perform various functions upon a field for display
+
+def wrap(value, dict):
+ return dict['start'] + value + dict['end']
+
+def doiify(value, dict):
+ # dois may start with:
+ # 10. - prefix http://dx.doi.org/
+ # doi: - strip doi: and replace with http://dx.doi.org/
+ # http://dx.doi.org/ already done, just linkify
+ resolver = dict.get("resolver", "http://dx.doi.org/")
+ link = None
+ if value.startswith("10."):
+ link = resolver + value
+ elif value.startswith("doi:"):
+ link = resolver + value[4:]
+ elif value.startswith("http://"):
+ link = value
+
+ if link is not None:
+ return '<a href="' + link + '">' + value + '</a>'
+ else:
+ return value
+
+def searchify(value, dict):
+ # for the given value, make it a link to a search of the value
+ return '<a href="?q=' + value + '" alt="search for ' + value + '" title="search for ' + value + '">' + value + '</a>'
+
+def implicify(value, dict):
+ # for the given value, make it a link to an implicit facet URL
+ return '<a href="/' + dict.get("field") + "/" + value + '" alt="go to ' + dict.get("field") + " - " + value + '" title="go to ' + dict.get("field") + " - " + value + '">' + value + '</a>'
+
+def _get_location_pairs(message, start_sub, finish_sub):
+ idx = 0
+ pairs = []
+ while message.find(start_sub, idx) > -1:
+ si = message.find(start_sub, idx)
+ sf = message.find(finish_sub, si)
+ pairs.append((si, sf))
+ idx = sf
+ return pairs
+
+def _create_url(url):
+ return "<a href=\"%(url)s\">%(url)s</a>" % {"url" : url}
+
+def linkify(nm, args):
+ parts = _get_location_pairs(nm, "http://", " ")
+
+ # read into a sortable dictionary
+ dict = {}
+ for (s, f) in parts:
+ dict[s] = f
+
+ # sort the starting points
+ keys = dict.keys()
+ keys.sort()
+
+ # determine the splitting points
+ split_at = [0]
+ for s in keys:
+ f = dict.get(s)
+ split_at.append(s)
+ split_at.append(f)
+
+ # turn the splitting points into pairs
+ pairs = []
+ for i in range(0, len(split_at)):
+ if split_at[i] == -1:
+ break
+ if i + 1 >= len(split_at):
+ end = len(nm)
+ elif split_at[i+1] == -1:
+ end = len(nm)
+ else:
+ end = split_at[i+1]
+ pair = (split_at[i], end)
+ pairs.append(pair)
+
+ frags = []
+ for s, f in pairs:
+ frags.append(nm[s:f])
+
+ for i in range(len(frags)):
+ if frags[i].startswith("http://"):
+ frags[i] = _create_url(frags[i])
+
+ message = "".join(frags)
+ return message
+
+
View
17 bibserver/dao.py
@@ -22,8 +22,8 @@ def init_db():
pass
def get_conn():
- host = str(config['ELASTIC_SEARCH_HOST'])
- db_name = config['ELASTIC_SEARCH_DB']
+ host = str(config["ELASTIC_SEARCH_HOST"])
+ db_name = config["ELASTIC_SEARCH_DB"]
conn = pyes.ES([host])
return conn, db_name
@@ -131,22 +131,23 @@ def query(cls, q='', terms=None, facet_fields=None, **kwargs):
ourq = pyes.query.StringQuery(q, default_operator='AND')
if terms:
for term in terms:
- termq = pyes.query.TermQuery(term, terms[term])
- ourq = pyes.query.BoolQuery(must=[ourq,termq])
+ for val in terms[term]:
+ termq = pyes.query.TermQuery(term, val)
+ ourq = pyes.query.BoolQuery(must=[ourq,termq])
+
ourq = ourq.search(**kwargs)
if facet_fields:
for field in facet_fields:
- ourq.facet.add_term_facet(field)
+ ourq.facet.add_term_facet(field, size=100)
out = conn.search(ourq, db, cls.__type__)
return out
@classmethod
def raw_query(self, query_string):
if not query_string:
msg = json.dumps({
- 'error': "Query endpoint. Please provide elastic search <a href='%s'>query parameters.</a>" % (
- 'http://www.elasticsearch.org/guide/reference/api/search/uri-request.html'
- )})
+ 'error': "Query endpoint. Please provide elastic search query parameters - see http://www.elasticsearch.org/guide/reference/api/search/uri-request.html"
+ })
return msg
host = str(config['ELASTIC_SEARCH_HOST']).rstrip('/')
View
123 bibserver/resultmanager.py
@@ -1,123 +0,0 @@
-import operator, unicodedata
-import bibserver.config
-import bibserver.dao
-
-class ResultManager(object):
- def __init__(self, results, config, args):
- self.results = results
- self.config = config
- self.args = args if args is not None else self.config.get_default_args()
- # TODO: set from query args properly rather than hardcode
- self.start = 0
- self.facet_counts = {
- 'facet_fields': {}
- }
- for facet,data in self.results['facets'].items():
- self.facet_counts['facet_fields'][facet] = {}
- for termdict in data['terms']:
- self.facet_counts['facet_fields'][facet][termdict['term']] = termdict['count']
-
- def get_ordered_facets(self, facet):
- if facet in self.config.facet_fields:
- return self._sort_facets_by_count(self.facet_counts['facet_fields'][facet])
-
- def in_args(self, facet, value=None):
- if value is not None:
- return self.args['q'].has_key(facet) and value in self.args['q'][facet]
- else:
- return self.args['q'].has_key(facet)
-
- def get_search_constraints(self):
- return self.args['q']
-
- def has_values(self, facet):
- if facet in self.config.facet_fields:
- return len(self.facet_counts['facet_fields'][facet]) > 0
- return False
-
- def is_start(self):
- return True
-
- def is_end(self):
- return True
-
- def set_size(self):
- return len(self.results['hits']['hits'])
-
- def numFound(self):
- return int(self.results['hits']['total'])
-
- def set(self):
- '''Return list of search result items'''
- return [rec['_source'] for rec in self.results['hits']['hits']]
-
- def start_offset(self, off):
- return self.start + off
-
- def finish(self):
- return self.start + self.set_size()
-
- def page_size(self):
- return self.args['rows']
-
- def get_str(self, result, field, raw=False):
- if result.get(field) is None:
- return ""
- if raw:
- if hasattr(result.get(field), "append"):
- return ", ".join([val for val in result.get(field)])
- else:
- return result.get(field)
- if hasattr(result.get(field), "append"):
- return ", ".join([self.config.get_field_display(field, val) for val in result.get(field)])
- else:
- return self.config.get_field_display(field, result.get(field))
-
- def get_meta(self):
- try:
- coll = self.results['hits']['hits'][0]["_source"]["collection"]
- if isinstance(coll,list):
- coll = coll[0]
- res = bibserver.dao.Record.query(q='collection:' + coll + ' AND type:collection')
- rec = res["hits"]["hits"][0]["_source"]
- meta = "<p>"
- if "source" in rec:
- meta = 'The source of this collection is <a href="'
- meta += rec["source"] + '">' + rec["source"] + '</a>. '
- if "received" in rec:
- meta += 'This collection was last updated on ' + rec["received"] + '. '
- if "source" in rec:
- meta += '<br />If changes have been made to the source file since then, '
- meta += '<a href="/upload?source=' + rec["source"] + '&collection=' + rec["collection"]
- meta += '">refresh this collection</a>.'
- meta += '</p>'
- return meta
- except:
- return ""
-
- def first_page_end(self):
- return self.args['rows']
-
- def last_page_start(self):
- return self.results.numFound - (self.results.numFound % self.args['rows'])
-
- def get_previous(self, num):
- pairs = []
- for i in range(num + 1, 0, -1):
- first = self.start - (self.args['rows'] * i)
- if first >= self.args['rows']: # i.e. greater than the first page
- pairs.append((first, first + self.args['rows']))
- return sorted(pairs, key=operator.itemgetter(0))
-
- def get_next(self, num):
- pairs = []
- for i in range(1, num + 1):
- first = self.start + (self.args['rows'] * i)
- last_page_size = self.numFound() % self.args['rows']
- if first + self.args['rows'] <= self.numFound() - last_page_size: # i.e. less than the last page
- pairs.append((first, first + self.args['rows']))
- return sorted(pairs, key=operator.itemgetter(0))
-
- def _sort_facets_by_count(self, dict):
- return sorted(dict.iteritems(), key=operator.itemgetter(1), reverse=True)
-
View
189 bibserver/setconfig.py
@@ -1,189 +0,0 @@
-from copy import deepcopy
-
-import bibserver.config
-
-
-class Configuration(object):
- def __init__(self, configuration_dict):
- '''Create Configuration object from a configuration dictionary.'''
- self.cfg = configuration_dict
-
- # build a map for display of fields
- self.facet_display_values = {}
- for field, props in self.cfg['facet_fields'].iteritems():
- self.facet_display_values[field] = props["display"]
-
- def __getattr__(self, attr):
- return self.cfg.get(attr, None)
-
- def get_facet_display(self, facet_name):
- return self.facet_display_values.get(facet_name, facet_name)
-
- def get_default_args(self):
- fields = deepcopy(self.facet_fields.keys())
- return {
- "q" : {},
- "start" : 0,
- "facet_field" : fields,
- "rows" : self.default_results_per_page
- }
-
- def get_value_display(self, facet, value):
- return value
-
- def get_field_display(self, field, value):
- if self.display_value_functions.has_key(field):
- d = self.display_value_functions[field]
- func_name = d.keys()[0]
- args = d[func_name]
- args["field"] = field
- func = globals()[func_name]
- return func(str(value), args)
- else:
- return value
-
- def get_field_name(self, field):
- return self.display_fields.get(field, field)
-
- def display_upper(self, facet, lower, upper):
- if self.upper_display_functions.has_key(facet):
- d = self.upper_display_functions[facet]
- func_name = d.keys()[0]
- args = d[func_name]
- func = globals()[func_name]
- return func(lower, upper, args)
- else:
- return True
-
-
-# the following methods probably no longer belong in here - should be in resultmanager, I think
-
-def value_map(value, d):
- return d.get(value, value)
-
-def regex_map(value, dict):
- capture = re.findall(dict["expression"], value)
- if len(capture) == 1:
- return capture[0]
- return value
-
-def wrap(value, dict):
- return dict['start'] + value + dict['end']
-
-def doiify(value, dict):
- # dois may start with:
- # 10. - prefix http://dx.doi.org/
- # doi: - strip doi: and replace with http://dx.doi.org/
- # http://dx.doi.org/ already done, just linkify
- resolver = dict.get("resolver", "http://dx.doi.org/")
- link = None
- if value.startswith("10."):
- link = resolver + value
- elif value.startswith("doi:"):
- link = resolver + value[4:]
- elif value.startswith("http://"):
- link = value
-
- if link is not None:
- return '<a href="' + link + '">' + value + '</a>'
- else:
- return value
-
-def searchify(value, dict):
- # for the given value, make it a link to a search of the value
- return '<a href="?q=' + value + '" alt="search for ' + value + '" title="search for ' + value + '">' + value + '</a>'
-
-def implicify(value, dict):
- # for the given value, make it a link to an implicit facet URL
- return '<a href="/' + dict.get("field") + "/" + value + '" alt="go to ' + dict.get("field") + " - " + value + '" title="go to ' + dict.get("field") + " - " + value + '">' + value + '</a>'
-
-def _get_location_pairs(message, start_sub, finish_sub):
- idx = 0
- pairs = []
- while message.find(start_sub, idx) > -1:
- si = message.find(start_sub, idx)
- sf = message.find(finish_sub, si)
- pairs.append((si, sf))
- idx = sf
- return pairs
-
-def _create_url(url):
- return "<a href=\"%(url)s\">%(url)s</a>" % {"url" : url}
-
-def linkify(nm, args):
- parts = _get_location_pairs(nm, "http://", " ")
-
- # read into a sortable dictionary
- dict = {}
- for (s, f) in parts:
- dict[s] = f
-
- # sort the starting points
- keys = dict.keys()
- keys.sort()
-
- # determine the splitting points
- split_at = [0]
- for s in keys:
- f = dict.get(s)
- split_at.append(s)
- split_at.append(f)
-
- # turn the splitting points into pairs
- pairs = []
- for i in range(0, len(split_at)):
- if split_at[i] == -1:
- break
- if i + 1 >= len(split_at):
- end = len(nm)
- elif split_at[i+1] == -1:
- end = len(nm)
- else:
- end = split_at[i+1]
- pair = (split_at[i], end)
- pairs.append(pair)
-
- frags = []
- for s, f in pairs:
- frags.append(nm[s:f])
-
- for i in range(len(frags)):
- if frags[i].startswith("http://"):
- frags[i] = _create_url(frags[i])
-
- message = "".join(frags)
- return message
-
-def years_different(lower, upper, dict):
- lyear = regex_map(lower, {"expression" : "([\\d]{4})-.*"})
- uyear = regex_map(upper, {"expression" : "([\\d]{4})-.*"})
- return lyear != uyear
-
-def date_range_count(args, result, dict):
- if not args['q'].has_key(dict['bounding_field']):
- values = result.get(dict['results_field'])
- if values is not None:
- return str(len(values))
- else:
- return 0
- start, end = args['q'][dict['bounding_field']]
- sdate = datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ")
- edate = datetime.strptime(end, "%Y-%m-%dT%H:%M:%SZ")
- values = result.get(dict['results_field'])
- if values is None:
- return 0
- count = 0
- for vdate in values:
- # convert to offset aware datetime
- strdate = '{0.year}-{0.month:{1}}-{0.day:{1}}T{0.hour:{1}}:{0.minute:{1}}:{0.second:{1}}Z'.format(vdate, '02')
- vdate = datetime.strptime(strdate, "%Y-%m-%dT%H:%M:%SZ")
- if vdate >= sdate and vdate <= edate:
- count += 1
- return str(count)
-
-def array_count(args, result, dict):
- values = result.get(dict['count_field'])
- if values is None:
- return 0
- return len(values)
-
View
6 bibserver/static/bibsoup.css
@@ -6,6 +6,10 @@ body {
margin-top: 20px;
}
+form{
+ display:inline;
+}
+
/************************************
* Flash notifications / messages
***********************************/
@@ -76,7 +80,7 @@ span.upload_label{
.search_box {
padding: 20px;
padding-top: 20px;
- width: 600px;
+ width: 650px;
margin-left: auto;
margin-right: auto;
}
View
75 bibserver/static/bibsoup.js
@@ -70,77 +70,14 @@ jQuery(document).ready(function() {
}
jQuery(".facet_heading").bind('click',showfacets);
- // redesign results per page
- // build a list of acceptable rpp and the links to enable them
- var rpp = new Object();
- var rpp_label = jQuery('.rpp_label').html();
- var current_rpp = "";
- jQuery('.results_per_page').children().each(function() {
- if ( !jQuery(this).hasClass('rpp_label') ) {
- if ( jQuery(this).hasClass('current_rpp') ) {
- rpp[ jQuery(this).html() ] = "";
- current_rpp = jQuery(this).html();
- } else {
- rpp[ jQuery(this).children('a').html() ] = jQuery(this).children('a').attr('href');
- }
- }
- });
- // replace the rpp div with a selector
- var new_rpp = rpp_label + ' <select id="rpp_selector">';
- for (var item in rpp) {
- new_rpp += '<option value="' + rpp[item] + '"';
- if ( item == current_rpp ) {
- new_rpp += ' "selected"';
- }
- new_rpp += '>' + item + '</option>';
- }
- new_rpp += '</select> per page.';
- jQuery('.results_per_page').html(new_rpp);
- // attach functionality to trigger rpp selections
+ // attach functionality to trigger rpp and page selections
+ /*jQuery('#paging_trigger').hide();
var rpp_select = function(event) {
- event.preventDefault();
- if ( jQuery(this).val() != "" ) {
- window.location = jQuery(this).val();
- }
- }
- jQuery('#rpp_selector').bind('change',rpp_select);
-
- // redesign paging
- // build a list of paging options and links to enable them
- var paging = new Object();
- var paging_label = jQuery('.paging_label').html();
- var current_page = "";
- jQuery('.paging').children().each(function() {
- if ( !jQuery(this).hasClass('paging_label') ) {
- if ( jQuery(this).hasClass('current_page') ) {
- paging[ jQuery(this).html() ] = "";
- current_page = jQuery(this).html();
- } else {
- paging[ jQuery(this).children('a').html() ] = jQuery(this).children('a').attr('href');
- }
- }
- });
- // replace the paging div with a selector
- var new_paging = paging_label + " " + '<select id="paging_selector">';
- for (var item in paging) {
- new_paging += '<option value="' + paging[item] + '"';
- if ( item == current_page ) {
- new_paging += "selected";
- }
- new_paging += '>' + item + '</option>';
- }
- new_paging += '</select> of ' + jQuery('.results_total').html().replace(" results",". &nbsp;&nbsp;&nbsp;&nbsp;");
- jQuery('.results_total').remove();
- jQuery('.paging').html(new_paging);
- // attach functionality to trigger paging selections
- var page_select = function(event) {
- event.preventDefault();
- if ( jQuery(this).val() != "" ) {
- window.location = jQuery(this).val();
- }
+ $(this).closest("form").submit();
}
- jQuery('#paging_selector').bind('change',page_select);
-
+ jQuery('#rpp_select').bind('change',rpp_select);
+ jQuery('#page_select').bind('change',rpp_select);*/
+
// redesign facet headers if they have no further options
jQuery('.facet').each(function() {
if ( jQuery(this).children().last().children().size() < 1 ) {
View
122 bibserver/urlmanager.py
@@ -1,122 +0,0 @@
-from copy import deepcopy
-import json, urllib2
-import bibserver.config
-
-class UrlManager(object):
- def __init__(self, config, args, implicit_facets):
- self.config = config
- self.args = args if args is not None else self.config.get_default_args()
- self.implicit_facets = implicit_facets
- self.base_args = self.strip_implicit_facets()
-
- def strip_implicit_facets(self):
- myargs = deepcopy(self.args)
- if self.implicit_facets is None:
- return myargs
- if not self.args.has_key('q'):
- return myargs
-
- for field in self.implicit_facets.keys():
- if myargs['q'].has_key(field):
- del myargs['q'][field]
- return myargs
-
- def get_search_form_action(self):
- return self.config.base_url
-
- def get_form_field_args(self):
- myargs = deepcopy(self.base_args)
- if myargs.has_key("search"):
- del myargs['search']
- j = json.dumps(myargs)
- return urllib2.quote(j)
-
- def get_add_url(self, field, value, upper=None):
- myargs = deepcopy(self.base_args)
- if myargs["q"].has_key(field):
- if upper is None and value not in myargs["q"][field]:
- myargs["q"][field].append(value)
- elif upper is not None:
- myargs["q"][field] = [value, upper]
- else:
- if upper is None:
- myargs["q"][field] = [value]
- else:
- myargs["q"][field] = [value, upper]
- if myargs.has_key('start'):
- del myargs['start']
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_add_date_url(self, field, value, upper=None):
- myargs = deepcopy(self.base_args)
- value = '{0.year}-{0.month:{1}}-{0.day:{1}}T{0.hour:{1}}:{0.minute:{1}}:{0.second:{1}}Z'.format(value, '02')
- if upper is not None and upper != -1:
- upper = '{0.year}-{0.month:{1}}-{0.day:{1}}T{0.hour:{1}}:{0.minute:{1}}:{0.second:{1}}Z'.format(upper, '02')
- if myargs["q"].has_key(field):
- if upper is None and value not in myargs["q"][field]:
- myargs["q"][field].append(value)
- elif upper is not None:
- myargs["q"][field] = [value, upper]
- else:
- if upper is None:
- myargs["q"][field] = [value]
- else:
- myargs["q"][field] = [value, upper]
- if myargs.has_key('start'):
- del myargs['start']
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_delete_url(self, field, value=None):
- myargs = deepcopy(self.base_args)
- if value is not None:
- myargs['q'][field].remove(value)
- if len(myargs['q'][field]) == 0:
- del myargs['q'][field]
- else:
- del myargs['q'][field]
- if myargs.has_key('start'):
- del myargs['start']
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_position_url(self, position):
- myargs = deepcopy(self.base_args)
- myargs["start"] = position
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_rpp_url(self, rpp):
- myargs = deepcopy(self.base_args)
- myargs["rows"] = int(rpp)
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_sort_url(self, field, direction):
- myargs = deepcopy(self.base_args)
- if myargs.has_key("sort"):
- isnew = True
- for i in range(len(myargs['sort'])):
- f, d = myargs['sort'][i]
- if f == field:
- myargs['sort'][i] = [field, direction]
- isnew = False
- break
- if isnew:
- myargs['sort'].append([field, direction])
- else:
- myargs["sort"] = [[field, direction]]
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
- def get_unsort_url(self, field):
- myargs = deepcopy(self.base_args)
- for i in range(len(myargs['sort'])):
- f, direction = myargs['sort'][i]
- if f == field:
- del myargs['sort'][i]
- break
- j = json.dumps(myargs)
- return self.config.base_url + "?a=" + urllib2.quote(j)
-
View
95 bibserver/web.py
@@ -1,29 +1,29 @@
import os
from datetime import datetime
import urllib2
+from copy import deepcopy
from flask import Flask, jsonify, json, request, redirect, abort
from flask.views import View, MethodView
from flaskext.mako import init_mako, render_template
-import bibserver.config
import bibserver.dao
-import bibserver.setconfig
-import bibserver.resultmanager
-import bibserver.urlmanager
+from bibserver.config import config
+import bibserver.iomanager
import bibserver.importer
app = Flask(__name__)
app.config['MAKO_DIR'] = 'templates'
init_mako(app)
+
@app.route('/')
def home():
# get list of available collections
result = bibserver.dao.Record.query(q="*:*",facet_fields=["collection"],size=1)
colls = result.get("facets").get("collection").get("terms")
upload = False
- return render_template('home/index.html', colls=colls, upload=bibserver.config.config["allow_upload"])
+ return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )
@app.route('/content/<path:path>')
@@ -116,76 +116,71 @@ def package(self):
return pkg
# enable upload unless not allowed in config
-if bibserver.config.config["allow_upload"] == "YES":
+if config["allow_upload"] == "YES":
app.add_url_rule('/upload', view_func=UploadView.as_view('upload'))
@app.route('/search')
@app.route('/search<path:path>')
@app.route('/<path:path>')
def search(path=''):
- c = {}
- config = bibserver.setconfig.Configuration(bibserver.config.config)
- query = request.args.get('q', '')
-
+ c = {}
+
+ # get query params
+ params = request.values.items()
+ print params
+
# get the args (if available) out of the request
a = request.values.get("a")
if a is not None:
a = urllib2.unquote(a)
+ c['a'] = urllib2.quote(a)
args = json.loads(a)
else:
- args = config.get_default_args()
+ args = {
+ "terms" : {},
+ "facet_fields" : config["facet_fields"],
+ "size" : config["default_results_per_page"],
+ "start" : 0
+ }
+
+ c['config'] = bibserver.config.Config(config)
+
+ if 'from' in request.values:
+ args['start'] = request.values.get('from')
+ c['config'].start = request.values.get('from')
+ else:
+ c['config'].start = 0
- args['search'] = query
- c['q'] = query
- c['config'] = config
+ if 'size' in request.values:
+ args['size'] = request.values.get('size')
+ c['config'].default_results_per_page = int(request.values.get('size'))
- # get implicit facets
- implicit_facets = {}
+ if 'q' in request.values:
+ args['q'] = request.values.get('q')
+ if 'q' in args:
+ c['q'] = args['q']
+
+ # get implicit facet
+ c['implicit_facet'] = {}
if path is not None and not path.startswith("/search"):
path = path.strip()
if path.endswith("/"):
path = path[:-1]
bits = path.split('/')
- if len(bits) % 2 == 0:
- config.base_url = config.base_url.replace(config.strip_for_implicit_paths,"") + path
- if not args.has_key('q'):
- args['q'] = {}
- for i in range(0, len(bits), 2):
- field = bits[i]
- value = bits[i+1]
- if args['q'].has_key(field):
- args['q'][field].append(value)
- else:
- args['q'][field] = [value]
- if implicit_facets.has_key(field):
- implicit_facets[field].append(value)
- else:
- implicit_facets[field] = [value]
+ if len(bits) == 2:
+ c['config'].base_url = '/' + path
+ if not args.has_key('terms'):
+ args['terms'] = {}
+ args['terms'][bits[0]] = [bits[1]]
+ c['implicit_facet'][bits[0]] = bits[1]
# get results and render
- c['url_manager'] = bibserver.urlmanager.UrlManager(config, args,
- implicit_facets)
- c['implicit_facets'] = implicit_facets
- querydict = convert_query_dict_for_es(args)
- results = bibserver.dao.Record.query(**querydict)
- c['results'] = bibserver.resultmanager.ResultManager(results, config, args)
+ results = bibserver.dao.Record.query(**args)
+ c['results'] = bibserver.iomanager.IOManager(results, c['config'], args)
return render_template('search/index.html', c=c)
-def convert_query_dict_for_es(querydict):
- outdict = {}
- outdict['q'] = querydict['search']
- outdict['facet_fields'] = querydict.get('facet_field', None)
- outdict['terms'] = {}
- for term, values in querydict.get('q', {}).items():
- # only use first value (TODO: can one ever have multi-values?)
- outdict['terms'][term] = values[0]
- outdict['size'] = querydict.get('rows', 10)
- outdict['start'] = querydict.get('start', 0)
- return outdict
-
-
if __name__ == "__main__":
bibserver.dao.init_db()
app.run(host='0.0.0.0', debug=True)
View
30 config.json
@@ -2,12 +2,15 @@
# name of service
"service_name" : "BibSoup",
- # noupload - set to "NO" if you dont want frontend upload to your bibserver install
+ # allow_upload - set to "NO" if you dont want frontend upload to your bibserver install
"allow_upload" : "YES",
# elasticsearch settings
"ELASTIC_SEARCH_HOST" : "127.0.0.1:9200",
"ELASTIC_SEARCH_DB" : "bibserver",
+
+ # identify how non-analyzed fields for faceting are differenetiated in the mappings
+ "facet_field" : ".exact",
"mappings" : {
"record" : {
@@ -35,35 +38,16 @@
# base URL of search service
"base_url" : "/search",
- "strip_for_implicit_paths" : "search",
# default results and paging options (integer - array of integers)
"default_results_per_page" : 10,
"results_per_page_options" : [10, 20, 50, 100],
- # The fields which are plain text/value facets, and their display names
- # (object with keys representing facet names, and values as objects containing
- # a single key/value pair for the display name)
- # "facet_fields" : { "<facet_name>" : { "display" : "<Display Name>" } ... }
- #
- "facet_fields" : {
- "collection.exact" : { "display" : "collection" },
- "type.exact" : { "display" : "type"},
- "journal.exact" : { "display" : "journal"},
- "author.exact" : {"display" : "author"},
- "year.exact" : {"display" : "year"}
- },
- "display_facet_order" : ["collection.exact","type.exact","journal.exact","author.exact","year.exact"],
+ # The fields for faceting on
+ "facet_fields" : ["collection","type","journal","author","year"],
# fields to display as headers in search results
- "display_fields" : {
- "title" : "title",
- "journal" : "journal",
- "author" : "author",
- "doi" : "doi",
- "url" : "url"
- },
- "display_field_order" : ["title","journal","author","doi","url"],
+ "display_fields" : ["title","journal","author","doi","url"],
# functions to perform on particular result fields on display
"display_value_functions" : {
View
13 templates/search/facet-extra.mako
@@ -1,5 +1,5 @@
-% for facet in c['config'].display_facet_order:
- % if c['config'].get_facet_display(facet) not in c['implicit_facets']:
+% for facet in c['config'].facet_fields:
+ % if c['config'].get_facet_display(facet) not in c['implicit_facet']:
<div class="facet">
% if c['results'].has_values(facet):
<div class="facet_heading">
@@ -10,8 +10,9 @@
% for value, count in c['results'].get_ordered_facets(facet):
% if c['results'].in_args(facet, value):
<em>
- ${c['config'].get_value_display(facet, value)} (${count})
- </em>&nbsp;<a class="delete_url" href="${c['url_manager'].get_delete_url(facet, value)}">x</a>
+ ${value} (${count})
+ </em>
+ &nbsp;<a class="delete_url" href="${c['results'].get_delete_url(facet, value)}">x</a>
<br/>
% endif
% endfor
@@ -22,8 +23,8 @@
% for value, count in c['results'].get_ordered_facets(facet):
% if not c['results'].in_args(facet, value):
- <a href="${c['url_manager'].get_add_url(facet, value)}">
- ${c['config'].get_value_display(facet, value)} (${count})
+ <a href="${c['results'].get_add_url(facet, value)}">
+ ${value} (${count})
</a><br/>
% endif
% endfor
View
41 templates/search/index.html
@@ -2,16 +2,23 @@
<div id="bibserver_search">
<div class="search_box">
- <form method="get" action="${c['url_manager'].get_search_form_action()}">
- <input type="text" name="q" value="${c['q']}"/>
- <input type="hidden" name="a" value="${c['url_manager'].get_form_field_args()}"/>
- <input type="submit" name="submit_search" value="Search"/>
+ <form method="get" action="${c['config'].base_url}">
+ % if 'q' in c:
+ <input type="text" name="q" value="${c['q']}" />
+ % else:
+ <input type="text" name="q" />
+ % endif
+ % if 'a' in c:
+ <input type="hidden" name="a" value="${c['a']}" />
+ % endif
+ <input type="submit" name="submit_search" value="Search" />
+ <a class="btn" href="${c['config'].base_url}">clear</a>
</form>
</div>
</div>
-% for facet in c['implicit_facets']:
- <h3>${facet} : ${c['implicit_facets'][facet][0]}</h3>
+% for facet in c['implicit_facet']:
+ <h3>${facet} : <a href="/${facet}/${c['implicit_facet'][facet]}">${c['implicit_facet'][facet]}</a></h3>
% endfor
<div id="navigation">
@@ -20,32 +27,16 @@
<div id="panel">
- % if c['results'].set_size() != 0:
- <span class="results_total">${c['results'].numFound()} results</span>
+ % if c['results'].numFound() != 0:
<%include file="paging.mako"/>
% endif
- % if c['results'].set_size() == 0:
+ % if c['results'].numFound() == 0:
No results
% else:
- <!-- sorting and result sizes -->
- <div class="results_per_page">
-
- <div class="rpp_label">Show</div>
- % for rpp in c['config'].results_per_page_options:
- % if rpp == c['results'].page_size():
- <div class="current_rpp">${rpp}</div>
- % else:
- <div class="potential_rpp"><a href="${c['url_manager'].get_rpp_url(rpp)}">${rpp}</a></div>
- % endif
- % endfor
- </div>
-
- <div class="spacer"></div>
-
- <!-- finally, the result set itself -->
+ <!-- the result set itself -->
<%include file="list-view.mako"/>
% endif
View
6 templates/search/list-view.mako
@@ -1,19 +1,19 @@
-<% if c['results'].set_size() == 0:
+<% if c['results'].numFound() == 0:
return
%>
<div class="list_view">
% for i in range(len(c['results'].set())):
<div class="list_result_${'odd' if i % 2 == 0 else 'even'}">
- % for field in c['config'].display_field_order:
+ % for field in c['config'].display_fields:
% if c['results'].get_str(c['results'].set()[i], field) != "":
<div class="list_result_field">${c['results'].get_str(c['results'].set()[i], field)}</div>
% endif
% endfor
<div class="list_result_hidden">
% for record in c['results'].set()[i]:
- % if record not in c['config'].display_field_order and record not in ["_rev","score"]:
+ % if record not in c['config'].display_fields:
<div class="list_result_field">${record} - ${c['results'].get_str(c['results'].set()[i], record)}</div>
% endif
% endfor
View
71 templates/search/paging.mako
@@ -1,27 +1,52 @@
<div class="paging">
- <div class="paging_label">Results</div>
-
- <!-- Paging -->
+ % if int(c['config'].start) > int(c['results'].page_size()):
+ <a class="btn small" alt="previous" title="previous" href="">prev</a>
+ % endif
+
+ <form action="" method="get">
+ <select class="small" name="from" id="page_select">
+
+% for i in range(0, ( c['results'].numFound() / c['config'].default_results_per_page ) + 1):
+ % if (i * c['config'].default_results_per_page) == int(c['config'].start):
+ <option value="${i * c['config'].default_results_per_page}" selected>
+ % else:
+ <option value="${i * c['config'].default_results_per_page}">
+ % endif
+ ${i * c['config'].default_results_per_page + 1} -
+ % if i * c['config'].default_results_per_page + 1 + c['config'].default_results_per_page < c['results'].numFound():
+ ${i * c['config'].default_results_per_page + c['config'].default_results_per_page}
+ % else:
+ ${c['results'].numFound()}
+ % endif
+ </option>
+% endfor
-% if not c['results'].is_start():
- <div class="potential_page"><a href="${c['url_manager'].get_position_url(0)}">1 - ${c['results'].first_page_end()}</a></div>
- <div class="paging_label">...</div>
-% endif
-
-% for start, finish in c['results'].get_previous(10):
- <div class="potential_page"><a href="${c['url_manager'].get_position_url(start)}">${start + 1} - ${finish}</a></div>
-% endfor
- <div class="current_page">${c['results'].start_offset(1)} - ${c['results'].finish()}</div>
-
-% for start, finish in c['results'].get_next(10):
- <div class="potential_page"><a href="${c['url_manager'].get_position_url(start)}">${start + 1} - ${finish}</a></div>
-% endfor
-
-% if not c['results'].is_end():
- <div class="paging_label">...</div>
- <div class="potential_page"><a href="${c['url_manager'].get_position_url(c['results'].last_page_start())}">
- ${c['results'].last_page_start() + 1} -
- ${c['results'].numFound()}</a></div>
-% endif
+ </select>
+
+ <span class="results_total">of ${c['results'].numFound()} results. Show </span>
+
+ <select class="small" name="size" id="rpp_select">
+
+ % for rpp in c['config'].results_per_page_options:
+ % if rpp == int(c['results'].page_size()):
+ <option selected>${rpp}</div>
+ % else:
+ <option value="${rpp}">${rpp}</option>
+ % endif
+ % endfor
+
+ </select>
+
+ <input type="submit" name="submit" value="update" id="paging_trigger" />
+ </form>
+
+
+ % if ( int(c['config'].start) + int(c['results'].page_size()) ) < c['results'].numFound():
+ <a class="btn small" title="next" alt="next" href="">next</a>
+ % endif
+
</div>
+
+<div class="spacer"></div>
+
View
4 test/base.py
@@ -1,16 +1,16 @@
import os
import json
-from bibserver.config import config
from bibserver import dao
+from bibserver.config import config
TESTDB = 'bibserver-test'
here = os.path.dirname(__file__)
fixtures_path = os.path.join(here, 'fixtures.json')
fixtures = json.load(open(fixtures_path))
-config['ELASTIC_SEARCH_DB'] = TESTDB
+config["ELASTIC_SEARCH_DB"] = TESTDB
dao.init_db()
__all__ = ['config', 'fixtures', 'dao', 'TESTDB', 'json']
View
2 test/test_dao.py
@@ -55,6 +55,6 @@ def test_query_facet(self):
assert facetterms[0]['count'] == 1
def test_query_term(self):
- out = dao.Record.query(terms={'type': 'book'})
+ out = dao.Record.query(terms={'type': ['book']})
assert_equal(out['hits']['total'], 1)
View
44 test/test_resultmanager.py
@@ -1,44 +0,0 @@
-from nose.tools import assert_equal
-
-from test.base import *
-
-import bibserver.resultmanager
-import bibserver.setconfig
-import bibserver.dao
-
-class TestResultManager:
- @classmethod
- def setup_class(cls):
- recdict = fixtures['records'][0]
- cls.record = dao.Record.upsert(recdict)
-
- @classmethod
- def teardown_class(cls):
- conn, db = dao.get_conn()
- conn.delete_index(TESTDB)
-
- def test_01(self):
- config = bibserver.setconfig.Configuration(bibserver.config.config)
- facet_fields = config.get_default_args()['facet_field']
- results = bibserver.dao.Record.query('tolstoy',
- facet_fields=facet_fields)
- args = None
- manager = bibserver.resultmanager.ResultManager(results, config, args)
-
- assert_equal(manager.numFound(), 1)
-
- prev = manager.get_previous(0)
- assert_equal(prev, [])
-
- assert_equal(manager.page_size(), 10)
-
- # commented out this test because ordering comes from ES anyway
- # this will probably not be needed after refactoring resultmanager
- #out = manager.get_ordered_facets('collection')
- #assert_equal(out, [('great',1), ('novels',1)])
-
- recorddicts = manager.set()
- print recorddicts
- out = manager.get_str(recorddicts[0], 'title')
- assert recorddicts[0]['title'] in out, out
-
View
2 test/test_web.py
@@ -58,6 +58,7 @@ def test_search(self):
assert res.status == '200 OK', res.status
assert 'Tolstoy' in res.data, res.data
+ '''query converter does not exist anymore
def test_queryobject(self):
indata = {
'search': u'pitman',
@@ -71,5 +72,6 @@ def test_queryobject(self):
outdata = web.convert_query_dict_for_es(indata)
assert_equal(outdata['size'], 10)
assert_equal(outdata['terms'], {'collection': 'pitman2'})
+ '''

0 comments on commit be8c74e

Please sign in to comment.