Skip to content
Browse files

version 2 of the eff tool

  • Loading branch information...
1 parent d4b028c commit 1e60b65efaf6f6ec8b233d341b01a216c5e4e2b1 @symroe symroe committed Oct 6, 2011
View
2 requirements.txt
@@ -19,3 +19,5 @@ typogrify
sorl-thumbnail==11.01
PIL
django-sentry
+xapian-haystack
+django-haystack
View
14 web/data/forms.py
@@ -27,9 +27,13 @@ class EffSearchForm(forms.Form):
from django.db.models import Count
- YEAR_CHOICES = EffData.objects.all().order_by('yearPaid').values('yearPaid').distinct()
- YEAR_CHOICES = [(v['yearPaid'],v['yearPaid']) for v in YEAR_CHOICES if v['yearPaid']]
- YEAR_CHOICES.insert(0, ('', 'All Years'))
+ # YEAR_CHOICES = EffData.objects.all().order_by('yearPaid').values('yearPaid').distinct()
+ # YEAR_CHOICES = [(v['yearPaid'],v['yearPaid']) for v in YEAR_CHOICES if v['yearPaid']]
+ # YEAR_CHOICES.insert(0, (0, 'All Years'))
- query = forms.CharField(required=True, label=_('Search terms'))
- yeara = forms.ChoiceField(choices=YEAR_CHOICES, required=False, label=_('Year Allocated'))
+ query = forms.CharField(required=True, label=_('Search terms'),
+ min_length=2,
+ error_messages={
+ 'min_length': _('Query must be longer than 2 characters')
+ })
+ # yeara = forms.ChoiceField(choices=YEAR_CHOICES, required=False, label=_('Year Allocated'))
View
29 web/data/index.py
@@ -20,32 +20,3 @@ class SchemeIndexer(Indexer):
class PortIndexer(Indexer):
fields = ['name','geo1', 'geo2', 'country']
space.add_index(Port, PortIndexer, attach_as='indexer')
-
-
-
-class EffIndexer(Indexer):
-
- fields = [
- 'country',
- 'name',
- 'measureText',
- 'amountEuAllocatedEuro',
- 'amountEuPaymentEuro',
- 'amountTotalAllocatedEuro',
- 'amountTotalPaymentEuro',
- 'yearAllocated',
- 'yearPaid',
- ]
-
- tags = [
- ('country', 'country'),
- ('name', 'name', 10),
- ('yeara', 'yearAllocated'),
- ('yearp', 'yearPaid'),
- ]
-
-space.add_index(EffData, EffIndexer, attach_as='indexer')
-
-
-
-
View
8 web/data/management/commands/import_eff.py
@@ -16,11 +16,17 @@ class SKV(csv.excel):
class Command(BaseCommand):
def handle(self, *args, **kwargs):
- f = csv.DictReader(open('../data/FishWebsite20110822.csv', 'r'), dialect='SKV')
+ f = csv.DictReader(open('../data/website20110922.txt', 'r'), dialect='SKV')
EffData.objects.all().delete()
for line in f:
if not line['yearAllocated']:
del line['yearAllocated']
+ else:
+ try:
+ int(line.get('yearAllocated'))
+ except ValueError:
+ del line['yearAllocated']
+
if not line['yearPaid']:
del line['yearPaid']
else:
View
18 web/data/models.py
@@ -190,7 +190,12 @@ class EffData(models.Model):
area2 = models.CharField(blank=True, null=True, max_length=255)
fund = models.CharField(blank=True, null=True, max_length=100)
name = models.CharField(blank=True, null=True, max_length=255)
+
+ axisText = models.TextField(blank=True, null=True)
+ actionText = models.TextField(blank=True, null=True)
+ projectDescription = models.TextField(blank=True, null=True)
measureText = models.TextField(blank=True, null=True)
+
amountEuAllocatedEuro = models.FloatField()
amountEuPaymentEuro = models.FloatField()
amountTotalAllocatedEuro = models.FloatField()
@@ -200,13 +205,24 @@ class EffData(models.Model):
def format_location(self):
fields = [
- unicode(country_codes(code=self.country)['name']) or "",
+ "<a href='?query=country:\"%s\"'>%s</a>" % (self.country, unicode(country_codes(code=self.country)['name']) or ""),
self.area1,
self.area2,
]
fields = [f for f in fields if f]
return "<br />".join(fields)
+ def format_description(self):
+ fields = [
+ (self.axisText, "axisText"),
+ (self.measureText, "measureText"),
+ (self.actionText, "actionText"),
+ (self.projectDescription, "projectDescription"),
+ ]
+ fields = [f for f in fields if f[0]]
+ f_list = ["<li>- <a href='?query=%s:\"%s\"'>%s</a>.</li>" % (f[1], f[0], f[0]) for f in fields]
+ return "".join([f for f in f_list])
+
View
145 web/data/templates/eff_search/search.html
@@ -16,17 +16,89 @@
</form>
</div>
{% if number_of_results %}
-
<div class="section">
<h3>{% trans "Results" %}</h3>
- <ul class="results_summary">
- <li><strong>Number of beneficiaries:</strong> {{ number_of_results|floatformat|intcomma }}</li>
- <li><strong>EU funds allocated:</strong> {{ totals.amountEuAllocatedEuro|currencyfmt:"EUR" }}</li>
- <li><strong>EU funds paid:</strong> {{ totals.amountEuPaymentEuro|currencyfmt:"EUR" }}</li>
- <li><strong>Total allocated:</strong> {{ totals.amountTotalAllocatedEuro|currencyfmt:"EUR" }}</li>
- <li><strong>Total paid:</strong> {{ totals.amountTotalPaymentEuro|currencyfmt:"EUR" }}</li>
+ <ul class="results_summary col">
+ <li>
+ <strong>Number of beneficiaries:</strong>
+ {% if number_of_results %}
+ {{ number_of_results|floatformat|intcomma }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+
+ </li>
+
+ <li>
+ <strong>EU funds allocated:</strong>
+ {% if totals.amountEuAllocatedEuro %}
+ {{ totals.amountEuAllocatedEuro|currencyfmt:"EUR" }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+
+ </li>
+ <li>
+ <strong>EU funds paid:</strong>
+ {% if totals.amountEuPaymentEuro %}
+ {{ totals.amountEuPaymentEuro|currencyfmt:"EUR" }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+ </li>
+ <li>
+ <strong>Total allocated:</strong>
+ {% if totals.amountTotalAllocatedEuro %}
+ {{ totals.amountTotalAllocatedEuro|currencyfmt:"EUR" }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+ </li>
+ <li>
+ <strong>Total paid:</strong>
+ {% if totals.amountTotalPaymentEuro %}
+ {{ totals.amountTotalPaymentEuro|currencyfmt:"EUR" }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+ </li>
</ul>
+
+ <div class="facets">
+ {% if "country" in filter_types %}
+ <h4>{% trans "Filter by country" %}:</h4>
+ {% trans "Results in: " %}{% code_to_name_dumb facets.fields.country_exact.0.0 %}<br />
+ <a href="{% parse_qs request.GET "country" "" %}">{% trans "Remove filter" %}</a>
+ {% endif %}
+
+ {% if facets.fields.country_exact.1 %}
+ <h4>{% trans "Filter by country" %}:</h4>
+ <ul>
+ {% for country in facets.fields.country_exact %}
+ <li><a href="{% parse_qs request.GET "country" country.0 %}">{% code_to_name_dumb country.0 %} ({{ country.1 }})</a></li>
+ {% endfor %}
+ </ul>
+ {% endif %}
+ </div>
+ <div class="facets">
+ {% if "yeara" in filter_types %}
+ <h4>{% trans "Filter by Year allocated" %}:</h4>
+ {% trans "Results in: " %}{{ facets.fields.yeara.0.0 }}<br />
+ <a href="{% parse_qs request.GET "yeara" "" %}">{% trans "Remove filter" %}</a>
+ {% endif %}
+ {% if facets.fields.yeara.1 %}
+ <h4>{% trans "Filter by Year allocated" %}:</h4>
+ <ul>
+ {% for year in facets.fields.yeara %}
+ {% if year.0 %}
+ <li><a href="{% parse_qs request.GET "yeara" year.0 %}">{% if year.0 != '0' %}{{ year.0 }}{% else %}{% trans "Unknown" %} {% endif %} ({{ year.1 }})</a></li>
+ {% endif %}
+ {% endfor %}
+ </ul>
+ {% endif %}
+ </div>
+
<div class="pagination">
<span class="step-links">
{% if results.has_previous %}
@@ -46,47 +118,63 @@
<br />
<table>
<thead>
- <th>{% trans "Name" %}</th>
- <th>{% trans "Location" %}</th>
- <th>{% trans "Description" %}</th>
- <th>{% trans "EU funds allocated" %}</th>
- <th>{% trans "EU funds paid" %}</th>
- <th>{% trans "Total allocated" %}</th>
- <th>{% trans "Total paid" %}</th>
+ <th>{% trans "Name" %} <a href="#help_name" class="help_link">?</a></th>
+ <th>{% trans "Location" %} <a href="#help_location" class="help_link">?</a></th>
+ <th>{% trans "Description" %} <a href="#help_description" class="help_link">?</a></th>
+ <th>{% trans "EU funds allocated" %} <a href="#help_funds_allocated" class="help_link">?</a></th>
+ <th>{% trans "EU funds paid" %} <a href="#help_funds_paid" class="help_link">?</a></th>
+ <th>{% trans "Total allocated" %} <a href="#help_total_allocated" class="help_link">?</a></th>
+ <th>{% trans "Total paid" %} <a href="#help_total_paid" class="help_link">?</a></th>
+ <th>{% trans "Year allocated" %} <a href="#help_year_paid" class="help_link">?</a></th>
</thead>
{% for result in results.object_list %}
- <tr>
+ <tr valign="top">
+ <td>
+ {% if result.object.name %}
+ {% if "name" in filter_types %}
+ {{ result.object.name }}
+ {% else %}
+ <a href='?query=name:"{{result.object.name}}"'>{{ result.object.name }}</a>
+ {% endif %}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
+ </td>
<td>
- {{ result.instance.name }}
+ {{ result.object.format_location|safe }}
</td>
+ <td><ul>{{ result.object.format_description|safe }}</ul></td>
<td>
- {{ result.instance.format_location|safe }}
+ {% if result.object.amountEuAllocatedEuro %}
+ {{ result.object.amountEuAllocatedEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
+ {% else %}
+ {% trans "Unknown" %}
+ {% endif %}
</td>
- <td>{{ result.instance.measureText }}</td>
<td>
- {% if result.instance.amountEuAllocatedEuro %}
- {{ result.instance.amountEuAllocatedEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
+ {% if result.object.amountEuPaymentEuro %}
+ {{ result.object.amountEuPaymentEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
{% else %}
{% trans "Unknown" %}
{% endif %}
</td>
<td>
- {% if result.instance.amountEuPaymentEuro %}
- {{ result.instance.amountEuPaymentEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
+ {% if result.object.amountTotalAllocatedEuro %}
+ {{ result.object.amountTotalAllocatedEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
{% else %}
{% trans "Unknown" %}
{% endif %}
</td>
<td>
- {% if result.instance.amountTotalAllocatedEuro %}
- {{ result.instance.amountTotalAllocatedEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
+ {% if result.object.amountTotalPaymentEuro %}
+ {{ result.object.amountTotalPaymentEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
{% else %}
{% trans "Unknown" %}
{% endif %}
</td>
<td>
- {% if result.instance.amountTotalPaymentEuro %}
- {{ result.instance.amountTotalPaymentEuro|currencyfmt:"EUR"|default_if_none:"Unknown" }}
+ {% if result.object.yearAllocated %}
+ {{ result.object.yearAllocated|default_if_none:"Unknown" }}
{% else %}
{% trans "Unknown" %}
{% endif %}
@@ -114,4 +202,9 @@
</div>
{% endif %}
+{% endblock %}
+
+{% block right %}
+ <h2>{{ side_bar_help.title }}</h2>
+ {{ side_bar_help.content|safe }}
{% endblock %}
View
2 web/data/templatetags/code_to_name.py
@@ -23,5 +23,5 @@ def code_to_name(parser, token):
@register.simple_tag
def code_to_name_dumb(code):
- return countryCodes.country_codes(code)['name']
+ return countryCodes.country_codes(code.upper())['name']
View
54 web/data/views.py
@@ -17,6 +17,7 @@
from django.utils.translation import get_language
from django.utils.translation import ugettext as _
from django.core.paginator import Paginator
+from multilingual.flatpages.models import MultilingualFlatPage
import models
from data.models import FishData, illegalFishing
@@ -30,6 +31,9 @@
from misc import countryCodes
+from haystack.query import SearchQuerySet
+from xapian_backend import SearchBackend, SearchQuery
+
def home(request):
ip_country = RequestContext(request)['ip_country']
top_vessels = Recipient.vessels.order_by('-amount')[:5]
@@ -39,7 +43,6 @@ def home(request):
top_schemes = Scheme.objects.top_schemes(year=0)
latest_annotations = RecipientComment.objects.all().order_by('-date')[:5]
- print latest_annotations
return render_to_response(
'home.html',
@@ -543,21 +546,30 @@ def data_agreement_form(request):
def effsearch(request):
- page = totals = results = results_count = None
+ page = totals = results = facets = results_count = None
+ filter_types = []
if request.GET:
form = EffSearchForm(request.GET)
if form.is_valid():
+
q = form.cleaned_data['query']
+
if request.GET.get('yeara'):
- q = "%s yeara:%s" % (q, request.GET.get('yeara'))
- results = EffData.indexer.search(q).flags(
- xapian.QueryParser.FLAG_PHRASE\
- | xapian.QueryParser.FLAG_BOOLEAN\
- | xapian.QueryParser.FLAG_LOVEHATE\
- ).prefetch()
-
- results_count = results.count()
+ q = "%s AND yeara:%s" % (q, request.GET.get('yeara'))
+ filter_types.append('yeara')
+ if request.GET.get('country'):
+ q = "%s AND country_exact:%s" % (q, request.GET.get('country').lower())
+ filter_types.append('country')
+
+ backend = SearchBackend()
+ query = backend.parse_query(q)
+
+ results = backend.search(query, facets=['country_exact', 'yeara'])
+ # results = backend.search(query)
+
+ results_count = results['hits']
+ facets = results['facets']
amountEuAllocatedEuro = \
amountEuPaymentEuro = \
amountTotalAllocatedEuro = \
@@ -570,8 +582,8 @@ def effsearch(request):
for k,v in totals.items():
totals[k] = float(v)
else:
- for result in results:
- i = result.instance
+ for result in results['results']:
+ i = result.object
amountEuAllocatedEuro = amountEuAllocatedEuro + i.amountEuAllocatedEuro
amountEuPaymentEuro = amountEuPaymentEuro + i.amountEuPaymentEuro
amountTotalAllocatedEuro = amountTotalAllocatedEuro + i.amountTotalAllocatedEuro
@@ -584,21 +596,27 @@ def effsearch(request):
for k,v in totals.items():
r.hset(cache_key, k, v)
r.expire(cache_key, 60*60*24*7) # Expire in one week
-
- page = Paginator(results, 100).page(request.GET.get('page', 1) or 1)
- print dir(page)
+
+ page = Paginator(results['results'], 25).page(request.GET.get('page', 1) or 1)
+
else:
form = EffSearchForm()
-
-
+ try:
+ side_bar_help = MultilingualFlatPage.objects.get_or_create(url='/eff/help/', title='EFF Help')
+ except MultilingualFlatPage.DoesNotExist:
+ side_bar_help = MultilingualFlatPage()
+
return render_to_response(
'eff_search/search.html',
{
'form' : form,
+ 'filter_types' : filter_types,
'results' : page,
+ 'facets' : facets,
'totals' : totals,
- 'number_of_results' : results_count
+ 'number_of_results' : results_count,
+ 'side_bar_help' : side_bar_help,
},
context_instance=RequestContext(request)
)
View
16 web/features/search_indexes.py
@@ -1,16 +0,0 @@
-import datetime
-from haystack.indexes import *
-from queued_search.indexes import QueuedSearchIndex
-from haystack import site
-from features.models import Feature
-
-
-class FeatureIndex(QueuedSearchIndex):
- text = CharField(document=True, use_template=True)
- published = BooleanField(model_attr='published', default=False)
-
- def get_queryset(self):
- """Used when the entire index for model is updated."""
- return Feature.objects.filter(published=True)
-
-site.register(Feature, FeatureIndex)
View
8 web/global_settings.py
@@ -96,6 +96,8 @@
'sorl.thumbnail',
'sentry',
'sentry.client',
+ 'haystack',
+
]
TEMPLATE_CONTEXT_PROCESSORS = (
@@ -153,4 +155,8 @@
STATS_PATH = ROOT_PATH + '/data/stats'
DEFAULT_YEAR = 0
-THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.redis_kvstore.KVStore'
+THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.redis_kvstore.KVStore'
+
+HAYSTACK_SITECONF = 'web.search_sites'
+HAYSTACK_SEARCH_ENGINE = 'xapian'
+HAYSTACK_XAPIAN_PATH = 'xapian-haystack.db'
View
12 web/listmaker/search_indexes.py
@@ -1,12 +0,0 @@
-import datetime
-from haystack.indexes import *
-from queued_search.indexes import QueuedSearchIndex
-from haystack import site
-from models import List
-
-
-class ListIndex(QueuedSearchIndex):
- text = CharField(document=True, use_template=True)
- name = CharField(model_attr='name', weight=2)
-
-site.register(List, ListIndex)
View
4 web/media/css/main.css
@@ -89,13 +89,17 @@ p.hint {color:#808080;font-size:0.8em; line-height:1.5em; padding:0.4em; margin-
h2 .traffic_light { padding:1px;}
.col { float:left; width:49%; padding-bottom:2em; clear:none;}
+.facets {width:25%;float:left; overflow:auto}
+.pagination {clear:both; width:100%; display:block}
.results_summary li strong {
width:200px;
display:block;
float:left;
}
+.help_link {vertical-align:super; font-size:0.9em}
+
/* features */
div.features {border-bottom:solid 1px #ccc;background:url(../images/featurebar.png);
display:none;padding-top:1em;}
View
2 web/misc/templatetags/flatpage_menu.py
@@ -5,7 +5,7 @@
def flatpage_menu(language=settings.LANGUAGE_CODE):
# Create an unordered list of all flatpages
- pages = MultilingualFlatPage.objects.all()
+ pages = MultilingualFlatPage.objects.filter(url__startswith="/about/")
menu = '<ul>'
for page in pages:

0 comments on commit 1e60b65

Please sign in to comment.
Something went wrong with that request. Please try again.