Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

full solr functionality, many improvements

  • Loading branch information...
commit 67a0a8ac7511185919c4f3c0caac74ee35c15fba 1 parent 74d0410
Charles DeTar authored
View
74 afg/management/commands/import_wikileaks.py
@@ -7,50 +7,13 @@
from django.core.management.base import BaseCommand
from django.db import connection, transaction
-from afg.models import DiaryEntry, Phrase
-
-fields = ["report_key", # 0
- "date", # 1
- "type", # 2
- "category", # 3
- "tracking_number", # 4
- "title", # 5
- "summary", # 6
- "region", # 7
- "attack_on", # 8
- "complex_attack", # 9
- "reporting_unit", # 10
- "unit_name", # 11
- "type_of_unit", # 12
- "friendly_wia", # 13
- "friendly_kia", # 14
- "host_nation_wia", # 15
- "host_nation_kia", # 16
- "civilian_wia", # 17
- "civilian_kia", # 18
- "enemy_wia", # 19
- "enemy_kia", # 20
- "enemy_detained", # 21
- "mgrs", # 22
- "latitude", # 23
- "longitude", # 24
- "originator_group", # 25
- "updated_by_group", # 26
- "ccir", # 27
- "sigact", # 28
- "affiliation", # 29
- "dcolor", # 30
- "classification", # 31
-]
+from afg.models import DiaryEntry, Phrase, import_fields
def clean_summary(text):
# Fix ampersand mess
while text.find("&") != -1:
text = text.replace("&", "&")
text = re.sub('&(?!(#[a-z\d]+|\w+);)/gi', "&", text)
-
- # Linebreaks
- text = text.replace("\n", "<br />")
return text
class Command(BaseCommand):
@@ -66,18 +29,26 @@ def handle(self, *args, **kwargs):
"""
return
+ fields = [a[0] for a in import_fields]
+ thru = lambda f: f
+ conversions = []
+ for f in import_fields:
+ if len(f) > 1:
+ conversions.append(f[1])
+ else:
+ conversions.append(thru)
+
+
phrases = defaultdict(set)
with open(args[0]) as fh:
reader = csv.reader(fh)
for c, row in enumerate(reader):
- print c
- for i in range(13, 22):
- row[i] = int(row[i] or 0)
- kwargs = dict(zip(fields, row))
- kwargs['summary'] = clean_summary(kwargs['summary'])
- kwargs['latitude'] = float(kwargs['latitude']) if kwargs['latitude'] else None
- kwargs['longitude'] = float(kwargs['longitude']) if kwargs['longitude'] else None
+ if c % 1000 == 0:
+ print c
+ values = map(lambda t: conversions[t[0]](t[1]), enumerate(row))
+ kwargs = dict(zip(fields, values))
entry = DiaryEntry.objects.create(**kwargs)
+
# Get words for phrases
summary = re.sub(r'<[^>]*?>', '', kwargs['summary'])
summary = re.sub(r'&[^;\s]+;', ' ', summary)
@@ -91,9 +62,15 @@ def handle(self, *args, **kwargs):
n = len(phrases)
cursor = connection.cursor()
+ # Drop the join reference constraint for efficiency. We're confident
+ # that the 4 million rows we're about to add all satisfy the
+ # constraint, and it saves about 5 hours of computation time.
+ cursor.execute('''ALTER TABLE "afg_phrase_entries" DROP CONSTRAINT "phrase_id_refs_id_48aa97f2"''')
+ transaction.commit_unless_managed()
for c, (phrase, entry_ids) in enumerate(phrases.iteritems()):
- if len(entry_ids) > 1:
- print c, n
+ if len(entry_ids) > 1 and len(entry_ids) <= 10:
+ if c % 1000 == 0:
+ print c, n
cursor.execute("INSERT INTO afg_phrase (phrase, entry_count) VALUES (%s, %s) RETURNING id", (phrase, len(entry_ids)))
phrase_id = cursor.fetchone()[0]
cursor.execute("""
@@ -101,3 +78,6 @@ def handle(self, *args, **kwargs):
""" + ",".join("(%s, %s)" % (phrase_id, entry_id) for entry_id in entry_ids)
)
transaction.commit_unless_managed()
+ cursor.execute('''ALTER TABLE "afg_phrase_entries" ADD CONSTRAINT "phrase_id_refs_id_48aa97f2" FOREIGN KEY ("phrase_id") REFERENCES "afg_phrase" ("id") DEFERRABLE INITIALLY DEFERRED;''')
+ transaction.commit_unless_managed()
+
View
67 afg/models.py
@@ -1,5 +1,57 @@
+import re
+import datetime
+
from django.db import models
+def clean_summary(text):
+ # Fix ampersand mess
+ while text.find("&amp;") != -1:
+ text = text.replace("&amp;", "&")
+ text = re.sub('&(?!(#[a-z\d]+|\w+);)/gi', "&amp;", text)
+ return text
+
+def force_int(a):
+ return int(a or 0)
+
+def float_or_null(f):
+ if f:
+ return float(f)
+ return None
+
+import_fields = [
+ ("report_key",), # 0
+ ("date",), # 1
+ ("type",), # 2
+ ("category",), # 3
+ ("tracking_number",), # 4
+ ("title",), # 5
+ ("summary", clean_summary), # 6
+ ("region",), # 7
+ ("attack_on",), # 8
+ ("complex_attack",), # 9
+ ("reporting_unit",), # 10
+ ("unit_name",), # 11
+ ("type_of_unit",), # 12
+ ("friendly_wia", force_int), # 13
+ ("friendly_kia", force_int), # 14
+ ("host_nation_wia", force_int), # 15
+ ("host_nation_kia", force_int), # 16
+ ("civilian_wia", force_int), # 17
+ ("civilian_kia", force_int), # 18
+ ("enemy_wia", force_int), # 19
+ ("enemy_kia", force_int), # 20
+ ("enemy_detained", force_int), # 21
+ ("mgrs",), # 22
+ ("latitude", float_or_null), # 23
+ ("longitude", float_or_null), # 24
+ ("originator_group",), # 25
+ ("updated_by_group",), # 26
+ ("ccir",), # 27
+ ("sigact",), # 28
+ ("affiliation",), # 29
+ ("dcolor",), # 30
+ ("classification",), # 31
+]
# No DB indexes because we're kicking all that to SOLR.
class DiaryEntry(models.Model):
report_key = models.CharField(max_length=255, unique=True)
@@ -35,7 +87,6 @@ class DiaryEntry(models.Model):
dcolor = models.CharField(max_length=255)
classification = models.CharField(max_length=255)
- # denormalization for sorting
def total_casualties(self):
return self.friendly_wia + self.friendly_kia + self.host_nation_wia + self.host_nation_kia + self.civilian_wia + self.civilian_kia + self.enemy_wia + self.enemy_kia
@@ -52,20 +103,6 @@ class Meta:
ordering = ['date']
verbose_name_plural = 'Diary entries'
- def casualty_summary(self):
- parts = []
- for attr in ('civilian', 'host_nation', 'friendly', 'enemy'):
- k = getattr(self, attr + '_kia')
- w = getattr(self, attr + '_wia')
- if k or w:
- counts = []
- if k:
- counts.append("%i killed" % k)
- if w:
- counts.append("%i wounded" % w)
- parts.append("%s: %s" % (attr.title().replace("_", " "), ", ".join(counts)))
- return "; ".join(parts)
-
class Phrase(models.Model):
phrase = models.CharField(max_length=255, unique=True, db_index=True)
entries = models.ManyToManyField(DiaryEntry)
View
6 afg/search_indexes.py
@@ -39,6 +39,12 @@ class DiaryEntryIndex(indexes.SearchIndex):
classification = indexes.CharField(model_attr='classification', faceted=True)
total_casualties = indexes.IntegerField(model_attr='total_casualties', faceted=True)
+ search_facet_display = ('date', 'type_', 'region', 'attack_on', 'type_of_unit', 'affiliation', 'dcolor', 'classification', 'category', 'total_casualties', 'civilian_kia', 'civilian_wia', 'host_nation_kia', 'host_nation_wia', 'friendly_kia', 'friendly_wia', 'enemy_kia', 'enemy_wia', 'enemy_detained')
+ offer_to_sort_by = (('Date', 'date'), ('Casualties', 'total_casualties')) # (display, field) pairs
+
+ min_date = datetime.datetime(2004, 1, 1, 0, 0, 0)
+ max_date = datetime.datetime(2010, 1, 1, 0, 0, 0)
+
def get_queryset(self):
return DiaryEntry.objects.all()
View
3  afg/templates/afg/entry_table.html
@@ -1,3 +1,4 @@
+{% load afg %}
<ul>
{% for entry, stub in entries %}
<li class='searchresult'>
@@ -5,7 +6,7 @@
<span class='date'>{{ entry.date }}</span><br />
<span class='category'>{{ entry.category }}</span>,
<span class='region'>{{ entry.region }}</span>,
- <span class='casualties' title='{{ entry.casualty_summary }}'>{{ entry.total_casualties }} casualties</span>
+ <span class='casualties' title='{{ entry|casualty_summary }}'>{{ entry.total_casualties }} casualties</span>
<p>{{ stub }}</p>
</li>
{% endfor %}
View
164 afg/templates/afg/search.html
@@ -19,61 +19,82 @@
</script>
<form method='get' action='{% url afg.search %}' onsubmit='return clean_get(this);'>
<input type='text' value='{{ params.q }}' name='q' />
+ {% if params.sort %}<input type='hidden' value='{{ params.sort }}' name='sort' />{% endif %}
<input type='submit' value='search' />
{% for param, choice_opts in choices.items %}
- <h2>{{ choice_opts.title }}</h2>
- {% if choice_opts.value %}
- <input type='hidden' name='{{ param }}' value='{{ choice_opts.value }}' />
- {% endif %}
- <ul>
- {% for choice in choice_opts.choices %}
- <li>
- {% if choice_opts.choices|length > 1 %}
- <a href="{{ qstring }}&{{ param }}={{choice.1}}">{% firstof choice.0 "None" %}</a> ({{ choice.2 }})
+ {% if choice_opts.type == 'min_max' %}
+ <h2>{{ choice_opts.title }}</h2>
+ {% if choice_opts.max_value == choice_opts.min_value %}
+ {{ choice_opts.max_value }}
{% else %}
- {% firstof choice.0 "None" %} ({{ choice.2 }})
+ <div style='width: 90%; margin-left: auto; margin-right: auto;'>
+ <div id='sparkline_{{param}}'></div>
+ <div id='slider_{{param}}'></div>
+ <div>
+ <div style='width: 20%; text-align: left; float: left;'>
+ <input name='{{param}}__gte' id='{{param}}__gte' value='{{ choice_opts.chosen_min }}' size=3 title='{{ choice_opts.min_value }}' />
+ </div>
+ <div style='width: 59%; text-align: center; float: left;'>
+ (<span id='count_{{param}}' style='text-align: center;'>{{ page.paginator.count }}</span> documents)
+ </div>
+ <div style='width: 20%; text-align: right; float: left;'>
+ <input name='{{param}}__lte' id='{{param}}__lte' value='{{ choice_opts.chosen_max }}' size=3 title='{{ choice_opts.max_value }}'/>
+ </div>
+ <div style='clear: both;'></div>
+ </div>
+ </div>
+ <script type='text/javascript'>
+ (function() {
+ var counts = [{{ choice_opts.counts|join:", " }}];
+ var vals = [{{ choice_opts.vals|join:", " }}];
+ // remove '0' for sparkline display; it out-ranges all other values
+ var sparkline_counts = [{{ choice_opts.counts|join:", " }}];
+ sparkline_counts.splice(0, 1);
+ $('#sparkline_{{param}}').sparkline(sparkline_counts, {width: '100%'});
+ var update = function(event, ui) {
+ if (ui.value == ui.values[0]) {
+ $("#{{param}}__gte").val(ui.value).trigger('change');
+ } else if (ui.value == ui.values[1]) {
+ $("#{{param}}__lte").val(ui.value).trigger('change');
+ }
+ var count = 0;
+ for (var i = 0; i < vals.length; i++) {
+ if (vals[i] >= ui.values[0] && vals[i] <= ui.values[1]) {
+ count += counts[i];
+ }
+ }
+ $("#count_{{param}}").html(count);
+ }
+ $('#slider_{{param}}').slider({
+ range: true,
+ values: [{% if choice_opts.chosen_min %} {{ choice_opts.chosen_min }}{% else %}{{ choice_opts.min_value }}{% endif %},
+ {% if choice_opts.chosen_max %} {{ choice_opts.chosen_max }}{% else %}{{ choice_opts.max_value }}{% endif %}],
+ max: {{ choice_opts.max_value }},
+ min: {{ choice_opts.min_value }},
+ change: update,
+ slide: update
+ });
+ })();
+ </script>
{% endif %}
- </li>
- {% endfor %}
- </ul>
- {% endfor %}
-
- {% for param, choice_opts in min_max_choices.items %}
- <h2>{{ choice_opts.title }}</h2>
- <div id='sparkline_{{param}}'></div>
- <div id='slider_{{param}}'></div>
- <div>
- <div style='float: left;'>
- <input name='{{param}}__gte' id='{{param}}__gte' value='{{ choice_opts.chosen_min }}' size=3 />
- (min: {{ choice_opts.min_value }})
- </div>
- <div style='float: right;'>
- (max: {{ choice_opts.max_value }})
- <input name='{{param}}__lte' id='{{param}}__lte' value='{{ choice_opts.chosen_max }}' size=3 />
- </div>
- <div style='clear: both;'></div>
- </div>
- <script type='text/javascript'>
- var vals = [{{choice_opts.counts|join:", "}}];
- // remove '0' which out-ranges all other values
- vals.splice(0, 1);
- $('#sparkline_{{param}}').sparkline(vals, {width: '100%'});
- var update = function(event, ui) {
- var val = $(this).slider('option', 'values');
- $("#{{param}}__gte").val(val[0]);
- $("#{{param}}__lte").val(val[1]);
- }
- $('#slider_{{param}}').slider({
- range: true,
- values: [{% if choice_opts.chosen_min %} {{ choice_opts.chosen_min }}{% else %}{{ choice_opts.min_value }}{% endif %},
- {% if choice_opts.chosen_max %} {{ choice_opts.chosen_max }}{% else %}{{ choice_opts.max_value }}{% endif %}],
- max: {{ choice_opts.max_value }},
- min: {{ choice_opts.min_value }},
- change: update,
- slide: update
- });
- </script>
+ {% else %}
+ <h2>{{ choice_opts.title }}</h2>
+ {% if choice_opts.value %}
+ <input type='hidden' name='{{ param }}' value='{{ choice_opts.value }}' />
+ {% endif %}
+ <ul>
+ {% for choice in choice_opts.choices %}
+ <li>
+ {% if choice_opts.choices|length > 1 %}
+ <a href="{{ qstring }}&{{ param }}={{choice.1}}">{% firstof choice.0 "None" %}</a> ({{ choice.2 }})
+ {% else %}
+ {% firstof choice.0 "None" %} ({{ choice.2 }})
+ {% endif %}
+ </li>
+ {% endfor %}
+ </ul>
+ {% endif %}
{% endfor %}
<p style='text-align: center;'>
<input type='submit' value='Search' />
@@ -100,11 +121,10 @@
{% endfor %}
{% if not about %}
<div class='sort'>Sort by:
- <a href='{{ sort.date }}' class='sortdir'>{% if sort.date_asc %}&uarr;{% else %}{% if sort.date_desc %}&darr;{% endif %}{% endif %}</a>
- <a href='{{ sort.date }}'>Date</a>
- |
- <a href='{{ sort.total_casualties }}' class='sortdir'>{% if sort.total_casualties_asc %}&uarr;{% else %}{% if sort.total_casualties_desc %}&darr;{% endif %}{% endif %}</a>
- <a href='{{ sort.total_casualties }}'>Casualties</a>
+ {% for sort_opts in sort_links %}
+ <a href='{{ sort_opts.link }}' class='sortdir'>{% if sort_opts.current %}{% if sort_opts.desc %}&darr;{% else %}&uarr;{% endif %}{% endif %}</a> <a href='{{ sort_opts.link }}'>{{ sort_opts.title }}</a>
+ {% if not forloop.last %} | {% endif %}
+ {% endfor %}
</div>
{% endif %}
</div>
@@ -120,4 +140,38 @@
No results.
{% endif %}{% endif %}
</div>
+<script type='text/javascript'>
+ // from http://kyleschaeffer.com/best-practices/input-prompt-text/
+ (function(){
+ $('input[title]').each(function(i){
+ $(this).addClass('input-prompt-' + i);
+ var promptSpan = $('<span class="input-prompt"/>');
+ $(promptSpan).attr('id', 'input-prompt-' + i);
+ $(promptSpan).append($(this).attr('title'));
+ $(promptSpan).click(function(){
+ $(this).hide();
+ $('.' + $(this).attr('id')).focus();
+ });
+ if($(this).val() != ''){
+ $(promptSpan).hide();
+ }
+ $(this).before(promptSpan);
+ $(this).change(function(){
+ if ($(this).val() == '') {
+ $('#input-prompt-' + i).show();
+ } else {
+ $('#input-prompt-' + i).hide();
+ }
+ });
+ $(this).focus(function(){
+ $('#input-prompt-' + i).hide();
+ });
+ $(this).blur(function(){
+ if($(this).val() == ''){
+ $('#input-prompt-' + i).show();
+ }
+ });
+ });
+ })();
+</script>
{% endblock %}
View
2  afg/utils.py
@@ -6,7 +6,7 @@
from django.shortcuts import render_to_response
def render_json(request, obj):
- #return HttpResponse(json.dumps(obj, indent=4)) # for debugging
+ return HttpResponse("<pre>" + json.dumps(obj, indent=4)) # for debugging
dumped = json.dumps(obj)
callback = request.GET.get('callback', None)
if callback:
View
317 afg/views.py
@@ -32,7 +32,7 @@ def show_entry(request, rid, template='afg/entry_page.html', api=False):
phrases = Phrase.objects.filter(entry_count__gt=1,
entry_count__lt=10, entries=entry)
-# Equivalent query pre-de-normalization:
+# Equivalent query without denormalization:
# phrases = list(Phrase.objects.raw("""
# SELECT sub.* FROM
# (SELECT p.id, p.phrase, COUNT(pe2.diaryentry_id) AS entry_count FROM
@@ -179,22 +179,22 @@ def search(request, about=False, api=False):
sqs = SearchQuerySet()
params = {}
- text_facets = ('type_', 'region', 'attack_on', 'type_of_unit', 'affiliation',
- 'dcolor', 'classification', 'category')
- integer_facets = ('total_casualties', 'civilian_kia', 'civilian_wia', 'host_nation_kia', 'host_nation_wia',
- 'friendly_kia', 'friendly_wia', 'enemy_kia', 'enemy_wia', 'enemy_detained')
- # prepare fields for faceting. `date` is special-cased later.
- for facet in text_facets:
- sqs = sqs.facet(facet)
- for facet in integer_facets:
- sqs = sqs.facet(facet)
-
# Full text search
q = request.GET.get('q', None)
if q:
sqs = sqs.auto_query(q).highlight()
params['q'] = q
+ # prepare fields for faceting; dates are special-cased later.
+ summary = None
+ date_fields = {}
+ for facet in DiaryEntryIndex.search_facet_display:
+ field = DiaryEntryIndex.fields[facet]
+ if isinstance(field, haystack.fields.DateTimeField):
+ date_fields[facet] = field
+ else:
+ sqs = sqs.facet(facet)
+
# Narrow query set by given facets
for key,val in request.GET.iteritems():
if val:
@@ -205,12 +205,22 @@ def search(request, about=False, api=False):
# "type" is a reserved name for Solr, so munge it to "type_"
field_name = "type_" if field_name == "type" else field_name
# Dates are handled specially below
- if field_name == 'date':
- continue
field = DiaryEntryIndex.fields.get(field_name, None)
- if field:
- val = sqs.query.clean(val)
- print field, val
+ if field and field.faceted:
+ if isinstance(field, haystack.fields.DateTimeField):
+ continue
+ elif isinstance(field, haystack.fields.IntegerField):
+ try:
+ val = int(val)
+ except ValueError:
+ continue
+ elif isinstance(field, haystack.fields.FloatField):
+ try:
+ val = float(val)
+ except ValueError:
+ continue
+ else:
+ val = sqs.query.clean(val)
if lookup == 'exact':
sqs = sqs.narrow(u'%s:"%s"' % (field.index_fieldname, val))
elif lookup == 'gte':
@@ -221,50 +231,61 @@ def search(request, about=False, api=False):
continue
params[key] = val
- # Narrow query set by given dates
- date = request.GET.get('date', '')
- if date:
- year, month, day = [int(d) for d in (date + "-0-0").split("-")[0:3]]
- else:
- # Legacy date format
- day = int(request.GET.get('date__day', 0))
- month = int(request.GET.get('date__month', 0))
- year = int(request.GET.get('date__year', 0))
- if year:
- if not month:
- start = datetime.datetime(year, 1, 1)
- end = datetime.datetime(year + 1, 1, 1) - datetime.timedelta(seconds=1)
- params['date'] = start.strftime("%Y")
- sqs = sqs.date_facet('date', start, end, 'month')
- elif not day:
- start = datetime.datetime(year, month, 1)
- next_month = datetime.datetime(year, month, 1) + datetime.timedelta(days=31)
- end = datetime.datetime(next_month.year, next_month.month, 1) - datetime.timedelta(seconds=1)
- params['date'] = start.strftime("%Y-%m")
- sqs = sqs.date_facet('date', start, end, 'day')
+ # Narrow query set by given date facets
+ for key, field in date_fields.iteritems():
+ date = request.GET.get(key, '')
+ if date:
+ try:
+ year, month, day = [int(d) for d in (date + "-0-0").split("-")[0:3]]
+ if year > 3000 or month > 12 or day > 31:
+ raise ValueError
+ except ValueError:
+ year, month, day = 0, 0, 0
else:
- start = datetime.datetime(year, month, day)
- end = datetime.datetime(year, month, day + 1) - datetime.timedelta(seconds=1)
- params['date'] = start.strftime("%Y-%m-%d")
- sqs = sqs.date_facet('date', start, end, 'day')
- sqs = sqs.narrow("date:[%s TO %s]" % (start.isoformat() + "Z", end.isoformat() + "Z"))
- else:
- start = datetime.datetime(2004, 1, 1, 0, 0, 0)
- end = datetime.datetime.now()
- sqs = sqs.date_facet('date', start, end, 'year')
-
+ # Legacy date format
+ day = int(request.GET.get(key + '__day', 0))
+ month = int(request.GET.get(key + '__month', 0))
+ year = int(request.GET.get(key + '__year', 0))
+ if year:
+ if not month:
+ start = datetime.datetime(year, 1, 1)
+ end = datetime.datetime(year + 1, 1, 1) - datetime.timedelta(seconds=1)
+ params[key] = start.strftime("%Y")
+ sqs = sqs.date_facet(key, start, end, 'month')
+ elif not day:
+ start = datetime.datetime(year, month, 1)
+ next_month = datetime.datetime(year, month, 1) + datetime.timedelta(days=31)
+ end = datetime.datetime(next_month.year, next_month.month, 1) - datetime.timedelta(seconds=1)
+ params[key] = start.strftime("%Y-%m")
+ sqs = sqs.date_facet(key, start, end, 'day')
+ else:
+ start = datetime.datetime(year, month, day)
+ end = datetime.datetime(year, month, day + 1) - datetime.timedelta(seconds=1)
+ params[key] = start.strftime("%Y-%m-%d")
+ sqs = sqs.date_facet('date', start, end, 'day')
+ sqs = sqs.narrow("%s:[%s TO %s]" % (key, start.isoformat() + "Z", end.isoformat() + "Z"))
+ else:
+ start = DiaryEntryIndex.min_date
+ end = DiaryEntryIndex.max_date
+ sqs = sqs.date_facet(key, start, end, 'year')
# sorting
- sort_by = request.GET.get('sort_by', 'date')
- sort_dir = request.GET.get('sort_dir', 'asc')
- direction_indicator = '-' if sort_dir == 'desc' else ''
- if sort_by in ('date', 'total_casualties'):
- sqs = sqs.order_by(direction_indicator + sort_by)
- params['sort_by'] = sort_by
- params['sort_dir'] = sort_dir
+ sort = request.GET.get('sort', '')
+ # Legacy sorting
+ if request.GET.get('sort_by', ''):
+ sort_by = request.GET.get('sort_by', '')
+ if sort_by == 'casualties':
+ sort_by = 'total_casualties'
+ sort_dir = request.GET.get('sort_dir', 'asc')
+ direction_indicator = '-' if sort_dir == 'desc' else ''
+ sort = direction_indicator + sort_by
+ if sort.strip('-') not in DiaryEntryIndex.fields:
+ sort = DiaryEntryIndex.offer_to_sort_by[0][1]
+ sqs = sqs.order_by(sort)
+ params['sort'] = sort
# Pagination
- p = paginator.Paginator(sqs.load_all(), 10)
+ p = paginator.Paginator(sqs, 10)
try:
page = p.page(int(request.GET.get('p', 1)))
except (ValueError, paginator.InvalidPage, paginator.EmptyPage):
@@ -283,83 +304,59 @@ def search(request, about=False, api=False):
total_count = sqs.count()
counts = sqs.facet_counts()
choices = utils.OrderedDict()
- date_facets = []
- for d,c in sorted(counts['dates']['date'].iteritems()):
- try:
- # magic method to parse ISO date format.
- dt = datetime.datetime(*map(int, re.split('[^\d]', d)[:-1]))
- if c > 0:
- date_facets.append((dt, c))
- except (TypeError, ValueError):
- pass
-
- # Date choices
- choices['date'] = {
- 'title': 'Date',
- 'value': params.get('date', ''),
- }
- year, month, day = (params.get('date', '') + "--").split("-")[0:3]
- if year:
- if month:
- if day:
- d = datetime.date(int(year), int(month), int(day))
- choices['date']['choices'] = [(d.strftime("%Y %B %e"), d.strftime("%Y-%m-%d"), total_count)]
- else:
- choices['date']['choices'] = [(d.strftime("%B %e"), d.strftime("%Y-%m-%d"), c) for d, c in date_facets]
- else:
- choices['date']['choices'] = [(d.strftime("%B"), d.strftime("%Y-%m"), c) for d, c in date_facets]
- else:
- choices['date']['choices'] = [(d.year, d.year, c) for d, c in date_facets]
-
- # Text field choices
- for field in text_facets:
- choices[field] = {
- 'title': field.replace('_', ' ').title(),
- 'choices': sorted((k, k, c) for k, c in counts['fields'][field] if c > 0),
- 'value': params.get(field, ''),
+ for key in DiaryEntryIndex.search_facet_display:
+ field = DiaryEntryIndex.fields[key]
+ choice = {
+ 'title': fix_constraint_name(key),
+ 'value': params.get(key, '')
}
-
- # Integer choices
- min_max_choices = utils.OrderedDict()
- for field in integer_facets:
- facets = sorted([(int(k), v) for k,v in counts['fields'][field]])
- min_max_choices[field] = {
- 'title': fix_constraint_name(field),
- 'counts': [v for k,v in facets],
- 'min_value': facets[0][0],
- 'max_value': facets[-1][0],
- 'min_count': facets[0][1],
- 'max_count': facets[-1][1],
- 'chosen_min': params.get(field + '__gte', ''),
- 'chosen_max': params.get(field + '__lte', ''),
- }
-
-
-
-# # Integer choices
-# for field in integer_facets:
-# try:
-# minimum = qs.order_by(field).values(field)[0][field]
-# maximum = qs.order_by("-%s" % field).values(field)[0][field]
-# except IndexError:
-# continue
-# if minimum == maximum and \
-# not params.get(field + '__gte') and \
-# not params.get(field + '__lte'):
-# continue
-# min_max_choices[field] = {
-# 'min': minimum,
-# 'max': maximum,
-# 'title': fix_constraint_name(field),
-# 'min_value': params.get(field + '__gte', ''),
-# 'max_value': params.get(field + '__lte', ''),
-# }
+ if isinstance(field, haystack.fields.CharField):
+ choice['choices'] = sorted((k, k, c) for k, c in counts['fields'][key] if c > 0)
+ choice['type'] = 'text'
+ elif isinstance(field, haystack.fields.DateTimeField):
+ choice['type'] = 'date'
+ date_facets = []
+ for d,c in sorted(counts['dates'][key].iteritems()):
+ try:
+ # magic method to parse ISO date format.
+ dt = datetime.datetime(*map(int, re.split('[^\d]', d)[:-1]))
+ if c > 0:
+ date_facets.append((dt, c))
+ except (TypeError, ValueError):
+ pass
+
+ year, month, day = (params.get(key, '') + "--").split("-")[0:3]
+ if year:
+ if month:
+ if day:
+ d = datetime.date(int(year), int(month), int(day))
+ choice['choices'] = [(d.strftime("%Y %B %e"), d.strftime("%Y-%m-%d"), total_count)]
+ else:
+ choice['choices'] = [(d.strftime("%B %e"), d.strftime("%Y-%m-%d"), c) for d, c in date_facets]
+ else:
+ choice['choices'] = [(d.strftime("%B"), d.strftime("%Y-%m"), c) for d, c in date_facets]
+ else:
+ choice['choices'] = [(d.year, d.year, c) for d, c in date_facets]
+ elif isinstance(field, haystack.fields.IntegerField):
+ # Integer choices
+ facets = sorted([(int(k), v) for k,v in counts['fields'][key] if v > 0])
+ if facets:
+ choice.update({
+ 'type': 'min_max',
+ 'counts': [v for k,v in facets],
+ 'vals': [k for k,v in facets],
+ 'min_value': facets[0][0],
+ 'max_value': facets[-1][0],
+ 'chosen_min': params.get(key + '__gte', ''),
+ 'chosen_max': params.get(key + '__lte', ''),
+ })
+ choices[key] = choice
search_url = reverse('afg.search')
# Links to remove constraints
constraints = {}
- exclude = set(('sort_by', 'sort_dir'))
+ exclude = set(('sort',))
for field in params.keys():
if field not in exclude:
value = params.pop(field)
@@ -371,40 +368,45 @@ def search(request, about=False, api=False):
params[field] = value
# Links to change sorting
- sort = {}
- for by in ('date', 'total_casualties'):
- sort_by = params.pop('sort_by', 'date')
- sort_dir = params.pop('sort_dir', 'asc')
- params['sort_by'] = by
- if sort_by == by:
- if sort_dir == 'asc':
- params['sort_dir'] = 'desc'
- sort[by + '_asc'] = True
- else:
- sort[by + '_desc'] = True
+ sort_links = []
+ current_sort = params.pop('sort')
+ current_key = sort.strip('-')
+ for display, new_key in DiaryEntryIndex.offer_to_sort_by:
+ # Change directions only if it's the same sort key
+ if current_key == new_key:
+ direction = '' if current_sort[0] == '-' else '-'
else:
- params['sort_dir'] = sort_dir
- sort[by] = "%s?%s" % (search_url, urllib.urlencode(params))
- params['sort_by'] = sort_by
- params['sort_dir'] = sort_dir
+ direction = '-' if current_sort[0] == '-' else ''
+ params['sort'] = direction + new_key
+ sort_links.append({
+ 'link': "%s?%s" % (search_url, urllib.urlencode(params)),
+ 'title': display,
+ 'desc': current_sort[0] == '-',
+ 'current': current_key == new_key,
+ })
+ params['sort'] = current_sort
if api:
remapped_choices = {}
for choice, opts in choices.iteritems():
- remapped_choices[choice] = {
- 'value': opts['value'],
- 'title': opts['title'],
- 'choices': []
- }
- for disp, val in opts['choices']:
- if disp or val:
- remapped_choices[choice]['choices'].append({
- 'value': val,
- 'display': disp,
- })
+ if opts['type'] == 'min_max':
+ remapped_choices[choice] = opts
+ else:
+ remapped_choices[choice] = {
+ 'value': opts['value'],
+ 'title': opts['title'],
+ 'choices': [],
+ }
+ for disp, val, count in opts['choices']:
+ if disp or val:
+ remapped_choices[choice]['choices'].append({
+ 'value': val,
+ 'display': disp,
+ 'count': count,
+ })
return utils.render_json(request, {
- 'pageination': {
+ 'pagination': {
'p': page.number,
'num_pages': page.paginator.num_pages,
'num_results': page.paginator.count,
@@ -414,11 +416,7 @@ def search(request, about=False, api=False):
'excerpt': x
} for (e,x) in entries],
'choices': remapped_choices,
- 'min_max_choices': min_max_choices,
- 'sort': {
- 'sort_by': params['sort_by'],
- 'sort_dir': params['sort_dir'],
- },
+ 'sort': params['sort'],
'params': params,
})
@@ -427,10 +425,9 @@ def search(request, about=False, api=False):
'entries': entries,
'params': request.GET,
'choices': choices,
- 'min_max_choices': min_max_choices,
'qstring': '%s?%s' % (search_url, urllib.urlencode(params)),
'constraints': constraints,
- 'sort': sort,
+ 'sort_links': sort_links,
})
def fix_constraint_name(field):
View
6 media/css/sass/style.sass
@@ -153,3 +153,9 @@ acronym, .toggle-acronyms
float: right
a.sortdir, a.sortdir:visited
text-decoration: none
+
+.input-prompt
+ position: absolute
+ color: #aaa
+ margin: 0.2em 0 0 0.5em
+
View
7 media/css/style.css
@@ -356,3 +356,10 @@ acronym, .toggle-acronyms {
.constraints .sort a.sortdir, .constraints .sort a.sortdir:visited {
text-decoration: none;
}
+
+/* line 157, ././style.sass */
+.input-prompt {
+ position: absolute;
+ color: #aaaaaa;
+ margin: 0.2em 0 0 0.5em;
+}
View
24 media/js/script.js
@@ -296,7 +296,7 @@ var ACRONYMS = [
[/\b(HVI)\b/g, "High-value Individual"],
[/\b(HWY)\b/g, "Highway"],
[/\b(IAW)\b/g, "In accordance with"],
- [/\b(ICOM)\b/g, "Radio"],
+ [/\b(ICOM)\b/g, "Icom [brand of radio]"],
[/\b(IDF)\b/g, "Indirect Fire"],
[/\b(IED)\b/g, "Improvised Explosive Device"],
[/\b(Illum)\b/g, "Illumination mortar, fired to provide light"],
@@ -463,18 +463,18 @@ var ACRONYMS = [
[/\b(S-5)\b/g, "Staff member responsible for civil-military operations"],
[/\b(SALUTE)\b/g, "Size/Activity/Location/Unit/Time/Equipment"],
[/\b(SALUTER)\b/g, "Size/Activity/Location/Unit/Time/Equipment/Result"],
- [/( S[-:])/g, " Size: "],
- [/( A[-:])/g, " Activity: "],
- [/( L[-:])/g, " Location: "],
- [/( T[-:])/g, " Time: "],
- [/( U[-:])/g, " Unit: "],
- [/( E[-:])/g, " Equipment: "],
- [/( R[-:])/g, " Result: "],
- [/\b(S\/\/REL)\b/g, "Secret or Selective release?"],
+ [/((\b| )S[-:])/g, " Size: "],
+ [/((\b| )A[-:])/g, " Activity: "],
+ [/((\b| )L[-:])/g, " Location: "],
+ [/((\b| )T[-:])/g, " Time: "],
+ [/((\b| )U[-:])/g, " Unit: "],
+ [/((\b| )E[-:])/g, " Equipment: "],
+ [/((\b| )R[-:])/g, " Result: "],
+ [/\b(S\/\/REL)\b/g, "Secret, release only to:"],
[/\b(SAF)\b/g, "Small arms fire"],
[/\b(Safire)\b/gi, "Small arms fire/Surface-to-Air Fire"],
[/\b(SAMBUSH)\b/gi, "Surface-to-Air Missle Ambush"],
- [/\b(SAW)\b/g, "Squad Automatic Weapon [ machine gun]"],
+ [/\b(SAW)\b/g, "Squad Automatic Weapon [machine gun]"],
[/\b(SC-26)\b/g, "Scorpion 26 - US special forces unit in Helmand"],
[/\b(SCIDA)\b/g, "Site Configuration and Installation Design Authority?"],
[/\b(SEWOC)\b/g, "Sigint Electronic Warfare Operational Centre"],
@@ -521,8 +521,7 @@ var ACRONYMS = [
[/\b(UXO)\b/g, "unexploded ordnance [or unfired]"],
[/\b(VBIED)\b/g, "Vehicle-Borne Improvised Explosive Device"],
[/\b(VCP)\b/g, "Vehicle Check Point"],
- [/\b(vic)\b/g, "vicinity"],
- [/\b(VIC)\b/g, "Vehicle"],
+ [/\b(vic)\b/gi, "vicinity"],
[/\b(vitals)\b/g, "Vital signs"],
[/\b(VP)\b/g, "Vulnerable point"],
[/\b(VPB)\b/g, "Vehicle patrol base"],
@@ -561,3 +560,4 @@ function toggleAcronyms(expand) {
}
});
}
+
Please sign in to comment.
Something went wrong with that request. Please try again.