Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
  • 7 commits
  • 15 files changed
  • 0 comments
  • 1 contributor
15 rundev_scraper.sh
... ... @@ -0,0 +1,15 @@
  1 +#!/bin/bash
  2 +DDIR=$PWD/_data/scraped_`date +%F-%T`
  3 +mkdir -p $DDIR
  4 +export SENTRY_DSN=""
  5 +export SCRAPY_DATA_DIR=$DDIR
  6 +export PYTHONPATH=.:/home/myzivi/django-myzivi:$PYTHONPATH
  7 +export DJANGO_SETTINGS_MODULE=ziviweb.settings
  8 +# Crawler
  9 +scrapy crawl zivi
  10 +# Postprocess (geotagging, translation, ...)
  11 +python ziviscrap/postprocess.py
  12 +# Insert into DB
  13 +python zivimap/import_items.py
  14 +# Delete old scraped directory
  15 +python rotate_data.py
66 zivimap/api.py
... ... @@ -1,8 +1,10 @@
1 1 import ziviweb.settings as settings
2 2 from django.core.urlresolvers import reverse
3   -from zivimap.models import WorkSpec, Address
4   -from tastypie.resources import ModelResource, Resource, Bundle
  3 +from zivimap.models import WorkSpec, Address, DateRange
  4 +from django.db.models import Q, Count
  5 +from tastypie.resources import ModelResource, Resource, Bundle, ALL
5 6 from tastypie import fields, utils
  7 +from datetime import datetime
6 8
7 9 class AddressResource(ModelResource):
8 10 class Meta:
@@ -11,21 +13,26 @@ class Meta:
11 13 include_resource_uri = False
12 14 limit = 0
13 15
14   -class MapSearchResource(ModelResource):
  16 +class WorkSpecSearchResource(ModelResource):
15 17 """
16 18 This is a specialized resource that returns a list of workspecs for a given
17 19 query. The workspecs description is shortened to only a few fields to
18   - save bandwidth.
  20 + reduce response size
19 21 """
20 22 class Meta:
21 23 queryset = WorkSpec.objects.all()
22   - resource_name = 'search'
  24 + resource_name = 'workspec_search'
23 25 allowed_methods = ['get']
24 26 include_resource_uri = False
25 27 # Without that, resource_uri is empty in the view
26 28 api_name = settings.API_VERSION
27 29 limit = 0
28   - fields = ['address_id', 'shortname', 'raw_phid']
  30 + max_limit = 0
  31 + fields = ['address_id', 'shortname', 'raw_phid', 'phid']
  32 + filtering = {
  33 + 'activity_domain': ('in',),
  34 + 'language': ('in',),
  35 + }
29 36
30 37 def dehydrate(self, bundle):
31 38 bundle.data['addrid'] = bundle.obj.address_id
@@ -34,21 +41,42 @@ def dehydrate(self, bundle):
34 41 def build_filters(self, filters=None):
35 42 if filters is None:
36 43 filters = {}
37   - orm_filters = super(MapSearchResource, self).build_filters(filters)
38   -
39   - # We accept a "latlngbb=47.365,8.783,47.367,8.785" parameter
40   - # specifying a latlng bounding box (as the one returned by google maps
41   - # getBounds())
42   - if 'latlngbb' in filters:
43   - latlngbb = filters['latlngbb']
44   - swlat, swlng, nelat, nelng = map(float, latlngbb.split(','))
  44 + orm_filters = super(WorkSpecSearchResource, self).build_filters(filters)
  45 + # By default, tastypie doesn't allow to filter on fields that are
  46 + # not in Meta.fields. Overwrite this
  47 + lang_filters = filters.getlist('language__in')
  48 + if lang_filters:
  49 + orm_filters['language__in'] = lang_filters
  50 + domain_filters = filters.getlist('activity_domain__in')
  51 + if domain_filters:
  52 + orm_filters['activity_domain__in'] = domain_filters
  53 + return orm_filters
45 54
46   - assert swlat < nelat
47   - assert swlng < nelng
48   - orm_filters['address__latitude__range'] = (swlat, nelat)
49   - orm_filters['address__longitude__range'] = (swlng, nelng)
  55 + def apply_filters(self, request, applicable_filters):
  56 + res = super(WorkSpecSearchResource, self).apply_filters(request,
  57 + applicable_filters)
  58 + # Handle complex filters here
  59 + start_min = request.GET.get('start_min', None)
  60 + end_max = request.GET.get('end_max', None)
  61 + if start_min is not None or end_max is not None:
  62 + drfilters = Q()
  63 + if start_min is not None:
  64 + start_date = datetime.strptime(start_min.strip(),
  65 + '%Y-%m-%d').date()
  66 + drfilters &= Q(start__gte=start_date)
  67 + if end_max is not None:
  68 + end_date = datetime.strptime(end_max.strip(),
  69 + '%Y-%m-%d').date()
  70 + drfilters &= Q(end__lte=end_date)
  71 + ranges = DateRange.objects.filter(drfilters).values('workspec_id')\
  72 + .annotate()
  73 + # ID of workspecs that don't have a daterange
  74 + # TODO: This should be merged in the subsequent workspec query
  75 + no_ranges = WorkSpec.objects.annotate(rc=Count('daterange'))\
  76 + .filter(rc=0).values('phid').annotate()
  77 + res = res.filter(Q(phid__in=ranges) | Q(phid__in=no_ranges))
  78 + return res
50 79
51   - return orm_filters
52 80
53 81 class WorkSpecResource(ModelResource):
54 82 address = fields.ToOneField(AddressResource, 'address', full=True)
89 zivimap/forms.py
... ... @@ -0,0 +1,89 @@
  1 +"""
  2 +Adaptation of django.forms.extra.SelectDateWidget to show only month and year
  3 +"""
  4 +from __future__ import unicode_literals
  5 +
  6 +import datetime
  7 +import re
  8 +import calendar
  9 +
  10 +from django.forms.widgets import Widget, Select
  11 +from django.utils import datetime_safe
  12 +from django.utils.dates import MONTHS
  13 +from django.utils.encoding import force_str
  14 +from django.utils.safestring import mark_safe
  15 +from django.utils.formats import get_format
  16 +from django.utils import six
  17 +from django.conf import settings
  18 +
  19 +__all__ = ('SelectDateWidget',)
  20 +
  21 +class SelectMonthYearWidget(Widget):
  22 + """
  23 + A Widget that splits date input into two <select> boxes and allow selection
  24 + of month/year
  25 + """
  26 + none_value = (0, '---')
  27 + month_field = '%s_month'
  28 + year_field = '%s_year'
  29 +
  30 + def __init__(self, defaultdate, day, attrs=None, years=None,
  31 + required=True):
  32 + # dayval is the value that will be set for the days. Can be 'first'
  33 + # or 'last'
  34 + # years is an optional list/tuple of years to use in the "year" select box.
  35 + self.attrs = attrs or {}
  36 + self.required = required
  37 + self.dayval = day
  38 + self.defaultdate = defaultdate
  39 + if years:
  40 + self.years = years
  41 + else:
  42 + this_year = datetime.date.today().year
  43 + self.years = range(this_year, this_year+10)
  44 +
  45 + def render(self, name, value, attrs=None):
  46 + try:
  47 + year_val, month_val = value.year, value.month
  48 + except AttributeError:
  49 + year_val, month_val = self.defaultdate.year, self.defaultdate.month
  50 +
  51 + choices = [(i, i) for i in self.years]
  52 + year_html = self.create_select(name, self.year_field, value, year_val, choices)
  53 + choices = list(six.iteritems(MONTHS))
  54 + month_html = self.create_select(name, self.month_field, value, month_val, choices)
  55 +
  56 + output = [month_html, year_html]
  57 + return mark_safe('\n'.join(output))
  58 +
  59 + def id_for_label(self, id_):
  60 + return '%s_month' % id_
  61 +
  62 + def value_from_datadict(self, data, files, name):
  63 + y = data.get(self.year_field % name)
  64 + m = data.get(self.month_field % name)
  65 + if y == m == "0": # No selection
  66 + return None
  67 + if y and m:
  68 + y = int(y)
  69 + m = int(m)
  70 + if self.dayval == 'first':
  71 + day = 1
  72 + else:
  73 + day = calendar.monthrange(y, m)[1]
  74 + return datetime.date(int(y), int(m), day)
  75 + else:
  76 + return None
  77 +
  78 + def create_select(self, name, field, value, val, choices):
  79 + if 'id' in self.attrs:
  80 + id_ = self.attrs['id']
  81 + else:
  82 + id_ = 'id_%s' % name
  83 + #if not (self.required and val):
  84 + #choices.insert(0, self.none_value)
  85 + local_attrs = self.build_attrs(id=field % id_)
  86 + local_attrs['class'] = field % 'select'
  87 + s = Select(choices=choices)
  88 + select_html = s.render(field % name, val, local_attrs)
  89 + return select_html
46 zivimap/locale/de/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
8 8 msgstr ""
9 9 "Project-Id-Version: PACKAGE VERSION\n"
10 10 "Report-Msgid-Bugs-To: \n"
11   -"POT-Creation-Date: 2013-02-18 17:50+0100\n"
  11 +"POT-Creation-Date: 2013-03-05 12:45+0100\n"
12 12 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
13 13 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
14 14 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -18,51 +18,60 @@ msgstr ""
18 18 "Content-Transfer-Encoding: 8bit\n"
19 19 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
20 20
21   -#: models.py:30
  21 +#: models.py:32
22 22 msgid "Health"
23 23 msgstr "Gesundheitswesen"
24 24
25   -#: models.py:30
  25 +#: models.py:32
26 26 msgid "Social"
27 27 msgstr "Sozialwesen"
28 28
29   -#: models.py:31
  29 +#: models.py:33
30 30 msgid "Culture"
31 31 msgstr "Kulturgütererhaltung"
32 32
33   -#: models.py:31
  33 +#: models.py:33
34 34 msgid "Nature protection"
35 35 msgstr "Umwelt- und Naturschutz"
36 36
37   -#: models.py:32
  37 +#: models.py:34
38 38 msgid "Forest work"
39 39 msgstr "Forstwesen"
40 40
41   -#: models.py:32
  41 +#: models.py:34
42 42 msgid "Agriculture"
43 43 msgstr "Landwirtschaft"
44 44
45   -#: models.py:33
  45 +#: models.py:35
46 46 msgid "Development & cooperation"
47 47 msgstr "Entwicklungszusammenar..."
48 48
49   -#: models.py:34
  49 +#: models.py:36
50 50 msgid "Disasters help"
51 51 msgstr "Bewältigung von Katast..."
52 52
53   -#: views.py:24
  53 +#: views.py:29 templates/search_form.html:9
54 54 msgid "Domains"
55 55 msgstr "Tätigkeitsbereich"
56 56
57   -#: views.py:28
  57 +#: views.py:33 templates/search_form.html:15
58 58 msgid "Languages"
59 59 msgstr "Sprache"
60 60
61   -#: templates/base.html:14 templates/base.html.py:26
62   -msgid "swiss civilian service search"
63   -msgstr "Schweizer zivildienst einsatz suche"
  61 +#: views.py:41 templates/search_form.html:21
  62 +msgid "Earliest start"
  63 +msgstr "Frühester Start"
  64 +
  65 +#: views.py:45 templates/search_form.html:27
  66 +msgid "Latest end"
  67 +msgstr "Spätestes Ende"
64 68
65   -#: templates/base.html:58
  69 +#: templates/base.html:16 templates/base.html.py:28
  70 +#, fuzzy
  71 +msgid "(non-official) swiss civilian service search"
  72 +msgstr "Schweizer zivildienst einsatz suche (inoffiziell)"
  73 +
  74 +#: templates/base.html:60
66 75 #, python-format
67 76 msgid ""
68 77 "\n"
@@ -76,11 +85,14 @@ msgid ""
76 85 " "
77 86 msgstr ""
78 87
79   -#: templates/index.html:45
  88 +#: templates/index.html:42
80 89 #, fuzzy
81 90 msgid "Search options"
82 91 msgstr "Suchen"
83 92
84   -#: templates/index.html:50
  93 +#: templates/index.html:46
85 94 msgid "Search"
86 95 msgstr "Suchen"
  96 +
  97 +#~ msgid "swiss civilian service search"
  98 +#~ msgstr "Schweizer zivildienst einsatz suche"
40 zivimap/locale/en/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
8 8 msgstr ""
9 9 "Project-Id-Version: PACKAGE VERSION\n"
10 10 "Report-Msgid-Bugs-To: \n"
11   -"POT-Creation-Date: 2013-02-18 17:50+0100\n"
  11 +"POT-Creation-Date: 2013-03-05 12:45+0100\n"
12 12 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
13 13 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
14 14 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,51 +17,59 @@ msgstr ""
17 17 "Content-Type: text/plain; charset=UTF-8\n"
18 18 "Content-Transfer-Encoding: 8bit\n"
19 19
20   -#: models.py:30
  20 +#: models.py:32
21 21 msgid "Health"
22 22 msgstr ""
23 23
24   -#: models.py:30
  24 +#: models.py:32
25 25 msgid "Social"
26 26 msgstr ""
27 27
28   -#: models.py:31
  28 +#: models.py:33
29 29 msgid "Culture"
30 30 msgstr ""
31 31
32   -#: models.py:31
  32 +#: models.py:33
33 33 msgid "Nature protection"
34 34 msgstr ""
35 35
36   -#: models.py:32
  36 +#: models.py:34
37 37 msgid "Forest work"
38 38 msgstr ""
39 39
40   -#: models.py:32
  40 +#: models.py:34
41 41 msgid "Agriculture"
42 42 msgstr ""
43 43
44   -#: models.py:33
  44 +#: models.py:35
45 45 msgid "Development & cooperation"
46 46 msgstr ""
47 47
48   -#: models.py:34
  48 +#: models.py:36
49 49 msgid "Disasters help"
50 50 msgstr ""
51 51
52   -#: views.py:24
  52 +#: views.py:29 templates/search_form.html:9
53 53 msgid "Domains"
54 54 msgstr ""
55 55
56   -#: views.py:28
  56 +#: views.py:33 templates/search_form.html:15
57 57 msgid "Languages"
58 58 msgstr ""
59 59
60   -#: templates/base.html:14 templates/base.html.py:26
61   -msgid "swiss civilian service search"
  60 +#: views.py:41 templates/search_form.html:21
  61 +msgid "Earliest start"
  62 +msgstr ""
  63 +
  64 +#: views.py:45 templates/search_form.html:27
  65 +msgid "Latest end"
  66 +msgstr ""
  67 +
  68 +#: templates/base.html:16 templates/base.html.py:28
  69 +msgid "(non-official) swiss civilian service search"
62 70 msgstr ""
63 71
64   -#: templates/base.html:58
  72 +#: templates/base.html:60
65 73 #, python-format
66 74 msgid ""
67 75 "\n"
@@ -75,10 +83,10 @@ msgid ""
75 83 " "
76 84 msgstr ""
77 85
78   -#: templates/index.html:45
  86 +#: templates/index.html:42
79 87 msgid "Search options"
80 88 msgstr ""
81 89
82   -#: templates/index.html:50
  90 +#: templates/index.html:46
83 91 msgid "Search"
84 92 msgstr ""
46 zivimap/locale/fr/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
8 8 msgstr ""
9 9 "Project-Id-Version: PACKAGE VERSION\n"
10 10 "Report-Msgid-Bugs-To: \n"
11   -"POT-Creation-Date: 2013-02-18 17:50+0100\n"
  11 +"POT-Creation-Date: 2013-03-05 12:45+0100\n"
12 12 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
13 13 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
14 14 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -18,51 +18,60 @@ msgstr ""
18 18 "Content-Transfer-Encoding: 8bit\n"
19 19 "Plural-Forms: nplurals=2; plural=(n > 1)\n"
20 20
21   -#: models.py:30
  21 +#: models.py:32
22 22 msgid "Health"
23 23 msgstr "Santé"
24 24
25   -#: models.py:30
  25 +#: models.py:32
26 26 msgid "Social"
27 27 msgstr "Social"
28 28
29   -#: models.py:31
  29 +#: models.py:33
30 30 msgid "Culture"
31 31 msgstr "Culture"
32 32
33   -#: models.py:31
  33 +#: models.py:33
34 34 msgid "Nature protection"
35 35 msgstr "Protection de la nature"
36 36
37   -#: models.py:32
  37 +#: models.py:34
38 38 msgid "Forest work"
39 39 msgstr "Travaux forestiers"
40 40
41   -#: models.py:32
  41 +#: models.py:34
42 42 msgid "Agriculture"
43 43 msgstr "Agriculture"
44 44
45   -#: models.py:33
  45 +#: models.py:35
46 46 msgid "Development & cooperation"
47 47 msgstr "Développement & coopéeration"
48 48
49   -#: models.py:34
  49 +#: models.py:36
50 50 msgid "Disasters help"
51 51 msgstr "Aide en cas de catastrophe"
52 52
53   -#: views.py:24
  53 +#: views.py:29 templates/search_form.html:9
54 54 msgid "Domains"
55 55 msgstr "Domaines"
56 56
57   -#: views.py:28
  57 +#: views.py:33 templates/search_form.html:15
58 58 msgid "Languages"
59 59 msgstr "Langue"
60 60
61   -#: templates/base.html:14 templates/base.html.py:26
62   -msgid "swiss civilian service search"
63   -msgstr "recherche d'affectations pour le service civil suisse"
  61 +#: views.py:41 templates/search_form.html:21
  62 +msgid "Earliest start"
  63 +msgstr "Début au plus tôt"
  64 +
  65 +#: views.py:45 templates/search_form.html:27
  66 +msgid "Latest end"
  67 +msgstr "Fin au plus tard"
64 68
65   -#: templates/base.html:58
  69 +#: templates/base.html:16 templates/base.html.py:28
  70 +#, fuzzy
  71 +msgid "(non-official) swiss civilian service search"
  72 +msgstr "recherche d'affectations pour le service civil suisse (non-officiel)"
  73 +
  74 +#: templates/base.html:60
66 75 #, python-format
67 76 msgid ""
68 77 "\n"
@@ -86,10 +95,13 @@ msgstr ""
86 95 " Pour plus d'informations, cliques sur \"About\".\n"
87 96 " "
88 97
89   -#: templates/index.html:45
  98 +#: templates/index.html:42
90 99 msgid "Search options"
91 100 msgstr "Options de recherche"
92 101
93   -#: templates/index.html:50
  102 +#: templates/index.html:46
94 103 msgid "Search"
95 104 msgstr "Chercher"
  105 +
  106 +#~ msgid "swiss civilian service search"
  107 +#~ msgstr "recherche d'affectations pour le service civil suisse"
7 zivimap/static/css/main.css
@@ -84,3 +84,10 @@ ul#search_form label {
84 84 display: inline;
85 85 }
86 86
  87 +#search_form select.select_month {
  88 + width: 100px;
  89 +}
  90 +
  91 +#search_form select.select_year {
  92 + width: 100px;
  93 +}
14 zivimap/static/js/map.js
@@ -14,9 +14,8 @@ function url_from_phid(phid) {
14 14 return ZIVI_WS_BASE + '&phid=' + phid;
15 15 }
16 16
17   -// initialWorkspecs is a list of JSON workspecs
18 17 // N is the namespace to use for this app
19   -function initMap(initialWorkspecs, initialAddresses, N) {
  18 +function initMap(N) {
20 19 N.WorkSpec = Backbone.Model.extend({
21 20 urlRoot: SEARCH_API,
22 21 });
@@ -214,7 +213,7 @@ function initMap(initialWorkspecs, initialAddresses, N) {
214 213 },
215 214 });
216 215
217   - N.addresses = new N.AddressList(initialAddresses);
  216 + N.addresses = new N.AddressList();
218 217 N.workspecs = new N.WorkSpecList();
219 218
220 219 //N.listview = new N.ListView({
@@ -226,5 +225,12 @@ function initMap(initialWorkspecs, initialAddresses, N) {
226 225 addresses: N.addresses,
227 226 });
228 227
229   - N.workspecs.reset(initialWorkspecs);
  228 + // Workspecs depends on all addresses being loaded (otherwise MapView will
  229 + // fail). So chain them
  230 + // TODO: Need to handle potential ajax errors
  231 + N.addresses.fetch({success:
  232 + function(collection, response) {
  233 + N.workspecs.fetch();
  234 + }
  235 + });
230 236 }
4 zivimap/templates/base.html
@@ -13,7 +13,7 @@
13 13 rel="stylesheet" media="screen">
14 14 <link href="http://code.jquery.com/ui/1.10.0/themes/base/jquery-ui.css"
15 15 rel="stylesheet" media="screen">
16   - <title>myzivi.ch | {% trans 'swiss civilian service search' %}</title>
  16 + <title>myzivi.ch | {% trans '(non-official) swiss civilian service search' %}</title>
17 17 </head>
18 18 <body>
19 19 <div id="top_navbar" class="navbar navbar-inverse navbar-static-top">
@@ -25,7 +25,7 @@
25 25 <span class="icon-bar"></span>
26 26 </a>
27 27 <a class="brand" href="#">myzivi.ch <br />
28   - <span class="nav-small">{% trans 'swiss civilian service search' %}</span></a>
  28 + <span class="nav-small">{% trans '(non-official) swiss civilian service search' %}</span></a>
29 29 <div class="nav-collapse collapse">
30 30 <ul class="nav">
31 31 <!--<li class="active"><a href="#">Home</a></li>-->
29 zivimap/templates/index.html
@@ -15,26 +15,11 @@
15 15 </script>
16 16
17 17 <script type="text/javascript">
18   - // TODO: Do clean bootstraping http://backbonejs.org/#FAQ-bootstrap
19   - WORKSPEC_API = "{% url 'api_dispatch_list' api_name="v1" resource_name="workspec" %}";
20   - ADDRESS_API = "{% url 'api_dispatch_list' api_name="v1" resource_name="address" %}";
21   - SEARCH_API = "{% url 'api_dispatch_list' api_name="v1" resource_name="search" %}";
  18 + ADDRESS_API = "{{ address_url }}";
  19 + SEARCH_API = "{{ workspec_search_url }}";
22 20
23   - // Daterange picker
24   - var dateMin = Date.parse('{{daterange_min|safe}}');
25   - var dateMax = Date.parse('{{daterange_max|safe}}');
26   - $('.daterange').daterangepicker({
27   - minDate: dateMin,
28   - maxDate: dateMax,
29   - startDate: dateMin,
30   - endDate: dateMax,
31   - format: 'dd/MM/yyyy'});
32   - $('.daterange').data('daterangepicker').notify();
33   -
34   - var _workspecs = JSON.parse("{{ workspecs|escapejs }}");
35   - var _addresses = JSON.parse("{{ addresses|escapejs }}");
36 21 var App = {};
37   - $(initMap(_workspecs, _addresses, App));
  22 + $(initMap(App));
38 23 </script>
39 24
40 25 <!-- Social stuff -->
@@ -55,13 +40,11 @@
55 40 <div id="filters_overlay">
56 41 <div id="filters_container">
57 42 <h4>{% trans 'Search options' %}</h4>
58   - <form action="{% url 'zivimap.views.index' %}" method="GET">
59   - <ul id="search_form">
60   - {{ search_form.as_ul }}
61   - </ul>
  43 + <form id="search_form" action="{% url 'zivimap.views.index' %}" method="GET">
  44 + {% include "search_form.html" with form=search_form %}
  45 +
62 46 <input class="btn btn-primary" type="submit" value="{% trans 'Search' %}" />
63 47 </form>
64   -
65 48 <!-- Social buttons -->
66 49 <div class="fb-like" data-send="false" data-layout="button_count" data-width="450" data-show-faces="false" data-font="arial"></div>
67 50
32 zivimap/templates/search_form.html
... ... @@ -0,0 +1,32 @@
  1 +{% load i18n %}
  2 +{% comment %}
  3 +{{ form.as_ul }}
  4 +{% endcomment %}
  5 +
  6 +<ul id="search_form">
  7 + {{ form.non_field_errors }}
  8 + <li>
  9 + <label for="id_domains_0">{% trans 'Domains' %}</label>
  10 + <ul>
  11 + {{ form.domains }}
  12 + </ul>
  13 + </li>
  14 + <li>
  15 + <label for="id_languages_0">{% trans 'Languages' %}</label>
  16 + <ul>
  17 + {{ form.languages }}
  18 + </ul>
  19 + </li>
  20 + <li>
  21 + <label for="id_start_min_month">{% trans 'Earliest start' %}</label>
  22 + <ul>
  23 + {{ form.start_min }}
  24 + </ul>
  25 + </li>
  26 + <li>
  27 + <label for="id_end_max_month">{% trans 'Latest end' %}</label>
  28 + <ul>
  29 + {{ form.end_max }}
  30 + </ul>
  31 + </li>
  32 +</ul>
93 zivimap/views.py
... ... @@ -1,13 +1,17 @@
1 1 from django.shortcuts import render, redirect
  2 +from django.core.urlresolvers import reverse
2 3 from django.core import serializers
3 4 from django import forms
4 5 from zivimap.api import *
5 6 from zivimap.models import Address, WorkSpec, DateRange
6 7 from django.db.models import Q, Count
7 8 from django.conf import settings
  9 +from django.utils.http import urlencode
  10 +from django.utils.datastructures import MultiValueDict
8 11 from django.utils import translation
9 12 from django.utils.translation import ugettext_lazy as _
10 13 import json
  14 +from zivimap.forms import SelectMonthYearWidget
11 15 from datetime import datetime
12 16
13 17 def all_resources(request, resource, queryset):
@@ -27,40 +31,19 @@ class SearchForm(forms.Form):
27 31 choices=WorkSpec.LANG_CHOICES,
28 32 widget=forms.CheckboxSelectMultiple,
29 33 label=_('Languages'))
30   - date_range = forms.CharField(required=False,
31   - widget=forms.TextInput(attrs={'class':'daterange',
32   - #'readonly':'readonly'
33   - }),
34   - label=_('Date range'))
35   -
36   -def search_workspecs(search_form):
37   - """Return a filter dict based on SearchForm cleaned_data"""
38   - assert search_form.is_valid()
39   - cd = search_form.cleaned_data
40   - Qfilters = Q()
41   - domains = cd['domains']
42   - if len(domains) > 0:
43   - Qfilters &= Q(activity_domain__in=domains)
44   - languages = cd['languages']
45   - if len(languages) > 0:
46   - Qfilters &= Q(language__in=languages)
47   - date_range = cd['date_range']
48   - if len(date_range) > 0:
49   - start_date, end_date = date_range.split('-')
50   - start_date = datetime.strptime(start_date.strip(), '%d/%m/%Y').date()
51   - end_date = datetime.strptime(end_date.strip(), '%d/%m/%Y').date()
52   - ranges = DateRange.objects.filter(start__gte=start_date,
53   - end__lte=end_date)
54   - ranges = ranges.values('workspec_id').annotate()
55   -
56   - # ID of workspecs that don't have a daterange
57   - no_ranges = WorkSpec.objects.annotate(rc=Count('daterange')).filter(rc=0).values('phid').annotate()
58   - Qfilters &= Q(phid__in=ranges) | Q(phid__in=no_ranges)
59   -
60   - # Need an order by here, otherwise, the clustering will change based
61   - # on the ordering of the WorkSpec set, which might be confusing (like
62   - # when searching without date range and with the whole date range)
63   - return WorkSpec.objects.filter(Qfilters).order_by('phid')
  34 + def __init__(self, date_min, date_max, *args, **kwargs):
  35 + super(SearchForm, self).__init__(*args, **kwargs)
  36 + years = range(date_min.year, date_max.year + 1)
  37 + self.fields.update({
  38 + 'start_min': forms.DateField(required=False,
  39 + input_formats=['%d.%m.%Y'],
  40 + widget=SelectMonthYearWidget(date_min, day='first', years=years),
  41 + label=_('Earliest start')),
  42 + 'end_max': forms.DateField(required=False,
  43 + input_formats=['%d.%m.%Y'],
  44 + widget=SelectMonthYearWidget(date_max, day='last', years=years),
  45 + label=_('Latest end'))
  46 + })
64 47
65 48 def sitemap(request):
66 49 return render(request, 'sitemap.xml')
@@ -70,22 +53,40 @@ def index(request):
70 53 lang = translation.get_language()
71 54 request.session['django_language'] = lang
72 55
73   - form = SearchForm(request.GET)
  56 + date_min, date_max = DateRange.get_min_max()
  57 + form = SearchForm(date_min, date_max, request.GET)
74 58 if form.is_valid():
75 59 cd = form.cleaned_data
76   - addresses = all_resources(request, AddressResource(),
77   - Address.objects.all())
78   - wsq = search_workspecs(form)
79   - ws = all_resources(request, MapSearchResource(), wsq)
  60 +
80 61 first_time_message = not request.session.get('visited', False)
81 62 request.session['visited'] = True
82   - date_min, date_max = DateRange.get_min_max()
83   - context = {'addresses': addresses,
84   - 'workspecs' : ws,
85   - 'search_form' : form,
86   - 'daterange_min' : date_min,
87   - 'daterange_max' : date_max,
88   - 'first_time_message' : first_time_message}
  63 +
  64 + workspec_search_url = reverse('api_dispatch_list', kwargs={
  65 + 'api_name': 'v1',
  66 + 'resource_name': 'workspec_search'})
  67 + query = MultiValueDict()
  68 + query['format'] = 'json'
  69 + for lang in cd['languages']:
  70 + query.appendlist('language__in', lang)
  71 + for dom in cd['domains']:
  72 + query.appendlist('activity_domain__in', dom)
  73 + start_min = cd['start_min']
  74 + end_max = cd['end_max']
  75 + if start_min:
  76 + query['start_min'] = start_min
  77 + if end_max:
  78 + query['end_max'] = end_max
  79 +
  80 + workspec_search_url += '?' + urlencode(query, doseq=True)
  81 + address_url = reverse('api_dispatch_list', kwargs={
  82 + 'api_name': 'v1',
  83 + 'resource_name': 'address'})
  84 +
  85 + context = {'search_form' : form,
  86 + 'first_time_message' : first_time_message,
  87 + 'workspec_search_url' : workspec_search_url,
  88 + 'address_url': address_url}
  89 +
89 90 return render(request, 'index.html', context)
90 91 else:
91 92 return redirect('index')
9 ziviscrap/settings.py
@@ -31,17 +31,18 @@
31 31 LOG_FILE = os.path.join(DATA_DIR, 'log.txt')
32 32 LOG_LEVEL = 'DEBUG'
33 33
34   -DOWNLOAD_DELAY = 0.1
  34 +DOWNLOAD_DELAY = 0.5
35 35
36 36 RETRY_ENABLED = True
37 37 RETRY_TIMES = 5
38   -
39   -# Disable redirect because the website redirects us on 500 error
40   -REDIRECT_ENABLED = False
41 38 # Retry 500 and 404 because sometimes, temporary errors seem to cause the
42 39 # server to return 404, but the page exists on next try
43 40 RETRY_HTTP_CODES = [500, 503, 504, 400, 404, 408, 300, 307]
44 41
  42 +
  43 +# Disable redirect because the website redirects us on 500 error
  44 +REDIRECT_ENABLED = False
  45 +
45 46 # Crawl responsibly by identifying yourself (and your website) on the user-agent
46 47 #USER_AGENT = 'ziviscrap (+http://www.yourdomain.com)'
47 48 USER_AGENT = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.16) Gecko/20120421 Firefox/11.0"
2  ziviweb/settings.py
@@ -162,7 +162,7 @@
162 162 'disable_existing_loggers': False,
163 163 'root': {
164 164 'level': 'WARNING',
165   - 'handlers': ['sentry', 'logfile'],
  165 + 'handlers': ['sentry', 'logfile', 'console'],
166 166 },
167 167 'filters': {
168 168 'require_debug_false': {
2  ziviweb/urls.py
@@ -11,7 +11,7 @@
11 11 v1_api = Api(api_name=settings.API_VERSION)
12 12 v1_api.register(AddressResource())
13 13 v1_api.register(WorkSpecResource())
14   -v1_api.register(MapSearchResource())
  14 +v1_api.register(WorkSpecSearchResource())
15 15
16 16 print v1_api.urls
17 17

No commit comments for this range

Something went wrong with that request. Please try again.