Merge pull request #360 from sleitner/rram-temp

Add year index for search MSAs and filter by year

frontend/src/js/metro-search.js

@@ -2,14 +2,15 @@
 
 $(document).ready(function() {
     var searchNameBox = $('#geoid'),
+        year = $('#year').val(),
         msaField = $('#msa-field'),
         search = new Bloodhound({
             datumTokenizer: function(d) {
                 return Bloodhound.tokenizers.whitespace(d.name);
             },
             queryTokenizer: Bloodhound.tokenizers.whitespace,
             remote: {
-                url: '/shapes/search/?auto=1&q=%QUERY',
+                url: '/shapes/search/?auto=1&q=%QUERY&year=' + year,
                 filter: function(resp) {
                     return resp.geos;
                 }

mapusaurus/censusdata/management/commands/load_summary_one.py

@@ -12,15 +12,15 @@ class Command(BaseCommand):
     """Loads Summary File 1 data from the decennial census. Official
     documentation for fields at
     http://www.census.gov/prod/cen2010/doc/sf1.pdf"""
-    args = "<path/to/XXgeo2010.sf1>"
+    args = "<path/to/XXgeo2010.sf1> <year>"
     help = """
         Load Decennial Census data for a state.
         Assumes XX#####2010.sf1 files are in the same directory."""
 
     def handle(self, *args, **options):
-        if not args:
-            raise CommandError("Needs a first argument, "
-                               + "path/to/XXgeo2010.sf1")
+        if len(args) != 2:
+            raise CommandError("Needs 2 arguments, "
+                               + "path/to/XXgeo2010.sf1 year")
         geoids_by_record = {}
         geofile = open(args[0], 'r')
         year = args[1]

mapusaurus/geo/search_indexes.py

@@ -8,6 +8,7 @@ class MetroIndex(indexes.SearchIndex, indexes.Indexable):
     text = indexes.CharField(document=True, model_attr='name')
     text_auto = indexes.EdgeNgramField(model_attr='name')
     geo_type = indexes.IntegerField(model_attr='geo_type')
+    year = indexes.IntegerField(model_attr='year')
 
     def get_model(self):
         return Geo

mapusaurus/geo/tests.py

@@ -24,8 +24,9 @@ def test_search_name(self, SQS):
         result.object.name = 'MSA 1'
         result.object.centlat = 45
         result.object.centlon = 52
-        SQS.filter.return_value = [result]
-        resp = self.client.get(reverse('geo:search'), {'q': 'Chicago'})
+        result.object.year = 2013
+        SQS.filter.return_value.filter.return_value = [result]
+        resp = self.client.get(reverse('geo:search'), {'q': 'Chicago', 'year': '2013'})
         self.assertTrue('Chicago' in str(SQS.filter.call_args))
         self.assertTrue('content' in str(SQS.filter.call_args))
         self.assertFalse('text_auto' in str(SQS.filter.call_args))
@@ -41,11 +42,12 @@ def test_search_name(self, SQS):
     @patch('geo.views.SearchQuerySet')
     def test_search_autocomplete(self, SQS):
         SQS = SQS.return_value.models.return_value.load_all.return_value
-        SQS.filter.return_value = [Mock()]
-        self.client.get(reverse('geo:search'), {'q': 'Chicago', 'auto': '1'})
+        SQS.filter.return_value.filter.return_value = [Mock()]
+        self.client.get(reverse('geo:search'), {'q': 'Chicago', 'auto': '1', 'year': '2013'})
         self.assertTrue('Chicago' in str(SQS.filter.call_args))
         self.assertFalse('content' in str(SQS.filter.call_args))
         self.assertTrue('text_auto' in str(SQS.filter.call_args))
+        self.assertTrue('year' in str(SQS.filter.return_value.filter.call_args))
 
 class UtilsTest(TestCase):
     def test_check_bounds(self):

mapusaurus/geo/views.py

@@ -68,18 +68,19 @@ class GeoSerializer(serializers.ModelSerializer):
     """Used in RESTful endpoints to serialize Geo objects; used in search"""
     class Meta:
         model = Geo
-        fields = ('geoid', 'geo_type', 'name', 'centlat', 'centlon')
+        fields = ('geoid', 'geo_type', 'name', 'centlat', 'centlon', 'year')
 
 
 @api_view(['GET'])
 @renderer_classes((JSONRenderer, ))     # until we need HTML
 def search(request):
     query_str = request.GET.get('q', '').strip()
+    year= request.GET.get('year', '').strip()
     query = SearchQuerySet().models(Geo).load_all()
     if request.GET.get('auto'):
-        query = query.filter(text_auto=AutoQuery(query_str))
+        query = query.filter(text_auto=AutoQuery(query_str)).filter(year=year)
     else:
-        query = query.filter(content=AutoQuery(query_str))
+        query = query.filter(content=AutoQuery(query_str)).filter(year=year)
     query = query[:25]
     results = [result.object for result in query]
     results = GeoSerializer(results, many=True).data

mapusaurus/respondents/search_indexes.py

@@ -12,6 +12,7 @@ class InstitutionIndex(indexes.SearchIndex, indexes.Indexable):
     lender_id = indexes.CharField()
     assets = indexes.IntegerField(model_attr='assets')
     num_loans = indexes.IntegerField(model_attr='num_loans')
+    year = indexes.IntegerField(model_attr='year')
 
     def get_model(self):
         return Institution

mapusaurus/respondents/templates/respondents/metro_search.html

@@ -33,8 +33,8 @@ <h3>Find a location to create a HMDA Visualization Map</h3>
             <div class="search-field">
                 <label for="geoid">Select an MSA:</label><br/>
                 <input type="text" id="geoid"><br/><br/>
-                <input type="hidden" name="lender" value="{{institution.agency_id}}{{institution.respondent_id}}">
-                <input type="hidden" name="year" value="{{year}}">
+                <input type="hidden" name="lender" value="{{year}}{{institution.agency_id}}{{institution.respondent_id}}">
+                <input type="hidden" id="year" name="year" value="{{year}}">
                 <input type="hidden" id="msa-field" name="metro">
                 <button type="submit" id="metro-search__submit" class="btn btn__disabled" disabled>Select</button>
 

mapusaurus/respondents/templates/respondents/partial/search_results_details.html

@@ -9,7 +9,7 @@
                 <div class="search-result-record__left-col">
                     <h4 class="search-result-record__title"><a href="{% url 'respondents:select_metro' res.year res.agency_id res.respondent_id %}">{{res.name}}</a></h4>
                     <div class="search-result-record__cfpb-id">
-                        <a href="{% url 'respondents:respondent_profile' res.agency_id res.respondent_id %}">{{res.agency_id}}{{res.respondent_id}}</a>
+                        <a href="{% url 'respondents:respondent_profile' res.year res.agency_id res.respondent_id %}">{{res.year}}{{res.agency_id}}{{res.respondent_id}}</a>
                     </div>
                     <div class="search-result-record__hq">
                         <small>HQ:</small> {{res.zip_code.city|title}}, {{res.zip_code.state|upper}}

mapusaurus/respondents/tests.py

@@ -130,7 +130,7 @@ def test_select_metro(self):
 
         results = self.client.get(
             reverse('respondents:select_metro',
-                    kwargs={'agency_id': '9', 'respondent': '9879879870', 'year': 2013}))
+                    kwargs={'agency_id': '9', 'respondent': '9879879870', 'year': 1234}))
         self.assertEqual(200, results.status_code)
 
         inst.delete()
@@ -142,10 +142,10 @@ def test_search_empty(self, SQS):
         self.client.get(reverse('respondents:search_results'))
         self.assertFalse(SQS.filter.called)
 
-        self.client.get(reverse('respondents:search_results'), {'q': ''})
+        self.client.get(reverse('respondents:search_results'), {'q': '', 'year': '2013'})
         self.assertFalse(SQS.filter.called)
 
-        self.client.get(reverse('respondents:search_results'), {'q': '     '})
+        self.client.get(reverse('respondents:search_results'), {'q': '     ', 'year': ''})
         self.assertFalse(SQS.filter.called)
 
     @patch('respondents.views.SearchQuerySet')
@@ -212,14 +212,14 @@ def test_search_id(self, SQS):
         for q in ['01123456799',
                   'Some Bank (01123456799)']:
             resp = self.client.get(reverse('respondents:search_results'),
-                                   {'q': q})
+                                   {'q': q, 'year': '2013'})
             self.assertTrue('01123456799' in str(SQS.filter.call_args))
             self.assertTrue('lender_id' in str(SQS.filter.call_args))
             self.assertTrue('Some Bank' in resp.content)
             self.assertRaises(ValueError, json.loads, resp.content)
 
         resp = self.client.get(reverse('respondents:search_results'),
-                               {'q': 'Some Bank (0112345799)'})
+                               {'q': 'Some Bank (0112345799)', 'year': '2013'})
         self.assertTrue('0112345799' in str(SQS.filter.call_args))
         self.assertFalse('lender_id' in str(SQS.filter.call_args))
         self.assertTrue('Some Bank' in resp.content)
@@ -235,7 +235,7 @@ def test_search_json(self, SQS):
         result.object = Institution(name='Some Bank')
 
         resp = self.client.get(reverse('respondents:search_results'),
-                               {'q': 'Bank'},
+                               {'q': 'Bank', 'year': '2013'},
                                HTTP_ACCEPT='application/json')
 
         resp = json.loads(resp.content)
@@ -251,7 +251,7 @@ def test_search_num_loans(self, SQS):
         result.num_loans = 45
         result.object = Institution(name='Some Bank')
 
-        request = RequestFactory().get('/', data={'q': 'Bank'})
+        request = RequestFactory().get('/', data={'q': 'Bank', 'year': '2013'})
         results = views.search_results(request)
         self.assertEqual(len(results.data['institutions']), 1)
         self.assertEqual(45, results.data['institutions'][0].num_loans)
@@ -266,49 +266,49 @@ def test_search_sort(self, SQS):
         self.assertTrue(load_all.order_by.called)
 
         request = RequestFactory().get('/', data={'q': 'Bank',
-                                                  'sort': 'another-sort'})
+                                                  'sort': 'another-sort', 'year': '2013'})
         views.search_results(request)
         self.assertTrue(load_all.order_by.called)
 
         for sort in ('assets', '-assets', 'num_loans', '-num_loans'):
             request = RequestFactory().get('/', data={'q': 'Bank',
-                                                      'sort': sort})
+                                                      'sort': sort, 'year': '2013'})
             views.search_results(request)
             self.assertTrue(load_all.order_by.called)
 
     @patch('respondents.views.SearchQuerySet')
     def test_search_pagination(self, SQS):
-        request = RequestFactory().get('/', data={'q': 'Bank'})
+        request = RequestFactory().get('/', data={'q': 'Bank', 'year': '2013'})
         results = views.search_results(request)
         # page number should default to 1
         self.assertEqual(results.data['page_num'], 1)
 
         request = RequestFactory().get('/', data={'q': 'Bank',
-                                                  'page': 3})
+                                                  'page': 3, 'year': '2013'})
         results = views.search_results(request)
         self.assertEqual(results.data['page_num'], 3)
         self.assertEqual(results.data['next_page'], 0)
         self.assertEqual(results.data['prev_page'], 2)
 
         request = RequestFactory().get('/', data={'q': 'Bank',
-                                                  'page': 'str'})
+                                                  'page': 'str', 'year': '2013'})
         results = views.search_results(request)
         self.assertEqual(results.data['page_num'], 1)
 
     @patch('respondents.views.SearchQuerySet')
     def test_search_num_results(self, SQS):
-        request = RequestFactory().get('/', data={'q': 'Bank'})
+        request = RequestFactory().get('/', data={'q': 'Bank', 'year': '2013'})
         results = views.search_results(request)
         # number of results should default to 25
         self.assertEqual(results.data['num_results'], 25)
 
         request = RequestFactory().get('/', data={'q': 'Bank',
-                                                  'num_results': 10})
+                                                  'num_results': 10, 'year': '2013'})
         results = views.search_results(request)
         self.assertEqual(results.data['num_results'], 10)
 
         request = RequestFactory().get('/', data={'q': 'Bank',
-                                                  'num_results': 'str'})
+                                                  'num_results': 'str', 'year': '2013'})
         results = views.search_results(request)
         self.assertEqual(results.data['num_results'], 25)
 

mapusaurus/respondents/urls.py

@@ -10,7 +10,7 @@
     url(r'^search/$', 'respondents.views.search_results',
         name='search_results'),
     url(r'^$', 'respondents.views.search_home', name='search_home'),
-	url(r'^respondent/(?P<agency_id>[0-9])(?P<respondent>[0-9-]{10})',
+	url(r'^respondent/(?P<year>[0-9]{4})(?P<agency_id>[0-9])(?P<respondent>[0-9-]{10})',
 		'respondents.views.respondent', name='respondent_profile'),
     url(r'^branchLocations/(?P<northEastLat>-?\d+\.\d{6})/(?P<northEastLon>-?\d+\.\d{6})/(?P<southWestLat>-?\d+\.\d{6})/(?P<southWestLon>-?\d+\.\d{6})$',
         'respondents.views.branch_locations_as_json', name='branch_locations'),

mapusaurus/respondents/views.py

@@ -50,7 +50,7 @@ def search_home(request):
 def select_metro(request, year, agency_id, respondent):
     """Once an institution is selected, search for a metro"""
     institution = get_object_or_404(Institution, respondent_id=respondent,
-                                    agency_id=int(agency_id))
+                                    agency_id=int(agency_id), year=year)
     return render(request, 'respondents/metro_search.html', {
         'institution': institution,
         'year': year
@@ -78,30 +78,35 @@ class Meta:
 def search_results(request):
     query_str = escape(request.GET.get('q', '')).strip()
     year = escape(request.GET.get('year', '')).strip()
+    if not year:
+        year = 2013 #snl temporary
+
     lender_id = False
     respondent_id = False
     for regex in LENDER_REGEXES:
         match = regex.match(query_str)
         if match:
-            lender_id = match.group('agency') + match.group('respondent')
+            lender_id = year + match.group('agency') + match.group('respondent')
     resp_only_match = RESP_RE.match(query_str)
     if resp_only_match:
         respondent_id = resp_only_match.group('respondent')
 
+    query = SearchQuerySet().models(Institution).load_all() # snl temporary
+
     current_sort = request.GET.get('sort')
     if current_sort == None:
         current_sort = '-assets'
 
     query = SearchQuerySet().models(Institution).load_all().order_by(current_sort)
 
     if lender_id:
-        query = query.filter(lender_id=Exact(lender_id))
+        query = query.filter(lender_id=Exact(lender_id),year=year)
     elif respondent_id:
-        query = query.filter(respondent_id=Exact(respondent_id))
-    elif query_str and escape(request.GET.get('auto')):
-        query = query.filter(text_auto=AutoQuery(query_str))
+        query = query.filter(respondent_id=Exact(respondent_id),year=year)
+    elif query_str and escape(request.GET.get('auto')): # snl temporary: escape creates a bug where None = True
+        query = query.filter(text_auto=AutoQuery(query_str),year=year)
     elif query_str:
-        query = query.filter(content=AutoQuery(query_str))
+        query = query.filter(content=AutoQuery(query_str), year=year)
     else:
         query = []