Merge 96ed34f into c72513b

CHANGELOG.md

@@ -1,6 +1,9 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## Unreleased
+- Add a monitor to watch for changes in census county values
+
 ## 0.9.92 - 2016-12-31
 - 2017 update for county-level mortgage-limit data 
 

countylimits/data_collection/changelog_2017.html

@@ -0,0 +1,42 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/data_collection/county_data_monitor.py

@@ -0,0 +1,74 @@
+import os
+
+from difflib import ndiff
+
+import requests
+from bs4 import BeautifulSoup as bs
+from django.core.mail import send_mail
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+CENSUS_CHANGELOG = 'https://www.census.gov/geo/reference/county-changes.html'
+LAST_CHANGELOG = '{}/last_changelog.html'.format(BASE_DIR)
+CHANGELOG_ID = 'tab_2010'
+
+
+def get_current_log():
+    changelog_response = requests.get(CENSUS_CHANGELOG)
+    soup = bs(changelog_response.text, 'lxml')
+    return soup.find("div", {"id": CHANGELOG_ID}).text
+
+
+def get_base_log():
+    with open(LAST_CHANGELOG, 'r') as f:
+        base_log = f.read()
+        return base_log
+
+
+def store_change_log(newlog):
+    with open(LAST_CHANGELOG, 'w') as f:
+        f.write(newlog)
+
+
+def get_lines(changelog):
+    return [line.strip() for line in changelog.split('\n') if line]
+
+
+def check_for_county_changes(email=None):
+    """
+    Check the census county changelog against a local copy of the last log
+    to see whether updates have been added. If changes are detected,
+    note the change and update our local 'last_changelog.html' file.
+    """
+    current_changelog = get_current_log()
+    current_lines = get_lines(current_changelog)
+    base_lines = get_lines(get_base_log())
+    if base_lines == current_lines:
+        msg = 'No county changes found, no emails sent.'
+        return msg
+    else:
+        msg = ('County changes need to be checked at {}\n'
+               'These changes were detected:'.format(CENSUS_CHANGELOG))
+        diffsets = []
+        diffset = ndiff(base_lines, current_lines)
+        diffsets.append(
+            d for d in diffset if d.startswith('- ') or d.startswith('+ '))
+        for diffsett in diffsets:
+            for diff in diffsett:
+                msg += '\n{}'.format(diff)
+        store_change_log(current_changelog)
+        msg += "\n\nOur 'last_changelog.html' file has been updated."
+        if email:
+            send_mail(
+                'Owning a Home alert: Change detected in census county data',
+                msg,
+                'tech@cfpb.gov',
+                email,
+                fail_silently=False
+            )
+
+            return (
+                "Emails were sent to {} with the following message: \n\n"
+                "{}".format(", ".join(email), msg)
+            )
+        else:
+            return msg

countylimits/data_collection/last_changelog.html

@@ -0,0 +1,42 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/data_collection/test_changelog_with_diff.html

@@ -0,0 +1,44 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+Reyes County, Kansas (46-102)
+  Changed name from Reno County effective April 1, 2019.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/management/commands/oah_check_county_changes.py

@@ -0,0 +1,27 @@
+from django.core.management.base import BaseCommand
+from countylimits.data_collection.county_data_monitor import (
+    check_for_county_changes)
+
+COMMAND_HELP = "Check the census county changelog against a local copy "
+"of the last log to see whether updates have been added. "
+"If changes are detected, send an email alert about the change "
+"and update our local 'last_changelog.html' file."
+PARSER_HELP = "This command accepts a space-separated string "
+"of email recipients who will be notified if county changes are detected."
+
+
+class Command(BaseCommand):
+    help = COMMAND_HELP
+
+    def add_arguments(self, parser):
+        parser.add_argument('--email',
+                            help=PARSER_HELP,
+                            nargs='+',
+                            type=str)
+
+    def handle(self, *args, **options):
+        if options['email']:
+            msg = check_for_county_changes(email=options['email'])
+        else:
+            msg = check_for_county_changes()
+        self.stdout.write(msg)

countylimits/tests.py

@@ -1,19 +1,138 @@
 import os
+import unittest
+
+import mock
+from mock import mock_open, patch
 from rest_framework import status
+
 from django.test import TestCase
 from django.utils.six import StringIO
+from django.core.management import call_command
+from django.core.management.base import CommandError
 
 from countylimits.models import CountyLimit, County, State
-from countylimits.management.commands.load_county_limits import Command
-from django.core.management.base import CommandError
+from countylimits.management.commands import load_county_limits
+from countylimits.data_collection.county_data_monitor import (
+    check_for_county_changes,
+    store_change_log,
+    get_current_log,
+    get_base_log,
+    get_lines)
+
 
 try:
     BASE_PATH = os.path.dirname(
         os.path.dirname(os.path.abspath(__file__))) + '/'
-except:
+except:  # pragma: no cover
     BASE_PATH = ''
 
 
+class CheckCountyChangesCommand(unittest.TestCase):
+
+    def setUp(self):
+        stdout_patch = mock.patch('sys.stdout')
+        stdout_patch.start()
+        self.addCleanup(stdout_patch.stop)
+
+    @mock.patch(
+        'countylimits.management.commands.oah_check_county_changes.'
+        'check_for_county_changes')
+    def test_check_county_without_email(self, mock_check):
+        mock_check.return_value = 'OK'
+        call_command('oah_check_county_changes')
+        self.assertEqual(mock_check.call_count, 1)
+
+    @mock.patch(
+        'countylimits.management.commands.oah_check_county_changes.'
+        'check_for_county_changes')
+    def test_check_county_with_email(self, mock_check):
+        mock_check.return_value = 'Emails were sent'
+        call_command('oah_check_county_changes', '--email', 'fake@example.com')
+        self.assertEqual(mock_check.call_count, 1)
+
+
+class DataCollectionTest(unittest.TestCase):
+    """Test data automation functions"""
+
+    def test_get_lines(self):
+        lines_in = "\n\nline 1\nline 2\n\n\nline 3\n\n"
+        expected_result = ['line 1', 'line 2', 'line 3']
+        lines_out = get_lines(lines_in)
+        self.assertEqual(lines_out, expected_result)
+
+    def test_get_base_log(self):
+        text = get_base_log()
+        self.assertIn('2010', text)
+
+    def test_store_changelog(self):
+        m = mock_open()
+        with patch("__builtin__.open", m, create=True):
+            store_change_log('fake log text')
+        self.assertTrue(m.call_count == 1)
+
+    @mock.patch('countylimits.data_collection.county_data_monitor'
+                '.requests.get')
+    @mock.patch('countylimits.data_collection.county_data_monitor.bs')
+    def test_get_current_log(self, mock_bs, mock_requests):
+        get_current_log()
+        self.assertEqual(mock_bs.call_count, 1)
+        self.assertEqual(mock_requests.call_count, 1)
+
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_current_log')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_base_log')
+    def test_county_data_monitor_no_change(self, mock_base, mock_current):
+        with open("{}/countylimits/data_collection/"
+                  "changelog_2017.html".format(BASE_PATH)) as f:
+            mock_base.return_value = mock_current.return_value = f.read()
+        self.assertIn(
+            'No county changes found',
+            check_for_county_changes())
+
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_current_log')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_base_log')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.store_change_log')
+    def test_county_data_monitor_with_change_no_email(
+            self, mock_store_log, mock_base, mock_current):
+        with open("{}/countylimits/data_collection/"
+                  "changelog_2017.html".format(BASE_PATH)) as f:
+            mock_base.return_value = f.read()
+            mock_current.return_value = (
+                mock_base.return_value + 'When dolphins fly.\n')
+        self.assertIn(
+            'When dolphins fly',
+            check_for_county_changes())
+        self.assertEqual(mock_current.call_count, 1)
+        self.assertEqual(mock_base.call_count, 1)
+        self.assertEqual(mock_store_log.call_count, 1)
+
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_current_log')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.get_base_log')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.send_mail')
+    @mock.patch(
+        'countylimits.data_collection.county_data_monitor.store_change_log')
+    def test_county_data_monitor_with_change_and_email(
+            self, mock_store_log, mock_mail, mock_base, mock_current):
+        with open("{}/countylimits/data_collection/"
+                  "changelog_2017.html".format(BASE_PATH)) as f:
+            mock_base.return_value = f.read()
+            mock_current.return_value = (
+                mock_base.return_value + 'When dolphins fly.\n')
+        msg = check_for_county_changes(email='fakemail@example.com')
+        self.assertIn('When dolphins fly', msg)
+        self.assertEqual(mock_mail.call_count, 1)
+        self.assertEqual(mock_current.call_count, 1)
+        self.assertEqual(mock_base.call_count, 1)
+        self.assertEqual(mock_store_log.call_count, 1)
+
+
 class CountyLimitTest(TestCase):
 
     url = '/oah-api/county/'
@@ -63,7 +182,7 @@ def test_unicode(self):
 
 class LoadCountyLimitsTestCase(TestCase):
 
-    c = Command()
+    c = load_county_limits.Command()
     out = StringIO()
 
     test_csv = '{}data/test/test.csv'.format(BASE_PATH)