Add a monitor to watch for census county changes

RMR staffers who update county-level mortgage data need to know when a
county changes name or FIPS code, as sometimes happens. This monitor
compares the posted census change log against a local base change log.
If a change is detected, it sends emails with the change and updates the
base change log so it can` watch for the next change.

The monitor can be called with the management command `oah_check_county_changes [--email]`
The `--email` option will accept one or more email recipients,
separated by spaces, who will be notified.

countylimits/data_collection/changelog_2017.html

@@ -0,0 +1,42 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/data_collection/county_data_monitor.py

@@ -0,0 +1,62 @@
+import os
+
+from difflib import ndiff
+
+import requests
+from bs4 import BeautifulSoup as bs
+from django.core.mail import send_mail
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+CENSUS_CHANGELOG = 'https://www.census.gov/geo/reference/county-changes.html'
+LAST_CHANGELOG = '{}/last_changelog.html'.format(BASE_DIR)
+CHANGELOG_ID = 'tab_2010'
+
+changelog_response = requests.get(CENSUS_CHANGELOG)
+soup = bs(changelog_response.text, 'lxml')
+current_changelog = soup.find("div", {"id": CHANGELOG_ID}).text
+
+with open(LAST_CHANGELOG, 'r') as f:
+    base_changelog = f.read()
+
+
+def get_lines(changelog):
+    return [line.strip() for line in changelog.split('\n') if line]
+
+
+def check_for_county_changes(email=None):
+    """
+    Check the census county changelog against a local copy of the last log
+    to see whether updates have been added. If changes are detected,
+    note the change and update our local 'last_changelog.html' file.
+    """
+    base_lines = get_lines(base_changelog)
+    current_lines = get_lines(current_changelog)
+    if base_lines == current_lines:
+        msg = 'No county changes found, no emails sent.'
+        return msg
+    else:
+        msg = ('County changes need to be checked at {}\n'
+               'These changes were detected:'.format(CENSUS_CHANGELOG))
+        with open(LAST_CHANGELOG, 'w') as f:
+            f.write(current_changelog)
+        diffsets = []
+        diffset = ndiff(base_lines, current_lines)
+        diffsets.append(
+            d for d in diffset if d.startswith('- ') or d.startswith('+ '))
+        for diffsett in diffsets:
+            for diff in diffsett:
+                msg += '\n{}'.format(diff)
+        msg += "\n\nOur 'last_changelog.html' file has been updated."
+        if email:
+            send_mail(
+                'Owning a Home alert: Change detected in census county data',
+                msg,
+                'tech@cfpb.gov',
+                email,
+                fail_silently=False
+            )
+
+        return (
+            "Emails were sent to {} with the following message: \n\n"
+            "{}".format(", ".join(email), msg)
+        )

countylimits/data_collection/last_changelog.html

@@ -0,0 +1,42 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/data_collection/test_changelog_with_diff.html

@@ -0,0 +1,44 @@
+
+2010
+Changes to Counties or County Equivalent Entities: 2010s 
+New Counties or County Equivalent Entities 
+
+Petersburg Borough, Alaska (02-195): 
+Created from part of former Petersburg Census Area (02-195) and part of Hoonah-Angoon Census Area (02-105) effective January 3, 2013; estimated population 3,203.
+
+
+Deleted Counties or County Equivalent Entities 
+
+Bedford (independent) city, Virginia (51-515): 
+Changed to town status and added to Bedford County (51-019) effective July 1, 2013.
+
+
+
+Name and/or Code Changes or Corrections for Counties or County Equivalent Entities 
+
+Kusilvak Census Area, Alaska (02-158)
+ Changed name and code from Wade Hampton Census Area (02-270) effective July 1, 2015.
+Wade Hampton Census Area, Alaska (02-270)
+  Changed name and code to Kusilvak Census Area (02-158) effective July 1, 2015.
+LaSalle Parish, Louisiana (22-059)
+Name corrected from La Salle Parish (removing space) reported as of January 1, 2011.
+Oglala Lakota County, South Dakota (46-102)
+  Changed name and code from Shannon County (46-113) effective May 1, 2015.
+Shannon County, South Dakota (46-113)
+   Changed name and code to Oglala Lakota County (46-102) effective May 1, 2015.
+Reyes County, Kansas (46-102)
+  Changed name from Reno County effective April 1, 2019.
+
+
+Substantial County or County Equivalent Boundary Changes 
+
+Hoonah-Angoon Census Area, Alaska (02-105):
+Part taken to create new Petersburg Borough (02-195) effective January 3, 2013; estimated detached population: 1
+
+Prince of Wales-Hyder Census Area, Alaska (02-198):
+Prince of Wales-Hyder Census Area (02-198) added part of the former Petersburg Census Area (02-195) effective January 3, 2013; estimated added population 613.
+
+Bedford County, Virginia (51-019):
+Added the former independent city of Bedford (51-515) effective July 1, 2013; estimated net added population 6,222.
+
+

countylimits/management/commands/oah_check_county_changes.py

@@ -0,0 +1,27 @@
+from django.core.management.base import BaseCommand
+from countylimits.data_collection.county_data_monitor import (
+    check_for_county_changes)
+
+COMMAND_HELP = "Check the census county changelog against a local copy "
+"of the last log to see whether updates have been added. "
+"If changes are detected, send an email alert about the change "
+"and update our local 'last_changelog.html' file."
+PARSER_HELP = "This command accepts a space-separated string "
+"of email recipients who will be notified if county changes are detected."
+
+
+class Command(BaseCommand):
+    help = COMMAND_HELP
+
+    def add_arguments(self, parser):
+        parser.add_argument('--email',
+                            help=PARSER_HELP,
+                            nargs='+',
+                            type=str)
+
+    def handle(self, *args, **options):
+        if options['email']:
+            msg = check_for_county_changes(email=options['email'])
+        else:
+            msg = check_for_county_changes()
+        self.stdout.write(msg)

requirements/base.txt

@@ -2,3 +2,6 @@ Django==1.8.15
 djangorestframework==3.1.3
 django-localflavor
 django-cors-headers
+beautifulsoup4==4.5.3
+requests==2.12.4
+lxml==3.7.2