Merge pull request #260 from openstates/unmatched-names

Unmatched names

scripts/to_database.py

@@ -3,8 +3,6 @@
 import sys
 import glob
 from functools import lru_cache
-import django
-from django import conf
 from django.db import transaction
 import click
 import openstates_metadata as metadata
@@ -14,6 +12,7 @@
     get_all_abbreviations,
     load_yaml,
     legacy_districts,
+    init_django,
 )
 
 
@@ -381,30 +380,6 @@ def load_directory(files, type, jurisdiction_id, purge):
     )
 
 
-def init_django():  # pragma: no cover
-    conf.settings.configure(
-        conf.global_settings,
-        SECRET_KEY="not-important",
-        DEBUG=False,
-        INSTALLED_APPS=(
-            "django.contrib.contenttypes",
-            "opencivicdata.core.apps.BaseConfig",
-            "opencivicdata.legislative.apps.BaseConfig",
-        ),
-        DATABASES={
-            "default": {
-                "ENGINE": "django.contrib.gis.db.backends.postgis",
-                "NAME": os.environ["PGDATABASE"],
-                "USER": os.environ["PGUSER"],
-                "PASSWORD": os.environ["PGPASSWORD"],
-                "HOST": os.environ["PGHOST"],
-            }
-        },
-        MIDDLEWARE_CLASSES=(),
-    )
-    django.setup()
-
-
 def create_parties():
     from opencivicdata.core.models import Organization
 

scripts/unmatched_legislators.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+"""Script to create CSVs of unmatched legislators given a state and session"""
+import csv
+from collections import Counter, defaultdict
+from utils import get_jurisdiction_id, init_django, get_all_abbreviations
+import click
+
+
+def archive_leg_to_csv(state_abbr=None):
+    from opencivicdata.legislative.models import PersonVote, BillSponsorship
+    from django.db.models import Count, F
+
+    output_filename = f"unmatched_{state_abbr}.csv"
+
+    jurisdiction_id = get_jurisdiction_id(state_abbr)
+
+    # name -> session -> count
+    missing_votes = Counter()
+    missing_sponsors = Counter()
+    sessions_for_name = defaultdict(set)
+
+    voters = (
+        PersonVote.objects.filter(
+            vote_event__legislative_session__jurisdiction_id=jurisdiction_id, voter_id=None,
+        )
+        .values("voter_name", session=F("vote_event__legislative_session__identifier"))
+        .annotate(n=Count("id"))
+    )
+    for voter in voters:
+        missing_votes[voter["voter_name"]] += voter["n"]
+        sessions_for_name[voter["voter_name"]].add(voter["session"])
+
+    bill_sponsors = (
+        BillSponsorship.objects.filter(
+            bill__legislative_session__jurisdiction_id=jurisdiction_id,
+            person_id=None,
+            organization_id=None,
+        )
+        .values("name", session=F("bill__legislative_session__identifier"))
+        .annotate(n=Count("id"))
+    )
+    for bill_sponsor in bill_sponsors:
+        missing_sponsors[bill_sponsor["name"]] += 1
+        sessions_for_name[bill_sponsor["name"]].add(bill_sponsor["session"])
+
+    all_names = sorted(sessions_for_name.keys())
+
+    with open(output_filename, "w") as outf:
+        out = csv.DictWriter(outf, ("name", "jurisdiction", "sessions", "votes", "sponsorships"))
+        out.writeheader()
+        for name in all_names:
+            obj = {
+                "name": name,
+                "jurisdiction": state_abbr,
+                "sessions": "; ".join(sorted(sessions_for_name[name])),
+                "votes": missing_votes[name],
+                "sponsorships": missing_sponsors[name],
+            }
+            out.writerow(obj)
+
+
+@click.command()
+@click.argument("abbreviations", nargs=-1)
+def export_unmatched(abbreviations=None):
+    if not abbreviations:
+        abbreviations = get_all_abbreviations()
+
+    for abbr in abbreviations:
+        archive_leg_to_csv(abbr)
+
+
+if __name__ == "__main__":
+    init_django()
+    export_unmatched()

scripts/utils.py

@@ -5,6 +5,8 @@
 import datetime
 import yaml
 import yamlordereddictloader
+import django
+from django import conf
 from collections import defaultdict
 from yaml.representer import Representer
 import openstates_metadata as metadata
@@ -101,3 +103,27 @@ def legacy_districts(**kwargs):
     for d in metadata.lookup(**kwargs).legacy_districts:
         legacy_districts[d.chamber_type].append(d.name)
     return legacy_districts
+
+
+def init_django():  # pragma: no cover
+    conf.settings.configure(
+        conf.global_settings,
+        SECRET_KEY="not-important",
+        DEBUG=False,
+        INSTALLED_APPS=(
+            "django.contrib.contenttypes",
+            "opencivicdata.core.apps.BaseConfig",
+            "opencivicdata.legislative.apps.BaseConfig",
+        ),
+        DATABASES={
+            "default": {
+                "ENGINE": "django.contrib.gis.db.backends.postgis",
+                "NAME": os.environ["PGDATABASE"],
+                "USER": os.environ["PGUSER"],
+                "PASSWORD": os.environ["PGPASSWORD"],
+                "HOST": os.environ["PGHOST"],
+            }
+        },
+        MIDDLEWARE_CLASSES=(),
+    )
+    django.setup()