Merge 6f32a55 into eec1360

openstates · Apr 10, 2019 · 5ef2fd2 · 5ef2fd2
2 parents eec1360 + 6f32a55
commit 5ef2fd2
Show file tree

Hide file tree

Showing 53 changed files with 7,723 additions and 0 deletions.
diff --git a/csv/ak_legislators.csv b/csv/ak_legislators.csv
diff --git a/csv/al_legislators.csv b/csv/al_legislators.csv
diff --git a/csv/ar_legislators.csv b/csv/ar_legislators.csv
diff --git a/csv/az_legislators.csv b/csv/az_legislators.csv
diff --git a/csv/ca_legislators.csv b/csv/ca_legislators.csv
diff --git a/csv/co_legislators.csv b/csv/co_legislators.csv
diff --git a/csv/ct_legislators.csv b/csv/ct_legislators.csv
diff --git a/csv/dc_legislators.csv b/csv/dc_legislators.csv
diff --git a/csv/de_legislators.csv b/csv/de_legislators.csv
diff --git a/csv/fl_legislators.csv b/csv/fl_legislators.csv
diff --git a/csv/ga_legislators.csv b/csv/ga_legislators.csv
diff --git a/csv/hi_legislators.csv b/csv/hi_legislators.csv
diff --git a/csv/ia_legislators.csv b/csv/ia_legislators.csv
diff --git a/csv/id_legislators.csv b/csv/id_legislators.csv
diff --git a/csv/il_legislators.csv b/csv/il_legislators.csv
diff --git a/csv/in_legislators.csv b/csv/in_legislators.csv
diff --git a/csv/ks_legislators.csv b/csv/ks_legislators.csv
diff --git a/csv/ky_legislators.csv b/csv/ky_legislators.csv
diff --git a/csv/la_legislators.csv b/csv/la_legislators.csv
diff --git a/csv/ma_legislators.csv b/csv/ma_legislators.csv
diff --git a/csv/md_legislators.csv b/csv/md_legislators.csv
diff --git a/csv/me_legislators.csv b/csv/me_legislators.csv
diff --git a/csv/mi_legislators.csv b/csv/mi_legislators.csv
diff --git a/csv/mn_legislators.csv b/csv/mn_legislators.csv
diff --git a/csv/mo_legislators.csv b/csv/mo_legislators.csv
diff --git a/csv/ms_legislators.csv b/csv/ms_legislators.csv
diff --git a/csv/mt_legislators.csv b/csv/mt_legislators.csv
diff --git a/csv/nc_legislators.csv b/csv/nc_legislators.csv
diff --git a/csv/nd_legislators.csv b/csv/nd_legislators.csv
diff --git a/csv/ne_legislators.csv b/csv/ne_legislators.csv
diff --git a/csv/nh_legislators.csv b/csv/nh_legislators.csv
diff --git a/csv/nj_legislators.csv b/csv/nj_legislators.csv
diff --git a/csv/nm_legislators.csv b/csv/nm_legislators.csv
diff --git a/csv/nv_legislators.csv b/csv/nv_legislators.csv
diff --git a/csv/ny_legislators.csv b/csv/ny_legislators.csv
diff --git a/csv/oh_legislators.csv b/csv/oh_legislators.csv
diff --git a/csv/ok_legislators.csv b/csv/ok_legislators.csv
diff --git a/csv/or_legislators.csv b/csv/or_legislators.csv
diff --git a/csv/pa_legislators.csv b/csv/pa_legislators.csv
diff --git a/csv/pr_legislators.csv b/csv/pr_legislators.csv
diff --git a/csv/ri_legislators.csv b/csv/ri_legislators.csv
diff --git a/csv/sc_legislators.csv b/csv/sc_legislators.csv
diff --git a/csv/sd_legislators.csv b/csv/sd_legislators.csv
diff --git a/csv/tn_legislators.csv b/csv/tn_legislators.csv
diff --git a/csv/tx_legislators.csv b/csv/tx_legislators.csv
diff --git a/csv/ut_legislators.csv b/csv/ut_legislators.csv
diff --git a/csv/va_legislators.csv b/csv/va_legislators.csv
diff --git a/csv/vt_legislators.csv b/csv/vt_legislators.csv
diff --git a/csv/wa_legislators.csv b/csv/wa_legislators.csv
diff --git a/csv/wi_legislators.csv b/csv/wi_legislators.csv
diff --git a/csv/wv_legislators.csv b/csv/wv_legislators.csv
diff --git a/csv/wy_legislators.csv b/csv/wy_legislators.csv
diff --git a/scripts/to_csv.py b/scripts/to_csv.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+import os
+import csv
+import glob
+import click
+from utils import (get_data_dir, get_jurisdiction_id, get_all_abbreviations,
+                   load_yaml, role_is_active)
+
+
+def write_csv(files, jurisdiction_id, output_filename):
+    with open(output_filename, "w") as outf:
+        out = csv.DictWriter(
+            outf,
+            ("id", "name",
+             "current_party", "current_district", "current_chamber",
+             "given_name", "family_name", "gender", "biography", "birth_date", "death_date",
+             "image", "links", "sources",
+             "capitol_address", "capitol_email", "capitol_voice", "capitol_fax",
+             "district_address", "district_email", "district_voice", "district_fax",
+             "twitter", "youtube", "instagram", "facebook",
+             )
+        )
+        out.writeheader()
+
+        for filename in files:
+            with open(filename) as f:
+                data = load_yaml(f)
+
+                # current party
+                for role in data["party"]:
+                    if role_is_active(role):
+                        current_party = role["name"]
+                        break
+
+                # current district
+                for role in data["roles"]:
+                    if role_is_active(role):
+                        current_chamber = role["type"]
+                        current_district = role["district"]
+
+                district_address = district_email = district_voice = district_fax = None
+                capitol_address = capitol_email = capitol_voice = capitol_fax = None
+                for cd in data.get("contact_details", {}):
+                    note = cd["note"].lower()
+                    if "district" in note:
+                        district_address = cd.get("address")
+                        district_email = cd.get("email")
+                        district_voice = cd.get("voice")
+                        district_fax = cd.get("fax")
+                    elif "capitol" in note:
+                        capitol_address = cd.get("address")
+                        capitol_email = cd.get("email")
+                        capitol_voice = cd.get("voice")
+                        capitol_fax = cd.get("fax")
+                    else:
+                        click.secho("unknown office: " + note, fg="red")
+
+                links = ";".join(l["url"] for l in data.get("links", []))
+                sources = ";".join(l["url"] for l in data.get("sources", []))
+
+                obj = {
+                    "id": data["id"],
+                    "name": data["name"],
+                    "current_party": current_party,
+                    "current_district": current_district,
+                    "current_chamber": current_chamber,
+                    "given_name": data.get("given_name"),
+                    "family_name": data.get("family_name"),
+                    "gender": data.get("gender"),
+                    "biography": data.get("biography"),
+                    "birth_date": data.get("birth_date"),
+                    "death_date": data.get("death_date"),
+                    "image": data.get("image"),
+                    "twitter": data.get("ids", {}).get("twitter"),
+                    "youtube": data.get("ids", {}).get("youtube"),
+                    "instagram": data.get("ids", {}).get("instagram"),
+                    "facebook": data.get("ids", {}).get("facebook"),
+                    "links": links,
+                    "sources": sources,
+                    "district_address": district_address,
+                    "district_email": district_email,
+                    "district_voice": district_voice,
+                    "district_fax": district_fax,
+                    "capitol_address": capitol_address,
+                    "capitol_email": capitol_email,
+                    "capitol_voice": capitol_voice,
+                    "capitol_fax": capitol_fax,
+                }
+                out.writerow(obj)
+
+    click.secho(f'processed {len(files)} files', fg='green')
+
+
+@click.command()
+@click.argument('abbreviations', nargs=-1)
+def to_csv(abbreviations):
+    """
+    Sync YAML files to DB.
+    """
+    if not abbreviations:
+        abbreviations = get_all_abbreviations()
+
+    for abbr in abbreviations:
+        click.secho('==== {} ===='.format(abbr), bold=True)
+        directory = get_data_dir(abbr)
+        jurisdiction_id = get_jurisdiction_id(abbr)
+        person_files = sorted(glob.glob(os.path.join(directory, 'people/*.yml')))
+        write_csv(person_files, jurisdiction_id, f"csv/{abbr}_legislators.csv")
+
+
+if __name__ == '__main__':
+    to_csv()