add dataset for HPD underlying conditions program (#333)

Adds dataset for HPD underlying conditions program.

Note that all of the values for orderissuedate are invalid currently (year of "0201" and "0202"), but I've submitted a report on Open Data and in the past HPD has fixed this kind of thing quickly so I'm going to make it as a text column for now and leave the values as-is, rather than trying to manually fix this in the sql or something.

README.md

@@ -19,6 +19,7 @@ Residents, lawyers, tenants, and organizers who want to use data in their strugg
 - [HPD Repair and Vacate Orders](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Vacate-Orders)
 - [HPD Affordable Production (Building and Project)](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Affordable-Production)
 - [HPD Certificate of No Harassment](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Certificate-of-No-Harassment)
+- [HPD Underlying Conditions Program](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Underlying-Conditions-Program)
 - [Department of Finance Rolling Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Rolling-Sales)
 - [Department of Finance Annualized Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Annualized-Sales)
 - [Department of Finance Property Tax Exemptions](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Exemptions)

src/nycdb/dataset_transformations.py

@@ -227,3 +227,7 @@ def dob_safety_violations(dataset):
 
 def dhs_daily_shelter_count(dataset):
     return to_csv(dataset.files[0].dest, header_replacements={'table': 'series_name'})
+
+
+def hpd_underlying_conditions(dataset):
+    return to_csv(dataset.files[0].dest)

src/nycdb/datasets/hpd_underlying_conditions.yml

@@ -0,0 +1,26 @@
+---
+files:
+  -
+    url: https://data.cityofnewyork.us/api/views/xpbf-ithr/rows.csv?accessType=DOWNLOAD
+    dest: hpd_underlying_conditions.csv
+sql:
+    - hpd_underlying_conditions.sql
+schema:
+  table_name: hpd_underlying_conditions
+  fields:
+    BuildingId: integer
+    Borough: text
+    NumberPhn: text
+    Street: text
+    TotalUnits: integer
+    OrderIssueDate: text
+    CurrentStatus: text
+    Postcode: char(5)
+    Latitude: numeric
+    Longitude: numeric
+    CommunityBoard: text
+    CouncilDistrict: text
+    Bin: char(7)
+    Bbl: char(10)
+    CensusTract2020: text
+    Nta2020: text

src/nycdb/sql/hpd_underlying_conditions.sql

@@ -0,0 +1,2 @@
+CREATE INDEX hpd_underlying_conditions_bbl_idx ON hpd_underlying_conditions (bbl);
+CREATE INDEX hpd_underlying_conditions_orderissuedate_idx ON hpd_underlying_conditions (orderissuedate);

src/nycdb/verify.py

@@ -100,7 +100,8 @@
     'dob_certificate_occupancy': {'dob_certificate_occupancy': 138_000},
     'dob_safety_violations': {'dob_safety_violations': 491_174},
     'boundaries': {'nyad': 65, 'nycg': 13, 'nyss': 28, 'nymc': 28, 'nycc': 51, 'nyed': 4286, 'nybb': 5, 'nycd': 71, 'nysd': 33, 'nypp': 77, 'nyha': 354, 'nyhc': 30, 'nyfc': 348, 'nyfb': 49, 'nyfd': 9},
-    'dhs_daily_shelter_count': {'dhs_daily_shelter_count': 7_000}
+    'dhs_daily_shelter_count': {'dhs_daily_shelter_count': 7_000},
+    'hpd_underlying_conditions': {'hpd_underlying_conditions': 500}
 }
 
 

src/tests/integration/data/hpd_underlying_conditions.csv

@@ -0,0 +1,6 @@
+BUILDING_ID,BOROUGH,Number (PHN),STREET,TOTAL_UNITS,ORDER_ISSUE_DATE,CURRENT_STATUS,POSTCODE,LATITUDE,LONGITUDE,COMMUNITY BOARD,COUNCIL DISTRICT,BIN,BBL,CENSUS TRACT (2020),NTA (2020)
+93750,Bronx,679,MAGENTA STREET,37,10/24/0201,Discharged,10467,40.875293,-73.867797,12,12,2056950,2046280001,374,Williamsbridge-Olinville
+85997,Bronx,3644,HOLLAND AVENUE,12,10/20/0201,Discharged,10467,40.879775,-73.863438,12,12,2057860,2046620040,380,Williamsbridge-Olinville
+57762,Bronx,1142,COLLEGE AVENUE,30,12/12/0202,Discharged,10456,40.831147,-73.914322,4,16,2002258,2024340015,175,Concourse-Concourse Village
+25930,Manhattan,200,NAGLE AVENUE,48,10/30/0201,Discharged,10034,40.862103,-73.923371,12,10,1064715,1022170035,291,Inwood
+89073,Bronx,1465,JESUP AVENUE,55,10/27/0201,Discharged,10452,40.842813,-73.91957,4,16,2008689,2028720207,21302,Mount Eden-Claremont (West)

src/tests/integration/test_nycdb.py

@@ -97,6 +97,10 @@ def has_one_row(*args):
     return bool(fetch_one_row(*args))
 
 
+def has_index(conn, index_name):
+    return has_one_row(conn, f"select 1 where to_regclass('public.{index_name}') is NOT NULL")
+
+
 def table_columns(conn, table_name):
     sql = "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = '{}'".format(
         table_name
@@ -791,3 +795,16 @@ def test_dhs_daily_shelter_count(conn):
     ecb_violations = nycdb.Dataset('dhs_daily_shelter_count', args=ARGS)
     ecb_violations.db_import()
     assert row_count(conn, 'dhs_daily_shelter_count') == 5
+
+
+def test_hpd_underlying_conditions(conn):
+    drop_table(conn, 'hpd_underlying_conditions')
+    dataset = nycdb.Dataset('hpd_underlying_conditions', args=ARGS)
+    dataset.db_import()
+    assert row_count(conn, 'hpd_underlying_conditions') == 5
+    assert has_index(conn, 'hpd_underlying_conditions_bbl_idx')
+    with conn.cursor(row_factory=dict_row) as curs:
+        curs.execute("select * from hpd_underlying_conditions WHERE bbl = '2046280001'")
+        rec = curs.fetchone()
+        assert rec is not None
+        assert rec['currentstatus'] == 'Discharged'