Merge branch 'main' into dohmh-rodent-inspections

README.md

@@ -15,11 +15,12 @@ Residents, lawyers, tenants, and organizers who want to use data in their strugg
 - [HPD Litigations](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Litigations)
 - [HPD Registrations](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Registrations)
 - [HPD Complaints](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Complaints)
-- [HPD Handyman Work Order (HWO) Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Handyman-Work-Order-(HWO)-Charges)
-- [HPD Open Market Order (OMO) Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Open-Market-Order-(OMO)-Charges)
+- [HPD Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Charges)
 - [HPD Repair and Vacate Orders](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Vacate-Orders)
 - [HPD Affordable Production (Building and Project)](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Affordable-Production)
 - [HPD Certificate of No Harassment](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Certificate-of-No-Harassment)
+- [HPD Alternative Enforcement Program (AEP)](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Alternative-Enforcement-Program-(AEP))
+- [HPD Underlying Conditions Program](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Underlying-Conditions-Program)
 - [Department of Finance Rolling Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Rolling-Sales)
 - [Department of Finance Annualized Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Annualized-Sales)
 - [Department of Finance Property Tax Exemptions](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Exemptions)

src/nycdb/dataset_transformations.py

@@ -186,7 +186,7 @@ def hpd_affordable_project(dataset):
 
 
 def hpd_conh(dataset):
-    return to_csv(dataset.files[0].dest)
+    return to_csv(dataset.files[0].dest, header_replacements={"neighborhoodtabulationareanta2020": "nta"})
 
 
 def dcp_housingdb(dataset):
@@ -209,12 +209,16 @@ def hpd_hwo_charges(dataset):
     return to_csv(dataset.files[0].dest)
 
 
+def hpd_omo_charges(dataset):
+    return to_csv(dataset.files[1].dest)
+
+
 def hpd_omo_invoices(dataset):
-    return to_csv(dataset.files[0].dest)
+    return to_csv(dataset.files[2].dest)
 
 
-def hpd_omo_charges(dataset):
-    return to_csv(dataset.files[1].dest)
+def hpd_fee_charges(dataset):
+    return to_csv(dataset.files[3].dest)
 
 
 def dob_safety_violations(dataset):
@@ -233,3 +237,11 @@ def dohmh_rodent_inspections(dataset):
         ),
         borough="borocode",
     )
+
+
+def hpd_aep(dataset):
+    return to_csv(dataset.files[0].dest, header_replacements={'ofbcviolationsatstart': 'bcviolationsatstart'})
+
+
+def hpd_underlying_conditions(dataset):
+    return to_csv(dataset.files[0].dest)

src/nycdb/datasets/boundaries.yml

@@ -1,125 +1,125 @@
 ---
 files:
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyad_23d.zip
-    dest: nyad_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycg_23d.zip
-    dest: nycg_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyss_23d.zip
-    dest: nyss_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nymc_23d.zip
-    dest: nymc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycc_23d.zip
-    dest: nycc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyed_23d.zip
-    dest: nyed_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nybb_23d.zip
-    dest: nybb_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycd_23d.zip
-    dest: nycd_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nysd_23d.zip
-    dest: nysd_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nypp_23d.zip
-    dest: nypp_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyha_23d.zip
-    dest: nyha_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyhc_23d.zip
-    dest: nyhc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfc_23d.zip
-    dest: nyfc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfb_23d.zip
-    dest: nyfb_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfd_23d.zip
-    dest: nyfd_23d.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyad_24b.zip
+    dest: nyad_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycg_24b.zip
+    dest: nycg_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyss_24b.zip
+    dest: nyss_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nymc_24b.zip
+    dest: nymc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycc_24b.zip
+    dest: nycc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyed_24b.zip
+    dest: nyed_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nybb_24b.zip
+    dest: nybb_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycd_24b.zip
+    dest: nycd_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nysd_24b.zip
+    dest: nysd_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nypp_24b.zip
+    dest: nypp_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyha_24b.zip
+    dest: nyha_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyhc_24b.zip
+    dest: nyhc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfc_24b.zip
+    dest: nyfc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfb_24b.zip
+    dest: nyfb_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfd_24b.zip
+    dest: nyfd_24b.zip
 sql: 
   - boundaries.sql
 schema:
   # State Assembly Districts
   - table_name: nyad
-    path: nyad_23d/nyad
-    dest: nyad_23d.zip
+    path: nyad_24b/nyad
+    dest: nyad_24b.zip
     type: shapefile
     srid: 2263
   # Congressional Districts
   - table_name: nycg
-    path: nycg_23d/nycg
-    dest: nycg_23d.zip
+    path: nycg_24b/nycg
+    dest: nycg_24b.zip
     type: shapefile
     srid: 2263
   # State Senate Districts
   - table_name: nyss
-    path: nyss_23d/nyss
-    dest: nyss_23d.zip
+    path: nyss_24b/nyss
+    dest: nyss_24b.zip
     type: shapefile
     srid: 2263
   # Municipal Court Districts
   - table_name: nymc
-    path: nymc_23d/nymc
-    dest: nymc_23d.zip
+    path: nymc_24b/nymc
+    dest: nymc_24b.zip
     type: shapefile
     srid: 2263
   # City Council Districts
   - table_name: nycc
-    path: nycc_23d/nycc
-    dest: nycc_23d.zip
+    path: nycc_24b/nycc
+    dest: nycc_24b.zip
     type: shapefile
     srid: 2263
   # Election Districts
   - table_name: nyed
-    path: nyed_23d/nyed
-    dest: nyed_23d.zip
+    path: nyed_24b/nyed
+    dest: nyed_24b.zip
     type: shapefile
     srid: 2263
   # Borough Boundaries
   - table_name: nybb
-    path: nybb_23d/nybb
-    dest: nybb_23d.zip
+    path: nybb_24b/nybb
+    dest: nybb_24b.zip
     type: shapefile
     srid: 2263
   # Community Districts
   - table_name: nycd
-    path: nycd_23d/nycd
-    dest: nycd_23d.zip
+    path: nycd_24b/nycd
+    dest: nycd_24b.zip
     type: shapefile
     srid: 2263
  # School Districts
   - table_name: nysd
-    path: nysd_23d/nysd
-    dest: nysd_23d.zip
+    path: nysd_24b/nysd
+    dest: nysd_24b.zip
     type: shapefile
     srid: 2263
   # Police Precincts
   - table_name: nypp
-    path: nypp_23d/nypp
-    dest: nypp_23d.zip
+    path: nypp_24b/nypp
+    dest: nypp_24b.zip
     type: shapefile
     srid: 2263
   # Health Area
   - table_name: nyha
-    path: nyha_23d/nyha
-    dest: nyha_23d.zip
+    path: nyha_24b/nyha
+    dest: nyha_24b.zip
     type: shapefile
     srid: 2263
   # Health Center
   - table_name: nyhc
-    path: nyhc_23d/nyhc
-    dest: nyhc_23d.zip
+    path: nyhc_24b/nyhc
+    dest: nyhc_24b.zip
     type: shapefile
     srid: 2263
   # Fire Companies
   - table_name: nyfc
-    path: nyfc_23d/nyfc
-    dest: nyfc_23d.zip
+    path: nyfc_24b/nyfc
+    dest: nyfc_24b.zip
     type: shapefile
     srid: 2263
   # Fire Battalions
   - table_name: nyfb
-    path: nyfb_23d/nyfb
-    dest: nyfb_23d.zip
+    path: nyfb_24b/nyfb
+    dest: nyfb_24b.zip
     type: shapefile
     srid: 2263
   # Fire Divisions
   - table_name: nyfd
-    path: nyfd_23d/nyfd
-    dest: nyfd_23d.zip
+    path: nyfd_24b/nyfd
+    dest: nyfd_24b.zip
     type: shapefile
     srid: 2263

src/nycdb/datasets/hpd_aep.yml

@@ -0,0 +1,29 @@
+---
+files:
+  -
+    url: https://data.cityofnewyork.us/api/views/hcir-3275/rows.csv?accessType=DOWNLOAD
+    dest: hpd_aep.csv
+sql:
+    - hpd_aep.sql
+schema:
+  table_name: hpd_aep
+  fields:
+    BuildingId: integer
+    Borough: text
+    Number: text
+    Street: text
+    TotalUnits: integer
+    AepStartDate: date
+    bcviolationsatstart: integer
+    CurrentStatus: text
+    DischargeDate: date
+    AepRound: text
+    Postcode: char(5)
+    Latitude: numeric
+    Longitude: numeric
+    CommunityBoard: text
+    CouncilDistrict: text
+    CensusTract: text
+    Bin: char(7)
+    Bbl: char(10)
+    Nta: text