Merge branch 'main' into add-hpd-aep

README.md

@@ -15,12 +15,12 @@ Residents, lawyers, tenants, and organizers who want to use data in their strugg
 - [HPD Litigations](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Litigations)
 - [HPD Registrations](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Registrations)
 - [HPD Complaints](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Complaints)
-- [HPD Handyman Work Order (HWO) Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Handyman-Work-Order-(HWO)-Charges)
-- [HPD Open Market Order (OMO) Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Open-Market-Order-(OMO)-Charges)
+- [HPD Charges](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Charges)
 - [HPD Repair and Vacate Orders](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Vacate-Orders)
 - [HPD Affordable Production (Building and Project)](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Affordable-Production)
 - [HPD Certificate of No Harassment](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Certificate-of-No-Harassment)
 - [HPD Alternative Enforcement Program (AEP)](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Alternative-Enforcement-Program-(AEP))
+- [HPD Underlying Conditions Program](https://github.com/nycdb/nycdb/wiki/Dataset:-HPD-Underlying-Conditions-Program)
 - [Department of Finance Rolling Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Rolling-Sales)
 - [Department of Finance Annualized Sales](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Annualized-Sales)
 - [Department of Finance Property Tax Exemptions](https://github.com/nycdb/nycdb/wiki/Dataset:-DOF-Exemptions)

src/nycdb/dataset_transformations.py

@@ -209,12 +209,16 @@ def hpd_hwo_charges(dataset):
     return to_csv(dataset.files[0].dest)
 
 
+def hpd_omo_charges(dataset):
+    return to_csv(dataset.files[1].dest)
+
+
 def hpd_omo_invoices(dataset):
-    return to_csv(dataset.files[0].dest)
+    return to_csv(dataset.files[2].dest)
 
 
-def hpd_omo_charges(dataset):
-    return to_csv(dataset.files[1].dest)
+def hpd_fee_charges(dataset):
+    return to_csv(dataset.files[3].dest)
 
 
 def dob_safety_violations(dataset):
@@ -227,3 +231,7 @@ def dhs_daily_shelter_count(dataset):
 
 def hpd_aep(dataset):
     return to_csv(dataset.files[0].dest, header_replacements={'ofbcviolationsatstart': 'bcviolationsatstart'})
+
+
+def hpd_underlying_conditions(dataset):
+    return to_csv(dataset.files[0].dest)

src/nycdb/datasets/boundaries.yml

@@ -1,125 +1,125 @@
 ---
 files:
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyad_23d.zip
-    dest: nyad_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycg_23d.zip
-    dest: nycg_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyss_23d.zip
-    dest: nyss_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nymc_23d.zip
-    dest: nymc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycc_23d.zip
-    dest: nycc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyed_23d.zip
-    dest: nyed_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nybb_23d.zip
-    dest: nybb_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycd_23d.zip
-    dest: nycd_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nysd_23d.zip
-    dest: nysd_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nypp_23d.zip
-    dest: nypp_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyha_23d.zip
-    dest: nyha_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyhc_23d.zip
-    dest: nyhc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfc_23d.zip
-    dest: nyfc_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfb_23d.zip
-    dest: nyfb_23d.zip
-  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfd_23d.zip
-    dest: nyfd_23d.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyad_24b.zip
+    dest: nyad_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycg_24b.zip
+    dest: nycg_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyss_24b.zip
+    dest: nyss_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nymc_24b.zip
+    dest: nymc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycc_24b.zip
+    dest: nycc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyed_24b.zip
+    dest: nyed_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nybb_24b.zip
+    dest: nybb_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nycd_24b.zip
+    dest: nycd_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nysd_24b.zip
+    dest: nysd_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nypp_24b.zip
+    dest: nypp_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyha_24b.zip
+    dest: nyha_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyhc_24b.zip
+    dest: nyhc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfc_24b.zip
+    dest: nyfc_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfb_24b.zip
+    dest: nyfb_24b.zip
+  - url: https://s-media.nyc.gov/agencies/dcp/assets/files/zip/data-tools/bytes/nyfd_24b.zip
+    dest: nyfd_24b.zip
 sql: 
   - boundaries.sql
 schema:
   # State Assembly Districts
   - table_name: nyad
-    path: nyad_23d/nyad
-    dest: nyad_23d.zip
+    path: nyad_24b/nyad
+    dest: nyad_24b.zip
     type: shapefile
     srid: 2263
   # Congressional Districts
   - table_name: nycg
-    path: nycg_23d/nycg
-    dest: nycg_23d.zip
+    path: nycg_24b/nycg
+    dest: nycg_24b.zip
     type: shapefile
     srid: 2263
   # State Senate Districts
   - table_name: nyss
-    path: nyss_23d/nyss
-    dest: nyss_23d.zip
+    path: nyss_24b/nyss
+    dest: nyss_24b.zip
     type: shapefile
     srid: 2263
   # Municipal Court Districts
   - table_name: nymc
-    path: nymc_23d/nymc
-    dest: nymc_23d.zip
+    path: nymc_24b/nymc
+    dest: nymc_24b.zip
     type: shapefile
     srid: 2263
   # City Council Districts
   - table_name: nycc
-    path: nycc_23d/nycc
-    dest: nycc_23d.zip
+    path: nycc_24b/nycc
+    dest: nycc_24b.zip
     type: shapefile
     srid: 2263
   # Election Districts
   - table_name: nyed
-    path: nyed_23d/nyed
-    dest: nyed_23d.zip
+    path: nyed_24b/nyed
+    dest: nyed_24b.zip
     type: shapefile
     srid: 2263
   # Borough Boundaries
   - table_name: nybb
-    path: nybb_23d/nybb
-    dest: nybb_23d.zip
+    path: nybb_24b/nybb
+    dest: nybb_24b.zip
     type: shapefile
     srid: 2263
   # Community Districts
   - table_name: nycd
-    path: nycd_23d/nycd
-    dest: nycd_23d.zip
+    path: nycd_24b/nycd
+    dest: nycd_24b.zip
     type: shapefile
     srid: 2263
  # School Districts
   - table_name: nysd
-    path: nysd_23d/nysd
-    dest: nysd_23d.zip
+    path: nysd_24b/nysd
+    dest: nysd_24b.zip
     type: shapefile
     srid: 2263
   # Police Precincts
   - table_name: nypp
-    path: nypp_23d/nypp
-    dest: nypp_23d.zip
+    path: nypp_24b/nypp
+    dest: nypp_24b.zip
     type: shapefile
     srid: 2263
   # Health Area
   - table_name: nyha
-    path: nyha_23d/nyha
-    dest: nyha_23d.zip
+    path: nyha_24b/nyha
+    dest: nyha_24b.zip
     type: shapefile
     srid: 2263
   # Health Center
   - table_name: nyhc
-    path: nyhc_23d/nyhc
-    dest: nyhc_23d.zip
+    path: nyhc_24b/nyhc
+    dest: nyhc_24b.zip
     type: shapefile
     srid: 2263
   # Fire Companies
   - table_name: nyfc
-    path: nyfc_23d/nyfc
-    dest: nyfc_23d.zip
+    path: nyfc_24b/nyfc
+    dest: nyfc_24b.zip
     type: shapefile
     srid: 2263
   # Fire Battalions
   - table_name: nyfb
-    path: nyfb_23d/nyfb
-    dest: nyfb_23d.zip
+    path: nyfb_24b/nyfb
+    dest: nyfb_24b.zip
     type: shapefile
     srid: 2263
   # Fire Divisions
   - table_name: nyfd
-    path: nyfd_23d/nyfd
-    dest: nyfd_23d.zip
+    path: nyfd_24b/nyfd
+    dest: nyfd_24b.zip
     type: shapefile
     srid: 2263

src/nycdb/datasets/dob_safety_violations.yml

@@ -24,5 +24,12 @@ schema:
     state: text
     zip: text
     bbl: char(10)
+    postcode: text
+    latitude: double precision
+    longitude: double precision
+    communityboard: text
+    councildistrict: text
+    censustract2020: text
+    neighborhoodtabulationareanta2020: text
 sql:
 - dob_safety_violations.sql

src/nycdb/datasets/hpd_charges.yml

@@ -0,0 +1,132 @@
+files:
+  -
+    url: https://data.cityofnewyork.us/api/views/sbnd-xujn/rows.csv?accessType=DOWNLOAD
+    dest: hpd_hwo_charges.csv
+  -
+    url: https://data.cityofnewyork.us/api/views/mdbu-nrqn/rows.csv?accessType=DOWNLOAD
+    dest: hpd_omo_charges.csv
+  -
+    url: https://data.cityofnewyork.us/api/views/emrz-5p35/rows.csv?accessType=DOWNLOAD
+    dest: hpd_omo_invoices.csv
+  -
+    url: https://data.cityofnewyork.us/api/views/cp6j-7bjj/rows.csv?accessType=DOWNLOAD
+    dest: hpd_fee_charges.csv
+sql:
+  - hpd_charges.sql
+schema:
+  -
+    table_name: hpd_hwo_charges
+    fields:
+      HWOID: integer
+      HWONumber: text
+      BuildingID: integer
+      BoroID: integer
+      Boro: text
+      HouseNumber: text
+      StreetName: text
+      Zip: char(5)
+      Block: integer
+      Lot: integer
+      LifeCycle: text
+      WorkTypeGeneral: text
+      HWOStatusReason: text
+      HWOCreateDate: date
+      IsAEP: text
+      IsCommercialDemolition: integer
+      FEMAEventID: numeric
+      FEMAEvent: text
+      HWODescription: text
+      HWOApprovedAmount: numeric
+      SalesTax: numeric
+      AdminFee: numeric
+      ChargeAmount: numeric
+      DateTransferDoF: date
+      Latitude: numeric
+      Longitude: numeric
+      CommunityBoard: numeric
+      CouncilDistrict: text
+      CensusTract: text
+      BIN: char(7)
+      BBL: char(10)
+      NTA: text
+  -
+    table_name: hpd_omo_charges
+    fields:
+      OMOID: integer
+      OMONumber: text
+      BuildingID: integer
+      BoroID: char(1)
+      Boro: text
+      HouseNumber: text
+      StreetName: text
+      Apartment: text
+      Zip: text
+      Block: integer
+      Lot: integer
+      LifeCycle: text
+      WorkTypeGeneral: text
+      OMOStatusReason: text
+      OMOAwardAmount: numeric
+      OMOCreateDate: date
+      NetChangeOrders: integer
+      OMOAwardDate: date
+      IsAEP: text
+      IsCommercialDemolition: text
+      ServiceChargeFlag: boolean
+      FEMAEventID: smallint
+      FEMAEvent: text
+      OMODescription: text
+      Latitude: numeric
+      Longitude: numeric
+      CommunityBoard: smallint
+      CouncilDistrict: smallint
+      CensusTract: integer
+      BIN:  integer
+      BBL: char(10)
+      NTA: text
+  -
+    table_name: hpd_omo_invoices
+    fields:
+      InvoiceID: text
+      InvoiceSequenceID: text
+      OMONumber: char(7)
+      InvoiceStatus: text
+      InvoiceDate: date
+      InvoiceBillAmount: numeric
+      InvoicePayAmount: numeric
+      SalesTax: numeric
+      AdminFee: numeric
+      PaymentID: numeric
+      ChargeAmount: numeric
+      DateTransferDoF: date
+      UniqueKey: text
+  -
+    table_name: hpd_fee_charges
+    fields:
+      Feeid: integer
+      Buildingid: integer
+      Boroid: integer
+      Boro: text
+      Housenumber: text
+      Streetname: text
+      Zip: char(5)
+      Block: integer
+      Lot: integer
+      Lifecycle: text
+      Feetypeid: integer
+      Feetype: text
+      Feesourcetypeid: integer
+      Feesourcetype: text
+      Feesourceid: integer
+      Feeissueddate: date
+      Feeamount: integer
+      Dofaccounttype: integer
+      Doftransferdate: date
+      Latitude: numeric
+      Longitude: numeric
+      CommunityBoard: text
+      CouncilDistrict: text
+      CensusTract: text
+      Bin: char(7)
+      Bbl: char(10)
+      Nta: text