Upgrade to pluto 19v2 (#209)

This PR reworks our WOW buildings table sql code to reference pluto_19v2 for the yearbuilt, lat, lng and unitsres fields. Thanks for @toolness's tooling, we were also able to fix our testing suite to account for these changes.
JustFixNYC · Mar 12, 2020 · 423ccc4 · 423ccc4
1 parent c9a4ae8
commit 423ccc4
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 105 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-nycdb==0.1.22
+https://github.com/nycdb/nycdb/archive/2560271cb47fc35d7e5584c673fd57cbcdb5880a.zip#subdirectory=src
 psycopg2-binary==2.7.7
 mypy==0.660
 PyYAML>=5.1

diff --git a/server/services/db.js b/server/services/db.js
@@ -19,7 +19,7 @@ const buildingInfoSQL =
      ELSE '' END BORO,
 	   LAT LATITUDE,
 	   LNG LONGITUDE
-   FROM PLUTO_18V2
+   FROM PLUTO_19V2
    WHERE BBL = $1`
 
 

diff --git a/sql/create_bldgs_table.sql b/sql/create_bldgs_table.sql
@@ -31,7 +31,7 @@ LEFT JOIN (
     unitsres,
     yearbuilt,
     lat, lng
-  FROM pluto_18v1
+  FROM pluto_19v2
 ) pluto ON (registrations.bbl = pluto.bbl)
 LEFT JOIN (
   SELECT

diff --git a/tests/data/marshal_evictions_19.csv b/tests/data/marshal_evictions_19.csv
@@ -0,0 +1,10 @@
+uniqueid,bbl,borough,cleaned_address,courtindexnumber,docketnumber,evictionaddress,evictionaptnum,evictionzip,executeddate,geosearch_address,marshalfirstname,marshallastname,residentialcommercialind,schedulestatus
+1,1000110007,MANHATTAN,16 BEAVER STREET,83817/16,111028,16 BEAVER STREET 4TH FLOOR OFFICES,,10004,2019-01-07,,MAXINE,CHEVLOWE,COMMERCIAL,SCHEDULED
+2,1000160100,MANHATTAN,375 SOUTH END AVENUE,59721/19,96127,375 SOUTH END AVENUE,19B,10280,2019-09-24,,HENRY,DALEY,RESIDENTIAL,SCHEDULED
+3,1000160100,MANHATTAN,375 SOUTH END AVENUE,55804/19,95689,375 SOUTH END AVENUE,21R,10280,2019-06-18,,HENRY,DALEY,RESIDENTIAL,SCHEDULED
+4,1000160100,MANHATTAN,395 SOUTH END AVENUE,79909/18,94007,395 SOUTH END AVENUE,17E,10280,2019-05-07,,HENRY,DALEY,RESIDENTIAL,SCHEDULED
+5,1000160100,MANHATTAN,375 SOUTH END AVENUE,51186/19,94076,375 SOUTH END AVENUE,29N,10280,2019-04-04,,HENRY,DALEY,RESIDENTIAL,SCHEDULED
+6,1000160125,MANHATTAN,225 LIBERTY STREET,73778/18,33800,225 LIBERTY STREET ENT RENTABLE AREA 30FL,,10281,2019-04-04,,BRUCE,KEMP,COMMERCIAL,SCHEDULED
+7,1000167509,MANHATTAN,21 SOUTH END AVENUE,78825/18A,489595,21 SOUTH END AVENUE,*,10280,2019-05-13,,DANNY,WEINHEIM,COMMERCIAL,SCHEDULED
+8,1000167510,MANHATTAN,99 BATTERY PLACE,52107/19,94675,99 BATTERY PLACE,16 A,10280,2019-06-11,,JUSTIN,GROSSMAN,RESIDENTIAL,SCHEDULED
+9,1000167519,MANHATTAN,300 NORTH END AVENUE,57439/19,490922,300 NORTH END AVENUE,21B,10282,2019-06-03,,DANNY,WEINHEIM,RESIDENTIAL,SCHEDULED
diff --git a/tests/data/pluto_19v2.zip b/tests/data/pluto_19v2.zip
diff --git a/tests/factories/marshal_evictions_19.py b/tests/factories/marshal_evictions_19.py
@@ -0,0 +1,19 @@
+from typing import NamedTuple
+
+
+class MarshalEvictions19(NamedTuple):
+    uniqueid: str = '1'
+    bbl: str = '1000110007'
+    borough: str = 'MANHATTAN'
+    cleaned_address: str = '16 BEAVER STREET'
+    courtindexnumber: str = '83817/16'
+    docketnumber: str = '111028'
+    evictionaddress: str = '16 BEAVER STREET 4TH FLOOR OFFICES'
+    evictionaptnum: str = ''
+    evictionzip: str = '10004'
+    executeddate: str = '1/7/19'
+    geosearch_address: str = ''
+    marshalfirstname: str = 'MAXINE'
+    marshallastname: str = 'CHEVLOWE'
+    residentialcommercialind: str = 'COMMERCIAL'
+    schedulestatus: str = 'SCHEDULED'
diff --git a/tests/factories/pluto_18v1.py b/tests/factories/pluto_18v1.py
diff --git a/tests/factories/pluto_19v2.py b/tests/factories/pluto_19v2.py
@@ -0,0 +1,101 @@
+from typing import NamedTuple
+
+
+class Pluto19v2(NamedTuple):
+    borough: str = 'QN'
+    block: str = '3611'
+    lot: str = '54'
+    cd: str = '405'
+    ct2010: str = '613.01'
+    cb2010: str = '3002'
+    schooldist: str = '24'
+    council: str = '30'
+    zipcode: str = '11385'
+    firecomp: str = 'L135'
+    policeprct: str = '104'
+    healtharea: str = '1700'
+    sanitboro: str = '4'
+    sanitsub: str = '2E'
+    address: str = '61-27 GATES AVENUE'
+    zonedist1: str = 'R5B'
+    zonedist2: str = ''
+    zonedist3: str = ''
+    zonedist4: str = ''
+    overlay1: str = ''
+    overlay2: str = ''
+    spdist1: str = ''
+    spdist2: str = ''
+    spdist3: str = ''
+    ltdheight: str = ''
+    splitzone: str = 'N'
+    bldgclass: str = 'B1'
+    landuse: str = '01'
+    easements: str = '0'
+    ownertype: str = ''
+    ownername: str = 'STIMPFL, ARIANNA'
+    lotarea: str = '2000'
+    bldgarea: str = '2910'
+    comarea: str = '0'
+    resarea: str = '2328'
+    officearea: str = '0'
+    retailarea: str = '0'
+    garagearea: str = '0'
+    strgearea: str = '0'
+    factryarea: str = '0'
+    otherarea: str = '0'
+    areasource: str = '2'
+    numbldgs: str = '0001'
+    numfloors: str = '2'
+    unitsres: str = '2'
+    unitstotal: str = '2'
+    lotfront: str = '20.00'
+    lotdepth: str = '100.00'
+    bldgfront: str = '20.00'
+    bldgdepth: str = '55.00'
+    ext: str = 'N'
+    proxcode: str = '3'
+    irrlotcode: str = 'N'
+    lottype: str = '5'
+    bsmtcode: str = '1'
+    assessland: str = '9720'
+    assesstot: str = '56580'
+    exempttot: str = '0'
+    yearbuilt: str = '1901'
+    yearalter1: str = '0'
+    yearalter2: str = '0'
+    histdist: str = ''
+    landmark: str = ''
+    builtfar: str = '1.46'
+    residfar: str = '1.35'
+    commfar: str = '0'
+    facilfar: str = '2'
+    borocode: str = '4'
+    bbl: str = '4036110054'
+    condono: str = ''
+    tract2010: str = '061301'
+    xcoord: str = '1012671'
+    ycoord: str = '197789'
+    zonemap: str = '13d'
+    zmcode: str = ''
+    sanborn: str = '403 069'
+    taxmap: str = '41901'
+    edesignum: str = ''
+    appbbl: str = ''
+    appdate: str = ''
+    mappluto_f: str = ''
+    plutomapid: str = '1'
+    version: str = '19v2'
+    sanitdistrict: str = '05'
+    healthcenterdistrict: str = '46'
+    firm07_flag: str = ''
+    pfirm15_flag: str = ''
+    rpaddate: str = '07/06/2019'
+    dcasdate: str = '11/15/2018'
+    zoningdate: str = '07/29/2019'
+    landmkdate: str = '07/05/2019'
+    basempdate: str = '07/26/2019'
+    masdate: str = ''
+    polidate: str = ''
+    edesigdate: str = '05/31/2019'
+    geom: str = '0106000020E61000000100000001030000000100000005000000B21F74BC707952C0198ABA20D65A4440B67948D76E7952C0C7BE0D68CD5A4440BA6B4C02707952C056C080D2CC5A4440880839E7717952C07F45098AD55A4440B21F74BC707952C0198ABA20D65A4440'
+    dcpedited: str = ''
diff --git a/tests/test_nycdb_loading.py b/tests/test_nycdb_loading.py
@@ -1,5 +1,5 @@
 from .factories.hpd_violations import HPDViolation
-from .factories.pluto_18v1 import Pluto18v1
+from .factories.pluto_19v2 import Pluto19v2
 from .factories.changes_summary import ChangesSummary
 
 
@@ -14,15 +14,15 @@ def test_loading_violations_works(db, nycdb_ctx):
 
 
 def test_loading_pluto_works(db, nycdb_ctx):
-    nycdb_ctx.write_zip('pluto_18v1.zip', {
-        'PLUTO_for_WEB/BK_18v1.csv': [
-            Pluto18v1(HistDist="Funky Historic District", Address="FUNKY STREET"),
-            Pluto18v1(HistDist="Monkey Historic District", Address="MONKEY STREET")
+    nycdb_ctx.write_zip('pluto_19v2.zip', {
+        'PLUTO_for_WEB/BK_19v2.csv': [
+            Pluto19v2(histdist="Funky Historic District", address="FUNKY STREET"),
+            Pluto19v2(histdist="Monkey Historic District", address="MONKEY STREET")
         ]
     })
-    nycdb_ctx.load_dataset('pluto_18v1')
+    nycdb_ctx.load_dataset('pluto_19v2')
     with db.cursor() as cur:
-        cur.execute("select * from pluto_18v1 where histdist='Funky Historic District'")
+        cur.execute("select * from pluto_19v2 where histdist='Funky Historic District'")
         assert cur.fetchone()['address'] == 'FUNKY STREET'
 
 

diff --git a/tests/test_sql.py b/tests/test_sql.py
@@ -4,9 +4,10 @@
 from .factories.hpd_registrations import HPDRegistration
 from .factories.marshal_evictions_17 import MarshalEvictions17
 from .factories.marshal_evictions_18 import MarshalEvictions18
+from .factories.marshal_evictions_19 import MarshalEvictions19
 from .factories.changes_summary import ChangesSummary
 from .factories.hpd_violations import HPDViolation
-from .factories.pluto_18v1 import Pluto18v1
+from .factories.pluto_19v2 import Pluto19v2
 
 # This test suite defines two landlords:
 #
@@ -112,13 +113,14 @@
 class TestSQL:
     @pytest.fixture(autouse=True, scope="class")
     def setup_class_fixture(self, db, nycdb_ctx):
-        nycdb_ctx.write_zip('pluto_18v1.zip', {
-            'PLUTO_for_WEB/BK_18v1.csv': [Pluto18v1()]
+        nycdb_ctx.write_zip('pluto_19v2.zip', {
+            'PLUTO_for_WEB/BK_19v2.csv': [Pluto19v2()]
         })
         nycdb_ctx.write_csv('hpd_violations.csv', [HPDViolation()])
         nycdb_ctx.write_csv('changes-summary.csv', [ChangesSummary()])
         nycdb_ctx.write_csv('marshal_evictions_17.csv', [MarshalEvictions17()])
         nycdb_ctx.write_csv('marshal_evictions_18.csv', [MarshalEvictions18()])
+        nycdb_ctx.write_csv('marshal_evictions_19.csv', [MarshalEvictions19()])
         nycdb_ctx.write_csv('hpd_registrations.csv', [
             FUNKY_REGISTRATION,
             MONKEY_REGISTRATION,

diff --git a/who-owns-what.yml b/who-owns-what.yml
@@ -1,7 +1,7 @@
 ---
 dependencies:
   # These are NYCDB datasets needed by the SQL scripts.
-  - pluto_18v1
+  - pluto_19v2
   - rentstab_summary
   - marshal_evictions
   - hpd_registrations