Ar build 2024 colp

dcpy/cli.py

@@ -1,14 +1,16 @@
 import typer
 import dcpy.lifecycle._cli as lifecycle
 import dcpy.connectors._cli as connectors
+import dcpy.utils._cli as utils
 
 # This is a separate module because it will eventually import most of dcpy
 # as we expand functionality. There maybe be cases where we don't want to do that,
 # e.g. if we just want to export one submodule (e.g. utils)
 
 if __name__ == "__main__":
-    app = typer.Typer()  # add_completion=False)
+    app = typer.Typer()
 
     app.add_typer(lifecycle.app, name="lifecycle")
     app.add_typer(connectors.app, name="connectors")
+    app.add_typer(utils.app, name="utils")
     app()

dcpy/library/templates/dcas_ipis.yml

@@ -2,8 +2,9 @@ dataset:
   name: dcas_ipis
   acl: public-read
   source:
-    script:
-      path: .library/upload/dcas_ipis.csv
+  source:
+    url:
+      path: s3://edm-recipes/inbox/dcas_ipis/{{ version }}/dcas_ipis.csv
     options:
       - "AUTODETECT_TYPE=NO"
       - "EMPTY_STRING_AS_NULL=YES"

dcpy/utils/_cli.py

@@ -0,0 +1,6 @@
+import typer
+
+from .s3 import app as s3_app
+
+app = typer.Typer()
+app.add_typer(s3_app, name="s3")

dcpy/utils/s3.py

@@ -419,5 +419,7 @@ def _cli_wrapper_upload_folder(
     )
 
 
-if __name__ == "__main__":
-    app()
+@app.command("download_file")
+def _cli_wrapper_download_file(bucket: str, s3_path: str, local_path: Path):
+    logger.info(f"Downloading file at {bucket}/{s3_path} to {local_path}")
+    download_file(bucket, s3_path, local_path)

products/colp/recipe.yml

@@ -1,13 +1,11 @@
 name: COLP
-version: "202311"
+version: "2024-06-01"
 product: db-colp
 inputs:
   missing_versions_strategy: find_latest
   datasets:
   - name: dcp_pluto
-    version: "23v3"
   - name: dcp_colp
-    version: "202305"
   - name: dcas_ipis
   - name: dof_air_rights_lots
   - name: dcp_boroboundaries_wi

products/colp/sql/geo_inputs.sql

@@ -36,15 +36,15 @@ WITH
 -- When identifying donating BBLs for air rights lots, do not include 89 lots
 air_rights_counts AS (
 	SELECT 
-		a.air_rights_bbl, 
-		a.donating_bbl, 
+		a.air_lot_bbl AS air_rights_bbl,
+		a.parent_bbl AS donating_bbl,
 		b.lot_count
 	FROM dof_air_rights_lots a
-	JOIN (SELECT air_rights_bbl, COUNT(*) as lot_count
+	JOIN (SELECT air_lot_bbl, COUNT(*) as lot_count
 		FROM dof_air_rights_lots 
-		GROUP BY air_rights_bbl) b
-	ON a.air_rights_bbl = b.air_rights_bbl
-	WHERE SUBSTRING(donating_bbl, 7, 2) <> '89'
+		GROUP BY air_lot_bbl) b
+	ON a.air_lot_bbl = b.air_lot_bbl
+	WHERE SUBSTRING(parent_bbl, 7, 2) <> '89'
 )
 -- If an air rights BBL matches with multiple donating BBLs, take the lot with matching last 3 digits
 SELECT 
@@ -68,4 +68,4 @@ LEFT JOIN
     WHERE lot_count > 1 
     AND RIGHT(air_rights_bbl, 3) = RIGHT(donating_bbl, 3)
 ) b
-ON a.bbl = b.air_rights_bbl;
+ON a.bbl = b.air_rights_bbl;