add columns future_phased_units_total and completed_units to kpdb

products/knownprojects/bash/02_build.sh

@@ -44,6 +44,5 @@ run_sql_command "CALL apply_correction('${BUILD_ENGINE_SCHEMA}', 'combined', 'co
 run_sql_file sql/create_kpdb.sql
 
 echo "Generate output tables"
-run_sql_command "ALTER TABLE _kpdb RENAME COLUMN geom TO geometry;"
 run_sql_file sql/product/kpdb.sql
 run_sql_file sql/summary/summary_record_phasing.sql

products/knownprojects/sql/create_kpdb.sql

@@ -1,5 +1,6 @@
 -- create _kpdb table
 DROP TABLE IF EXISTS _kpdb_combined_and_deduped;
+DROP TABLE IF EXISTS _future_units_details;
 DROP TABLE IF EXISTS _kpdb;
 
 SELECT
@@ -32,44 +33,55 @@ WITH
 net_units_details AS (
     SELECT
         *,
-        NULLIF(units_net, 0) IS NOT NULL AS has_future_units,
-        within_5_years + from_5_to_10_years + after_10_years AS net_units_sum,
+        CASE
+            WHEN
+                status = 'DOB 5. Completed Construction'
+                THEN units_net
+            ELSE 0
+        END AS completed_units,
         units_net - (within_5_years + from_5_to_10_years + after_10_years) AS net_units_diff
     FROM _kpdb_combined_and_deduped
 ),
 
 future_units_details AS (
     SELECT
         *,
+        NULLIF(units_net, 0) IS NOT NULL AND completed_units = 0 AS has_future_units,
         CASE
             WHEN
-                has_future_units AND NOT has_project_phasing
+                NOT has_project_phasing AND completed_units = 0
                 THEN units_net
             ELSE 0
         END AS future_units_without_phasing
     FROM net_units_details
 )
 
 SELECT *
-INTO _kpdb
+INTO _future_units_details
 FROM future_units_details;
 
 --  add the net unit difference (the rounding error) to the first phase with units
-UPDATE _kpdb SET
+UPDATE _future_units_details SET
     within_5_years = within_5_years + net_units_diff,
     net_units_diff = 0
 WHERE has_future_units AND net_units_diff != 0 AND NULLIF(prop_within_5_years, 0) IS NOT NULL;
 
-UPDATE _kpdb SET
+UPDATE _future_units_details SET
     from_5_to_10_years = from_5_to_10_years + net_units_diff,
     net_units_diff = 0
 WHERE has_future_units AND net_units_diff != 0 AND NULLIF(prop_5_to_10_years, 0) IS NOT NULL;
 
-UPDATE _kpdb SET
+UPDATE _future_units_details SET
     after_10_years = after_10_years + net_units_diff,
     net_units_diff = 0
 WHERE has_future_units AND net_units_diff != 0 AND NULLIF(prop_after_10_years, 0) IS NOT NULL;
 
+SELECT
+    *,
+    within_5_years + from_5_to_10_years + after_10_years AS future_phased_units_total
+INTO _kpdb
+FROM _future_units_details;
+
 -- determine missing borough values
 UPDATE _kpdb a
 SET
@@ -93,3 +105,5 @@ SET
         WHEN a.source = 'DOB' THEN SUBSTRING(a.record_id, 1, 1)
     END
 WHERE borough IS NULL;
+
+ALTER TABLE _kpdb RENAME COLUMN geom TO geometry;

products/knownprojects/sql/product/_product_models.yml

@@ -24,7 +24,9 @@ models:
       - name: units_net
       - name: has_project_phasing
       - name: has_future_units
+      - name: future_phased_units_total
       - name: future_units_without_phasing
+      - name: completed_units
       - name: prop_within_5_years
       - name: prop_5_to_10_years
       - name: prop_after_10_years
@@ -49,18 +51,30 @@ models:
 
       # test the expectation of has_future_units
       - dbt_utils.expression_is_true:
-          name: kpdb_has_future_units
-          expression: units_net != 0
+          name: kpdb_future_units
+          expression: future_phased_units_total + future_units_without_phasing != 0
           config:
             where: has_future_units
 
-      # test the expectation of future_units_without_phasing
+      # test the expectation of has_project_phasing
       - dbt_utils.expression_is_true:
           name: kpdb_future_units_without_phasing
           expression: future_units_without_phasing = 0
           config:
             where: has_project_phasing
 
+      # test the expectation of completed_units
+      - dbt_utils.expression_is_true:
+          name: kpdb_completed_units
+          expression: completed_units = 0
+          config:
+            where: has_project_phasing
+
+      # test the sum of project unit counts
+      - dbt_utils.expression_is_true:
+          name: kpdb_sum_units
+          expression: units_net = future_phased_units_total + future_units_without_phasing + completed_units
+
       # test the sum of all project phase proportions
       - dbt_utils.expression_is_true:
           name: kpdb_sum_prop_year_buckets
@@ -71,15 +85,13 @@ models:
       # test the sum of project phase unit counts
       - dbt_utils.expression_is_true:
           name: kpdb_sum_units_year_buckets
-          expression: units_net = within_5_years + from_5_to_10_years + after_10_years
+          expression: future_phased_units_total = within_5_years + from_5_to_10_years + after_10_years
           config:
             where: has_project_phasing
 
   - name: kpdb_deduplicated
     description: "Known Projects DB project records deduplicated by Record ID"
     columns:
-      - name: project_id
-      - name: source
       - name: record_id
         tests:
           - not_null

products/knownprojects/sql/product/kpdb.sql

@@ -28,7 +28,9 @@ SELECT
     units_net,
     has_project_phasing,
     has_future_units,
+    future_phased_units_total,
     future_units_without_phasing,
+    completed_units,
     prop_within_5_years,
     prop_5_to_10_years,
     prop_after_10_years,
@@ -64,7 +66,9 @@ SELECT DISTINCT ON (record_id)
     units_net,
     has_project_phasing,
     has_future_units,
+    future_phased_units_total,
     future_units_without_phasing,
+    completed_units,
     prop_within_5_years,
     prop_5_to_10_years,
     prop_after_10_years,