diff --git a/recidiviz/calculator/query/state/views/analyst_data/analyst_data_views.py b/recidiviz/calculator/query/state/views/analyst_data/analyst_data_views.py index da3c2e7ad5..5cb43ffbd6 100644 --- a/recidiviz/calculator/query/state/views/analyst_data/analyst_data_views.py +++ b/recidiviz/calculator/query/state/views/analyst_data/analyst_data_views.py @@ -180,6 +180,9 @@ from recidiviz.calculator.query.state.views.analyst_data.us_tn.us_tn_caf_q8 import ( US_TN_CAF_Q8_VIEW_BUILDER, ) +from recidiviz.calculator.query.state.views.analyst_data.us_tn.us_tn_cellbed_assignment_raw import ( + US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_BUILDER, +) from recidiviz.calculator.query.state.views.analyst_data.us_tn.us_tn_classification_raw import ( US_TN_CLASSIFICATION_RAW_VIEW_BUILDER, ) @@ -332,4 +335,5 @@ US_TN_SEGREGATION_STAYS_VIEW_BUILDER, US_TN_CLASSIFICATION_RAW_VIEW_BUILDER, WORKFLOWS_PERSON_IMPACT_FUNNEL_STATUS_SESSIONS_VIEW_BUILDER, + US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_BUILDER, ] diff --git a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_cellbed_assignment_raw.py b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_cellbed_assignment_raw.py new file mode 100644 index 0000000000..5a8071f6cb --- /dev/null +++ b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_cellbed_assignment_raw.py @@ -0,0 +1,69 @@ +# Recidiviz - a data platform for criminal justice reform +# Copyright (C) 2024 Recidiviz, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# ============================================================================= +"""Computes current facility / unit from raw TN data""" + +from recidiviz.big_query.big_query_view import SimpleBigQueryViewBuilder +from recidiviz.calculator.query.state.dataset_config import ANALYST_VIEWS_DATASET +from recidiviz.utils.environment import GCP_PROJECT_STAGING +from recidiviz.utils.metadata import local_project_id_override + +US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_NAME = "us_tn_cellbed_assignment_raw" + +US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_DESCRIPTION = ( + """Computes current facility / unit from raw TN data""" +) + +US_TN_CELLBED_ASSIGNMENT_RAW_QUERY_TEMPLATE = """ + -- TODO(#24959): Deprecate usage when re-run is over + -- TODO(#27428): Once source of facility ID in TN is reconciled, this can be removed + SELECT + state_code, + person_id, + /* From TN: "The ‘Requested’ location columns contain the new housing assignment + (where the person is being moved to). The ‘Assigned’ columns show the person’s assigned bed + when the new cell bed assignment was entered. Same for the ‘Actual’ columns – this is where he + ‘actually’ was when the request was made (this is only different from Assigned when the person + has been moved to another institution temporarily)." Our interpretation of this is that Requested and + Actual work sometimes together and sometimes on a "lag"; Actual should be updated to show the same + thing as Requested, but sometimes it isnt, so we rely on Requested most, then Actual. + For current population, Requested is always hydrated, so the COALESCE is not strictly + needed but is a catch all if Requested is ever missing */ + COALESCE(RequestedSiteID, ActualSiteID, AssignedSiteID) AS facility_id, + COALESCE(RequestedUnitID, ActualUnitID, AssignedUnitID) AS unit_id, + FROM `{project_id}.us_tn_raw_data_up_to_date_views.CellBedAssignment_latest` c + INNER JOIN `{project_id}.normalized_state.state_person_external_id` pei + ON c.OffenderID = pei.external_id + AND pei.state_code = "US_TN" + -- The latest assignment is not always the one with an open assignment. This can occur when someone is assigned to a facility + -- but temporarily sent to another facility (e.g. Special needs facility). Most people only have 1 open assignment, unless + -- they are currently temporarily sent elsewhere (~11 people out of 18k) so we further deduplicate by choosing the latest assignment + WHERE EndDate IS NULL + QUALIFY ROW_NUMBER() OVER(PARTITION BY OffenderID ORDER BY CAST(AssignmentDateTime AS DATETIME) DESC) = 1 + +""" + +US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_BUILDER = SimpleBigQueryViewBuilder( + dataset_id=ANALYST_VIEWS_DATASET, + view_id=US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_NAME, + description=US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_DESCRIPTION, + view_query_template=US_TN_CELLBED_ASSIGNMENT_RAW_QUERY_TEMPLATE, + should_materialize=True, +) + +if __name__ == "__main__": + with local_project_id_override(GCP_PROJECT_STAGING): + US_TN_CELLBED_ASSIGNMENT_RAW_VIEW_BUILDER.build_and_print() diff --git a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_max_stays.py b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_max_stays.py index 0e83cf2ef9..b8c5e8380b 100644 --- a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_max_stays.py +++ b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_max_stays.py @@ -45,17 +45,8 @@ INNER JOIN `{project_id}.normalized_state.state_person` USING(person_id, state_code) -- TODO(#27428): Remove this join when custody level information aligns with location information - INNER JOIN ( - SELECT - OffenderID AS external_id, - COALESCE(RequestedSiteID, ActualSiteID, AssignedSiteID) AS facility_id, - COALESCE(RequestedUnitID, ActualUnitID, AssignedUnitID) AS unit_id, - FROM `{project_id}.us_tn_raw_data_up_to_date_views.CellBedAssignment_latest` - -- Ensures that someone is actively assigned to a TDOC bed at the moment - WHERE EndDate IS NULL - QUALIFY ROW_NUMBER() OVER(PARTITION BY OffenderID ORDER BY CAST(AssignmentDateTime AS DATETIME) DESC) = 1 - ) - USING(external_id) + INNER JOIN `{{project_id}}.analyst_data.us_tn_cellbed_assignment_raw_materialized` + USING(person_id, state_code) WHERE c.state_code = 'US_TN' AND c.custody_level = 'MAXIMUM' AND c.end_date_exclusive is null diff --git a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_prior_record_preprocessed.py b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_prior_record_preprocessed.py index 046d5fcd9f..8004f530c0 100644 --- a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_prior_record_preprocessed.py +++ b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_prior_record_preprocessed.py @@ -47,7 +47,7 @@ pei.person_id, pei.state_code, ofs.description, - DispositionDate AS disposition_date, + CAST(CAST(DispositionDate AS DATETIME) AS DATE) AS disposition_date, CAST(CAST(ArrestDate AS DATETIME) AS DATE) AS sentence_effective_date, CAST(CAST(OffenseDate AS DATETIME) AS DATE) AS offense_date, CourtName AS conviction_county, diff --git a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_segregation_stays.py b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_segregation_stays.py index bcc24d25f7..19d11d4a6a 100644 --- a/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_segregation_stays.py +++ b/recidiviz/calculator/query/state/views/analyst_data/us_tn/us_tn_segregation_stays.py @@ -54,32 +54,15 @@ s.SegragationReason AS segregation_reason, s.SegregationStatus AS segregation_status, s.SegregationType AS segregation_type, - c.current_facility_id, - c.current_unit_id + c.facility_id AS current_facility_id, + c.unit_id AS current_unit_id, FROM `{{project_id}}.us_tn_raw_data_up_to_date_views.Segregation_latest` s INNER JOIN `{{project_id}}.normalized_state.state_person_external_id` pei ON s.OffenderID = pei.external_id AND pei.state_code = 'US_TN' -- TODO(#27428): Remove this join when custody level information aligns with location information - LEFT JOIN ( - SELECT - OffenderID AS external_id, - /* From TN: "The ‘Requested’ location columns contain the new housing assignment - (where the person is being moved to). The ‘Assigned’ columns show the person’s assigned bed - when the new cell bed assignment was entered. Same for the ‘Actual’ columns – this is where he - ‘actually’ was when the request was made (this is only different from Assigned when the person - has been moved to another institution temporarily)." Our interpretation of this is that Requested and - Actual work sometimes together and sometimes on a "lag"; Actual should be updated to show the same - thing as Requested, but sometimes it isnt, so we rely on Requested most, then Actual. - For current population, Requested is always hydrated, so the COALESCE is not strictly - needed but is a catch all if Requested is ever missing */ - COALESCE(RequestedSiteID, ActualSiteID, AssignedSiteID) AS current_facility_id, - COALESCE(RequestedUnitID, ActualUnitID, AssignedUnitID) AS current_unit_id, - FROM `{{project_id}}.us_tn_raw_data_up_to_date_views.CellBedAssignment_latest` - WHERE EndDate IS NULL - QUALIFY ROW_NUMBER() OVER(PARTITION BY OffenderID ORDER BY CAST(AssignmentDateTime AS DATETIME) DESC) = 1 - ) c - USING(external_id) + LEFT JOIN `{{project_id}}.analyst_data.us_tn_cellbed_assignment_raw_materialized` c + USING(person_id, state_code) -- There are a very small number of duplicates on person id and start date in the segregation table QUALIFY ROW_NUMBER() OVER(PARTITION BY pei.person_id, start_date ORDER BY end_date DESC) = 1 ), diff --git a/recidiviz/calculator/query/state/views/workflows/firestore/resident_record.py b/recidiviz/calculator/query/state/views/workflows/firestore/resident_record.py index 90954bc482..8ebeda8727 100644 --- a/recidiviz/calculator/query/state/views/workflows/firestore/resident_record.py +++ b/recidiviz/calculator/query/state/views/workflows/firestore/resident_record.py @@ -84,9 +84,6 @@ us_me_raw_data_up_to_date_dataset=raw_latest_views_dataset_for_region( state_code=StateCode.US_ME, instance=DirectIngestInstance.PRIMARY ), - us_tn_raw_data_up_to_date_dataset=raw_latest_views_dataset_for_region( - state_code=StateCode.US_TN, instance=DirectIngestInstance.PRIMARY - ), us_me_task_eligibility_criteria_dataset=task_eligibility_criteria_state_specific_dataset( StateCode.US_ME ), diff --git a/recidiviz/calculator/query/state/views/workflows/firestore/resident_record_ctes.py b/recidiviz/calculator/query/state/views/workflows/firestore/resident_record_ctes.py index 37c47f1ffa..79e1dd095b 100644 --- a/recidiviz/calculator/query/state/views/workflows/firestore/resident_record_ctes.py +++ b/recidiviz/calculator/query/state/views/workflows/firestore/resident_record_ctes.py @@ -57,12 +57,46 @@ _RESIDENT_RECORD_INCARCERATION_DATES_CTE = f""" incarceration_dates AS ( + -- Adding a TN specific admission date that is admission to TDOC facility, in addition to overall incarceration + -- admission date + SELECT + ic.*, + MAX(t.projected_completion_date_max) + OVER(w) AS release_date, + MAX(c.start_date) + OVER(w) AS us_tn_facility_admission_date, + FROM + incarceration_cases ic + LEFT JOIN ( + SELECT cs.person_id, cs.state_code, cs.start_date + FROM `{{project_id}}.{{sessions_dataset}}.compartment_sub_sessions_materialized` cs + INNER JOIN `{{project_id}}.reference_views.location_metadata_materialized` + ON facility = location_external_id + WHERE cs.state_code = 'US_TN' + AND compartment_level_1 = 'INCARCERATION' + AND compartment_level_2 = 'GENERAL' + AND location_type = 'STATE_PRISON' + AND CURRENT_DATE('US/Eastern') BETWEEN cs.start_date + AND {nonnull_end_date_clause('cs.end_date_exclusive')} + ) c + USING(person_id, state_code) + LEFT JOIN `{{project_id}}.{{sessions_dataset}}.incarceration_projected_completion_date_spans_materialized` t + ON ic.person_id = t.person_id + AND ic.state_code = t.state_code + AND CURRENT_DATE('US/Eastern') + BETWEEN t.start_date AND {nonnull_end_date_clause('t.end_date_exclusive')} + WHERE ic.state_code IN ("US_TN") + WINDOW w as (PARTITION BY ic.person_id) + + UNION ALL + SELECT ic.* EXCEPT(admission_date), MAX(t.start_date) OVER(w) AS admission_date, MAX({nonnull_end_date_clause('t.end_date')}) - OVER(w) AS release_date + OVER(w) AS release_date, + CAST(NULL AS DATE) AS us_tn_facility_admission_date, --TODO(#16175) ingest intake and release dates FROM incarceration_cases ic @@ -83,7 +117,8 @@ SELECT ic.* EXCEPT(admission_date), NULL AS admission_date, - NULL AS release_date + NULL AS release_date, + CAST(NULL AS DATE) AS us_tn_facility_admission_date, FROM incarceration_cases ic WHERE state_code="US_MO" @@ -92,7 +127,8 @@ SELECT ic.*, MAX(t.projected_completion_date_max) - OVER(w) AS release_date + OVER(w) AS release_date, + CAST(NULL AS DATE) AS us_tn_facility_admission_date, FROM incarceration_cases ic LEFT JOIN `{{project_id}}.{{sessions_dataset}}.incarceration_projected_completion_date_spans_materialized` t @@ -100,7 +136,7 @@ AND ic.state_code = t.state_code AND CURRENT_DATE('US/Eastern') BETWEEN t.start_date AND {nonnull_end_date_clause('t.end_date_exclusive')} - WHERE ic.state_code NOT IN ("US_ME", "US_MO") + WHERE ic.state_code NOT IN ("US_ME", "US_MO", "US_TN") WINDOW w as (PARTITION BY ic.person_id) ), """ @@ -112,7 +148,7 @@ {revert_nonnull_start_date_clause('admission_date')} AS admission_date, {revert_nonnull_end_date_clause('release_date')} AS release_date FROM incarceration_dates - GROUP BY 1,2,3,4,5,6,7,8 + GROUP BY 1,2,3,4,5,6,7,8,9 ), """ @@ -167,7 +203,6 @@ housing_unit AS ( SELECT person_id, - -- TODO(#27428): Once source of facility ID in TN is reconciled, this can be removed CAST(NULL AS STRING) AS facility_id, housing_unit AS unit_id FROM `{{project_id}}.{{normalized_state_dataset}}.state_incarceration_period` @@ -180,28 +215,19 @@ SELECT person_id, - -- TODO(#27428): Once source of facility ID in TN is reconciled, this can be removed CAST(NULL AS STRING) AS facility_id, IF(complex_number=building_number, complex_number, complex_number || " " || building_number) AS unit_id FROM current_bed_stay UNION ALL - --TODO(#24959): Deprecate usage when re-run is over + -- TODO(#27428): Once source of facility ID in TN is reconciled, this can be removed SELECT person_id, - -- TODO(#27428): Once source of facility ID in TN is reconciled, this can be removed - COALESCE(RequestedSiteID, ActualSiteID, AssignedSiteID) AS facility_id, - COALESCE(RequestedUnitID, ActualUnitID, AssignedUnitID) AS unit_id, - FROM `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.CellBedAssignment_latest` c - INNER JOIN `{{project_id}}.{{normalized_state_dataset}}.state_person_external_id` pei - ON c.OffenderID = pei.external_id - AND pei.state_code = "US_TN" - -- The latest assignment is not always the one with an open assignment. This can occur when someone is assigned to a facility - -- but temporarily sent to another facility (e.g. Special needs facility). Most people only have 1 open assignment, unless - -- they are currently temporarily sent elsewhere (~11 people out of 18k) so we further deduplicate by choosing the latest assignment - WHERE EndDate IS NULL - QUALIFY ROW_NUMBER() OVER(PARTITION BY OffenderID ORDER BY CAST(AssignmentDateTime AS DATETIME) DESC) = 1 + facility_id, + unit_id + FROM `{{project_id}}.analyst_data.us_tn_cellbed_assignment_raw_materialized` + ), """ @@ -279,6 +305,7 @@ custody_level.custody_level, ic.admission_date, ic.release_date, + ic.us_tn_facility_admission_date, FROM incarceration_cases_wdates ic LEFT JOIN custody_level @@ -306,6 +333,7 @@ custody_level, admission_date, release_date, + us_tn_facility_admission_date, opportunities_aggregated.all_eligible_opportunities, portion_served_needed, GREATEST(portion_needed_eligible_date, months_remaining_eligible_date) AS sccp_eligibility_date, diff --git a/recidiviz/task_eligibility/utils/us_tn_query_fragments.py b/recidiviz/task_eligibility/utils/us_tn_query_fragments.py index d3d6993189..1bb204e48f 100644 --- a/recidiviz/task_eligibility/utils/us_tn_query_fragments.py +++ b/recidiviz/task_eligibility/utils/us_tn_query_fragments.py @@ -23,9 +23,6 @@ aggregate_adjacent_spans, create_sub_sessions_with_attributes, ) -from recidiviz.task_eligibility.utils.state_dataset_query_fragments import ( - get_sentences_current_span, -) DISCIPLINARY_HISTORY_MONTH_LOOKBACK = "60" @@ -266,15 +263,7 @@ def us_tn_classification_forms( disciplinary_history_month_lookback: str = DISCIPLINARY_HISTORY_MONTH_LOOKBACK, ) -> str: return f""" - WITH current_offenses AS ( - SELECT - person_id, - ARRAY_AGG(offense IGNORE NULLS) AS form_information_current_offenses, - FROM - ({get_sentences_current_span(in_projected_completion_array=True)}) - GROUP BY 1 - ), - latest_classification AS ( + WITH latest_classification AS ( SELECT person_id, external_id, DATE(ClassificationDate) AS form_information_latest_classification_date, @@ -332,22 +321,55 @@ def us_tn_classification_forms( `{{project_id}}.{{analyst_dataset}}.incarceration_incidents_preprocessed_materialized` dis WHERE state_code = "US_TN" - ), - -- This CTE only keeps disciplinaries with an assault, for Q1 and Q2 info - assaultive_disciplinary_history AS ( + ), + -- Union case notes + case_notes_cte AS ( + -- This only keeps disciplinaries with an assault, for Q1 and Q2 info SELECT person_id, - TO_JSON( - ARRAY_AGG( - STRUCT(note_title, note_body, event_date, "ASSAULTIVE DISCIPLINARIES" AS criteria) - ORDER BY event_date DESC) - ) AS case_notes - FROM + "ASSAULTIVE DISCIPLINARIES" AS criteria, + event_date, + note_title, + note_body, + FROM disciplinaries WHERE assault_score IS NOT NULL AND event_date >= DATE_SUB(CURRENT_DATE('US/Eastern'), INTERVAL {disciplinary_history_month_lookback} MONTH) - GROUP BY 1 + + UNION ALL + + SELECT + person_id, + "PRIOR RECORD OFFENSES" AS criteria, + disposition_date AS event_date, + description AS note_title, + CAST(NULL AS STRING) AS note_body, + FROM `{{project_id}}.{{analyst_dataset}}.us_tn_prior_record_preprocessed_materialized` + + UNION ALL + + SELECT + person_id, + "TN, ISC, DIVERSION SENTENCES" AS criteria, + date_imposed AS event_date, + description AS note_title, + CAST(projected_completion_date_max AS STRING) AS note_body, + FROM `{{project_id}}.sessions.sentences_preprocessed_materialized` + WHERE state_code = "US_TN" + ), + case_notes_array AS ( + SELECT + person_id, + -- Group all notes into an array within a JSON + TO_JSON( + ARRAY_AGG( + STRUCT(note_title, note_body, event_date, criteria) + ) + ) AS case_notes, + FROM + case_notes_cte + GROUP BY 1 ), -- This CTE only keeps guilty disciplinaries, for Q6 and Q7 info guilty_disciplinaries AS ( @@ -356,76 +378,14 @@ def us_tn_classification_forms( person_id, event_date, incident_class, - MAX(event_date) OVER(PARTITION BY person_id) AS latest_disciplinary, note_body, FROM disciplinaries - -- For q7, we only want to consider guilty disciplinaries with non-missing disciplinary class + -- For q6 and q7, we only want to consider guilty disciplinaries with non-missing disciplinary class WHERE disposition = 'GU' AND note_body != "" AND incident_details NOT LIKE "%VERBAL WARNING%" - ), - guilty_disciplinary_history AS ( - SELECT - person_id, - TO_JSON( - ARRAY_AGG( - CASE WHEN event_date = latest_disciplinary - THEN STRUCT(note_body, event_date) - END - IGNORE NULLS) - ) AS latest_disciplinary, - TO_JSON( - ARRAY_AGG( - CASE WHEN event_date > DATE_SUB(CURRENT_DATE('US/Eastern'), INTERVAL 6 MONTH) - THEN STRUCT(note_body, event_date) - END - IGNORE NULLS - ORDER BY event_date DESC) - ) AS case_notes_guilty_disciplinaries_last_6_month, - FROM - guilty_disciplinaries - GROUP BY 1 ), - most_serious_disciplinaries AS ( - SELECT - person_id, - TO_JSON( - ARRAY_AGG( - STRUCT(event_date, note_body) - ORDER BY event_date DESC - ) - - ) AS form_information_q7_notes - FROM - ( - SELECT * - FROM guilty_disciplinaries - WHERE event_date >= DATE_SUB(CURRENT_DATE('US/Eastern'), INTERVAL 18 MONTH) - QUALIFY RANK() OVER(PARTITION BY person_id ORDER BY CASE WHEN incident_class = 'A' THEN 1 - WHEN incident_class = 'B' THEN 2 - WHEN incident_class = 'C' THEN 3 - END ASC) = 1 - ) - GROUP BY 1 - ), - detainers_cte AS ( - SELECT person_id, - TO_JSON( - ARRAY_AGG( - STRUCT(start_date AS detainer_received_date, - detainer_felony_flag AS detainer_felony_flag, - detainer_misdemeanor_flag AS detainer_misdemeanor_flag, - jurisdiction AS jurisdiction, - description AS description, - charge_pending AS charge_pending - ) - ) - ) AS form_information_q8_notes - FROM ({detainers_cte()}) - WHERE end_date IS NULL - GROUP BY 1 - ), recommended_scores AS ( -- Schedule B is only scored if Schedule A is 9 or less, so this CTE "re scores" schedule B scores if needed SELECT * @@ -456,15 +416,35 @@ def us_tn_classification_forms( ) ), q6_score_info AS ( - SELECT *, - CASE WHEN form_information_q6_score IN (-4, -2, -1) THEN latest_disciplinary - WHEN form_information_q6_score IN (1,4) THEN case_notes_guilty_disciplinaries_last_6_month - END AS form_information_q6_notes - FROM - recommended_scores - LEFT JOIN - guilty_disciplinary_history - USING(person_id) + SELECT person_id, + TO_JSON( + ARRAY_AGG( + STRUCT(note_body, event_date) + IGNORE NULLS + ORDER BY event_date DESC) + ) AS form_information_q6_notes + FROM ( + SELECT * + FROM guilty_disciplinaries + WHERE event_date >= DATE_SUB(CURRENT_DATE('US/Eastern'), INTERVAL 6 MONTH) + QUALIFY ROW_NUMBER() OVER(PARTITION BY person_id ORDER BY event_date DESC) <= 2 + ) + GROUP BY 1 + ), + q7_score_info AS ( + SELECT + person_id, + TO_JSON(STRUCT(event_date, note_body)) AS form_information_q7_notes + FROM guilty_disciplinaries + WHERE event_date >= DATE_SUB(CURRENT_DATE('US/Eastern'), INTERVAL 18 MONTH) + -- Keep most serious incident. If there are duplicates, keep latest incident within that class + QUALIFY ROW_NUMBER() OVER(PARTITION BY person_id + ORDER BY CASE WHEN incident_class = 'A' THEN 1 + WHEN incident_class = 'B' THEN 2 + WHEN incident_class = 'C' THEN 3 + END ASC, + event_date DESC + ) = 1 ), level_care AS ( SELECT @@ -494,7 +474,9 @@ def us_tn_classification_forms( r.TreatmentGoal, DATE(r.RecommendationDate) AS RecommendationDate, DATE(r.RecommendedEndDate) AS RecommendedEndDate, - pr.VantagePointTitle + pr.VantagePointTitle, + c.latest_refusal_date, + c.latest_refusal_contact FROM `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.VantagePointAssessments_latest` a LEFT JOIN `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.VantagePointRecommendations_latest` r ON a.OffenderID = r.OffenderID @@ -504,6 +486,17 @@ def us_tn_classification_forms( USING(Recommendation, Pathway) LEFT JOIN `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.VantagePointProgram_latest` pr ON pr.VantageProgramID = r.VantagePointProgamID + LEFT JOIN ( + SELECT + OffenderID, + CAST(CAST(ContactNoteDateTime AS datetime) AS DATE) AS latest_refusal_date, + ContactNoteType AS latest_refusal_contact + FROM `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.ContactNoteType_latest` + WHERE ContactNoteType IN ("IRAR", "XRIS", "CCMC") + QUALIFY ROW_NUMBER() OVER(PARTITION BY OffenderID ORDER BY CAST(CAST(ContactNoteDateTime AS datetime) AS DATE) DESC) = 1 + ) c + ON a.OffenderID = c.OffenderID + AND latest_refusal_date >= DATE(a.CompletedDate) ), active_vantage_recommendations AS ( SELECT @@ -520,7 +513,23 @@ def us_tn_classification_forms( FROM vantage WHERE Recommendation IS NOT NULL GROUP BY 1 - + ), + detainers_cte AS ( + SELECT person_id, + TO_JSON( + ARRAY_AGG( + STRUCT(start_date AS detainer_received_date, + detainer_felony_flag AS detainer_felony_flag, + detainer_misdemeanor_flag AS detainer_misdemeanor_flag, + jurisdiction AS jurisdiction, + description AS description, + charge_pending AS charge_pending + ) + ) + ) AS form_information_q8_notes + FROM ({detainers_cte()}) + WHERE end_date IS NULL + GROUP BY 1 ) SELECT tes.state_code, tes.reasons, @@ -531,10 +540,9 @@ def us_tn_classification_forms( latest_classification.form_information_latest_override_reason, latest_CAF.form_information_last_CAF_date, latest_CAF.form_information_last_CAF_total, - current_offenses.form_information_current_offenses, - assaultive_disciplinary_history.case_notes, + case_notes_array.case_notes, q6_score_info.form_information_q6_notes, - most_serious_disciplinaries.form_information_q7_notes, + q7_score_info.form_information_q7_notes, detainers_cte.form_information_q8_notes, recommended_scores.* EXCEPT(person_id), stg.STGID AS form_information_gang_affiliation_id, @@ -549,7 +557,11 @@ def us_tn_classification_forms( ELSE 'GEN' END AS form_information_status_at_hearing_seg, CASE WHEN tes.task_name = 'ANNUAL_RECLASSIFICATION_REVIEW' THEN 'ANNUAL' ELSE 'SPECIAL' END AS form_information_classification_type, - latest_vantage.RiskLevel AS form_information_latest_vantage_risk_level, + -- if there has been a refusal more recently than the latest assessment completion date, then we want to show + -- the contact note / date rather than the risk level + IF(latest_vantage.latest_refusal_date IS NULL, + latest_vantage.RiskLevel, + CONCAT(latest_vantage.latest_refusal_contact, ', ', latest_vantage.latest_refusal_date)) AS form_information_latest_vantage_risk_level, latest_vantage.CompletedDate AS form_information_latest_vantage_completed_date, active_vantage_recommendations.active_recommendations AS form_information_active_recommendations, incompatible.incompatible_array AS form_information_incompatible_array, @@ -560,9 +572,6 @@ def us_tn_classification_forms( INNER JOIN `{{project_id}}.{{normalized_state_dataset}}.state_person_external_id` pei USING(person_id) - LEFT JOIN - current_offenses - USING(person_id) LEFT JOIN latest_classification USING(person_id) @@ -573,13 +582,10 @@ def us_tn_classification_forms( recommended_scores USING(person_id) LEFT JOIN - assaultive_disciplinary_history - USING(person_id) - LEFT JOIN - guilty_disciplinary_history + case_notes_array USING(person_id) LEFT JOIN - most_serious_disciplinaries + q7_score_info USING(person_id) LEFT JOIN detainers_cte @@ -608,10 +614,18 @@ def us_tn_classification_forms( SELECT DISTINCT OffenderID, CASE WHEN IncompatibleType = 'S' THEN 'STAFF' - ELSE IncompatibleOffenderID + ELSE facility_id END AS incompatible_ids, IncompatibleType, - FROM `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.IncompatiblePair_latest` + FROM `{{project_id}}.{{us_tn_raw_data_up_to_date_dataset}}.IncompatiblePair_latest` i + INNER JOIN + `{{project_id}}.{{normalized_state_dataset}}.state_person_external_id` pei + ON + i.IncompatibleOffenderID = pei.external_id + AND + pei.state_code = 'US_TN' + INNER JOIN `{{project_id}}.analyst_data.us_tn_cellbed_assignment_raw_materialized` + USING(person_id, state_code) WHERE IncompatibleRemovedDate IS NULL ) GROUP BY 1 diff --git a/recidiviz/tests/workflows/etl/workflows_resident_etl_delegate_test.py b/recidiviz/tests/workflows/etl/workflows_resident_etl_delegate_test.py index b08974834a..0adf41e127 100644 --- a/recidiviz/tests/workflows/etl/workflows_resident_etl_delegate_test.py +++ b/recidiviz/tests/workflows/etl/workflows_resident_etl_delegate_test.py @@ -94,6 +94,7 @@ def test_transform_row(self) -> None: "releaseDate": "2027-03-28", "sccpEligibilityDate": None, "allEligibleOpportunities": ["usMoRestrictiveHousingStatusHearing"], + "usTnFacilityAdmissionDate": None, }, row, ) @@ -131,6 +132,7 @@ def test_transform_row(self) -> None: "usMeWorkRelease", "usMeFurloughRelease", ], + "usTnFacilityAdmissionDate": None, }, row, ) @@ -167,6 +169,7 @@ def test_transform_row(self) -> None: "usMeWorkRelease", "usMeFurloughRelease", ], + "usTnFacilityAdmissionDate": None, }, row, ) @@ -200,6 +203,7 @@ def test_transform_row(self) -> None: "releaseDate": "2024-05-01", "sccpEligibilityDate": None, "allEligibleOpportunities": ["usTnCustodyLevelDowngrade"], + "usTnFacilityAdmissionDate": None, }, row, ) diff --git a/recidiviz/view_registry/raw_data_reference_reasons.yaml b/recidiviz/view_registry/raw_data_reference_reasons.yaml index 8036e0ec0c..cc647faba4 100644 --- a/recidiviz/view_registry/raw_data_reference_reasons.yaml +++ b/recidiviz/view_registry/raw_data_reference_reasons.yaml @@ -1144,12 +1144,8 @@ US_TN: workflows_views.client_record: |- Usage reason unknown. Grandfathered in 04/18/2024. CellBedAssignment: - analyst_data.us_tn_max_stays: |- - Usage reason unknown. Grandfathered in 04/18/2024. - analyst_data.us_tn_segregation_stays: |- - Usage reason unknown. Grandfathered in 04/18/2024. - workflows_views.resident_record: |- - Usage reason unknown. Grandfathered in 04/18/2024. + analyst_data.us_tn_cellbed_assignment_raw: |- + Raw data used while ingest issues are being resolved (TODO(#24959) and TODO(#27428)). Classification: analyst_data.us_tn_classification_raw: |- Usage reason unknown. Grandfathered in 04/18/2024. @@ -1197,6 +1193,10 @@ US_TN: Usage reason unknown. Grandfathered in 04/18/2024. task_eligibility_criteria_us_tn.special_conditions_are_current: |- Usage reason unknown. Grandfathered in 04/18/2024. + workflows_views.us_tn_annual_reclassification_review_record: |- + Needed to pull risk assessment refusal codes and we don't ingest contact information. + workflows_views.us_tn_custody_level_downgrade_record: |- + Needed to pull risk assessment refusal codes and we don't ingest contact information. workflows_views.us_tn_transfer_to_compliant_reporting_record: |- Usage reason unknown. Grandfathered in 04/18/2024. DailyCommunitySupervisionForRecidiviz: diff --git a/recidiviz/workflows/etl/workflows_resident_etl_delegate.py b/recidiviz/workflows/etl/workflows_resident_etl_delegate.py index db61169260..59f216e632 100644 --- a/recidiviz/workflows/etl/workflows_resident_etl_delegate.py +++ b/recidiviz/workflows/etl/workflows_resident_etl_delegate.py @@ -52,6 +52,7 @@ def transform_row(self, row: str) -> Tuple[str, dict]: "releaseDate": data.get("release_date"), "portionServedNeeded": data.get("portion_served_needed"), "sccpEligibilityDate": data.get("sccp_eligibility_date"), + "usTnFacilityAdmissionDate": data.get("us_tn_facility_admission_date"), } if "all_eligible_opportunities" in data: