From b871ea07c8a0cd0ef22420d2f1004676aee88463 Mon Sep 17 00:00:00 2001 From: fumanjie Date: Fri, 12 Sep 2025 12:20:45 -0400 Subject: [PATCH 1/3] DBDAART-15627 Convert PRV build process from using TMSIS tables to using TMSIS views. --- taf/PRV/PRV.py | 30 +++++++++++++----- taf/PRV/PRV01.py | 2 +- taf/PRV/PRV02.py | 52 +++++++++++++++---------------- taf/PRV/PRV03.py | 76 +++++++++++++++++++++++----------------------- taf/PRV/PRV04.py | 28 ++++++++--------- taf/PRV/PRV05.py | 28 ++++++++--------- taf/PRV/PRV06.py | 36 +++++++++++----------- taf/PRV/PRV07.py | 30 +++++++++--------- taf/PRV/PRV08.py | 22 +++++++------- taf/PRV/PRV09.py | 24 +++++++-------- taf/PRV/PRV10.py | 26 ++++++++-------- taf/TAF_Closure.py | 16 +++++++--- 12 files changed, 197 insertions(+), 173 deletions(-) diff --git a/taf/PRV/PRV.py b/taf/PRV/PRV.py index 2c0570f5..68ae32e4 100644 --- a/taf/PRV/PRV.py +++ b/taf/PRV/PRV.py @@ -70,14 +70,29 @@ def screen_runid(self, intbl, runtbl, runvars, outtbl, runtyp='C'): T.submitting_state = R.submitting_state and upper(T.submitting_state_prov_id) = R.submitting_state_prov_id""" + # retain tmsis table column names used in subsequent merge and in TAF_Metadata.py + renames = [ + 'tmsis_run_id as tms_run_id', + 'submtg_state_cd as submitting_state', + 'submtg_state_prvdr_id as submitting_state_prov_id', + ] + + if runtyp == 'L': + renames.append('prvdr_lctn_id as prov_location_id') + # diststyle key distkey(submitting_state_prov_id) # compound sortkey (&&&runvars) as z = f""" create or replace temporary view {outtbl} as select T.* - from - {intbl} T + from ( + select + *, + { ','.join(renames) } + from + {intbl} + ) T inner join {runtbl} R { on.format(runvars) } order by @@ -105,7 +120,7 @@ def copy_activerows(self, intbl, collist, whr, outtbl): from { intbl } where - tms_is_active=1 + tmsis_actv_ind=1 { whr } order by tms_run_id, @@ -125,14 +140,15 @@ def copy_activerows_nts(self, intbl, collist, outtbl): from ( select *, - submitting_state as submtg_state_cd + tmsis_run_id as tms_run_id, + submtg_state_cd as submitting_state from {intbl} where - tms_is_active = 1 and - tms_reporting_period is not null and + tmsis_actv_ind = 1 and + tmsis_rptg_prd is not null and tot_rec_cnt > 0 and - trim(TRAILING FROM submitting_state) not in ('94','96') + trim(TRAILING FROM submtg_state_cd) not in ('94','96') ) where 1=1 { self.prv.ST_FILTER() } diff --git a/taf/PRV/PRV01.py b/taf/PRV/PRV01.py index 2a5d3537..77575525 100644 --- a/taf/PRV/PRV01.py +++ b/taf/PRV/PRV01.py @@ -18,7 +18,7 @@ def process_01_header(self, outtbl): cols01 = ['tms_run_id', 'submitting_state'] # copy provider header table - self.copy_activerows_nts("tmsis.File_Header_Record_Provider", cols01, 'Prov01_Header_Copy') + self.copy_activerows_nts("tmsis.Tmsis_Fhdr_Rec_Prvdr", cols01, 'Prov01_Header_Copy') # self.prv.countrows('Prov01_Header_Copy', 'cnt_active', 'PRV01_Active') diff --git a/taf/PRV/PRV02.py b/taf/PRV/PRV02.py index bac95209..1be957e8 100644 --- a/taf/PRV/PRV02.py +++ b/taf/PRV/PRV02.py @@ -20,7 +20,7 @@ def process_02_main(self, runtbl, outtbl): # screen out all but the latest run id runlist = ['tms_run_id', 'submitting_state'] - self.screen_runid('tmsis.Prov_Attributes_Main', + self.screen_runid('tmsis.Tmsis_Prvdr_Attr_Mn', runtbl, runlist, 'Prov02_Main_Latest1', @@ -31,33 +31,33 @@ def process_02_main(self, runtbl, outtbl): 'cnt_latest', 'PRV02_Latest') - cols02 = ['tms_run_id', - 'tms_reporting_period', - 'submitting_state', - 'submitting_state as submtg_state_cd', - 'record_number', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - 'prov_attributes_eff_date', - 'prov_attributes_end_date', - 'prov_doing_business_as_name', - 'prov_legal_name', - 'prov_organization_name', - 'prov_tax_name', - 'facility_group_individual_code', - 'teaching_ind', - 'prov_first_name', - 'prov_middle_initial', - 'prov_last_name', - 'sex', - 'ownership_code', - 'prov_profit_status', - '%fix_old_dates(date_of_birth)', - '%fix_old_dates(date_of_death)', - 'accepting_new_patients_ind', - 'atypical_prov_ind', + cols02 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + 'rec_num as record_number', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + 'prvdr_attr_efctv_dt as prov_attributes_eff_date', + 'prvdr_attr_end_dt as prov_attributes_end_date', + 'prvdr_dba_name as prov_doing_business_as_name', + 'prvdr_lgl_name as prov_legal_name', + 'prvdr_org_name as prov_organization_name', + 'prvdr_tax_name as prov_tax_name', + 'fac_grp_indvdl_cd as facility_group_individual_code', + 'tchng_ind as teaching_ind', + 'prvdr_1st_name as prov_first_name', + 'prvdr_mdl_initl_name as prov_middle_initial', + 'prvdr_last_name as prov_last_name', + 'sex_cd as sex', + 'ownrshp_cd as ownership_code', + 'prvdr_prft_stus_cd as prov_profit_status', + '%fix_old_dates(birth_dt, date_of_birth)', + '%fix_old_dates(death_dt, date_of_death)', + 'acpt_new_ptnts_ind as accepting_new_patients_ind', + 'atyp_prov_ind as atypical_prov_ind', ] - whr02 = 'upper(submitting_state_prov_id) is not null' + whr02 = 'upper(submtg_state_prvdr_id) is not null' self.copy_activerows('Prov02_Main_Latest1', cols02, diff --git a/taf/PRV/PRV03.py b/taf/PRV/PRV03.py index d7d993b5..1398d656 100644 --- a/taf/PRV/PRV03.py +++ b/taf/PRV/PRV03.py @@ -18,7 +18,7 @@ def process_03_locations(self, maintbl, outtbl): 'submitting_state', 'submitting_state_prov_id'] - self.screen_runid('tmsis.Prov_Location_And_Contact_Info', + self.screen_runid('tmsis.Tmsis_Prvdr_Lctn_Cntct', maintbl, runlist, 'Prov03_Locations_Latest1') @@ -26,27 +26,27 @@ def process_03_locations(self, maintbl, outtbl): # row count # self.prv.countrows(Prov03_Locations_Latest1, cnt_latest, PRV03_Latest) - cols03 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(prov_location_id) as prov_location_id', - 'prov_addr_type', - 'prov_location_and_contact_info_eff_date', - 'prov_location_and_contact_info_end_date', - 'addr_ln1', - 'addr_ln2', - 'addr_ln3', - 'addr_city', - '%upper_case(addr_state) as addr_state', - 'addr_zip_code', - 'addr_county', - 'addr_border_state_ind'] + cols03 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(prvdr_lctn_id) as prov_location_id', + 'prvdr_adr_type_cd as prov_addr_type', + 'prvdr_lctn_cntct_efctv_dt as prov_location_and_contact_info_eff_date', + 'prvdr_lctn_cntct_end_dt as prov_location_and_contact_info_end_date', + 'adr_line_1_txt as addr_ln1', + 'adr_line_2_txt as addr_ln2', + 'adr_line_3_txt as addr_ln3', + 'adr_city_name as addr_city', + '%upper_case(adr_state_cd) as addr_state', + 'adr_zip_cd as addr_zip_code', + 'adr_cnty_cd as addr_county', + 'adr_brdr_state_ind as addr_border_state_ind'] # copy 03(Location) provider rows - whr03 = 'prov_addr_type=1 or prov_addr_type=3 or prov_addr_type=4' + whr03 = 'prvdr_adr_type_cd=1 or prvdr_adr_type_cd=3 or prvdr_adr_type_cd=4' self.copy_activerows('Prov03_Locations_Latest1', cols03, @@ -101,14 +101,14 @@ def process_03_locations(self, maintbl, outtbl): left join (select - tms_run_id, - submitting_state, - submitting_state_prov_id, - prov_location_id + tmsis_run_id as tms_run_id, + submtg_state_cd as submitting_state, + submtg_state_prvdr_id as submitting_state_prov_id, + prvdr_lctn_id as prov_location_id from - TMSIS.Prov_Licensing_Info + TMSIS.Tmsis_Prvdr_Lcnsg where - prov_location_id='000') L + prvdr_lctn_id='000') L on M.tms_run_id=L.tms_run_id and M.submitting_state=L.submitting_state and @@ -116,14 +116,14 @@ def process_03_locations(self, maintbl, outtbl): left join (select - tms_run_id, - submitting_state, - submitting_state_prov_id, - prov_location_id + tmsis_run_id as tms_run_id, + submtg_state_cd as submitting_state, + submtg_state_prvdr_id as submitting_state_prov_id, + prvdr_lctn_id as prov_location_id from - TMSIS.Prov_Identifiers + TMSIS.Tmsis_Prvdr_Id where - prov_location_id='000') I + prvdr_lctn_id='000') I on M.tms_run_id=I.tms_run_id and M.submitting_state=I.submitting_state and @@ -131,14 +131,14 @@ def process_03_locations(self, maintbl, outtbl): left join (select - tms_run_id, - submitting_state, - submitting_state_prov_id, - prov_location_id + tmsis_run_id as tms_run_id, + submtg_state_cd as submitting_state, + submtg_state_prvdr_id as submitting_state_prov_id, + prvdr_lctn_id as prov_location_id from - TMSIS.Prov_Bed_Type_Info + TMSIS.Tmsis_Prvdr_Bed_Type where - prov_location_id='000') B + prvdr_lctn_id='000') B on M.tms_run_id=B.tms_run_id and M.submitting_state=B.submitting_state and diff --git a/taf/PRV/PRV04.py b/taf/PRV/PRV04.py index 78d87ac6..10b60011 100644 --- a/taf/PRV/PRV04.py +++ b/taf/PRV/PRV04.py @@ -19,7 +19,7 @@ def process_04_licensing(self, loctbl, outtbl): 'submitting_state_prov_id', 'prov_location_id'] - self.screen_runid('tmsis.Prov_Licensing_Info', + self.screen_runid('tmsis.Tmsis_Prvdr_Lcnsg', loctbl, runlist, 'Prov04_Licensing_Latest1', @@ -28,21 +28,21 @@ def process_04_licensing(self, loctbl, outtbl): # row count # self.prv.countrows(Prov04_Licensing_Latest1, cnt_latest, PRV04_Latest) - cols04 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(prov_location_id) as prov_location_id', - '%upper_case(license_or_accreditation_number) as license_or_accreditation_number', - 'license_type', - '%upper_case(license_issuing_entity_id) as license_issuing_entity_id', - 'prov_license_eff_date', - 'prov_license_end_date'] + cols04 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(prvdr_lctn_id) as prov_location_id', + '%upper_case(lcns_or_acrdtn_num) as license_or_accreditation_number', + 'lcns_type_cd as license_type', + '%upper_case(lcns_issg_ent_id_txt) as license_issuing_entity_id', + 'prvdr_lcns_efctv_dt as prov_license_eff_date', + 'prvdr_lcns_end_dt as prov_license_end_date'] # copy 04(Licensing) provider rows - whr04 = 'license_type is not null and upper(license_or_accreditation_number) is not null' + whr04 = 'lcns_type_cd is not null and upper(lcns_or_acrdtn_num) is not null' self.copy_activerows('Prov04_Licensing_Latest1', cols04, diff --git a/taf/PRV/PRV05.py b/taf/PRV/PRV05.py index f83ccc07..3bdd9e84 100644 --- a/taf/PRV/PRV05.py +++ b/taf/PRV/PRV05.py @@ -19,7 +19,7 @@ def process_05_identifiers(self, loctbl, outtbl): 'submitting_state_prov_id', 'prov_location_id'] - self.screen_runid('tmsis.Prov_Identifiers', + self.screen_runid('tmsis.Tmsis_Prvdr_Id', loctbl, runlist, 'Prov05_Identifiers_Latest1', @@ -28,21 +28,21 @@ def process_05_identifiers(self, loctbl, outtbl): # row count # self.prv.countrows(Prov05_Identifiers_Latest1, cnt_latest, PRV05_Latest) - cols05 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(prov_location_id) as prov_location_id', - '%upper_case(prov_identifier) as prov_identifier', - 'prov_identifier_type', - '%upper_case(prov_identifier_issuing_entity_id) as prov_identifier_issuing_entity_id', - 'prov_identifier_eff_date', - 'prov_identifier_end_date'] + cols05 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(prvdr_lctn_id) as prov_location_id', + '%upper_case(prvdr_id) as prov_identifier', + 'prvdr_id_type_cd as prov_identifier_type', + '%upper_case(prvdr_id_issg_ent_id_txt) as prov_identifier_issuing_entity_id', + 'prvdr_id_efctv_dt as prov_identifier_eff_date', + 'prvdr_id_end_dt as prov_identifier_end_date'] # copy 05(identifiers) provider rows - whr05 = 'prov_identifier_type is not null and upper(prov_identifier) is not null' + whr05 = 'prvdr_id_type_cd is not null and upper(prvdr_id) is not null' self.copy_activerows('Prov05_Identifiers_Latest1', cols05, diff --git a/taf/PRV/PRV06.py b/taf/PRV/PRV06.py index 85821bd4..760ba5f7 100644 --- a/taf/PRV/PRV06.py +++ b/taf/PRV/PRV06.py @@ -18,7 +18,7 @@ def process_06_taxonomy(self, maintbl, outtbl): 'submitting_state', 'submitting_state_prov_id'] - self.screen_runid('tmsis.Prov_Taxonomy_Classification', + self.screen_runid('tmsis.Tmsis_Prvdr_Txnmy_Clsfctn', maintbl, runlist, 'Prov06_Taxonomy_Latest1') @@ -26,26 +26,26 @@ def process_06_taxonomy(self, maintbl, outtbl): # row count # self.prv.countrows(Prov06_Taxonomy_Latest1, cnt_latest, PRV06_Latest) - cols06 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - 'upper(submitting_state_prov_id) as submitting_state_prov_id', + cols06 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + 'upper(submtg_state_prvdr_id) as submitting_state_prov_id', """case - when (prov_classification_type='2' or prov_classification_type='3') and - length(trim(prov_classification_code))<2 and length(trim(prov_classification_code))>0 and - nullif(trim(upper(prov_classification_code)),'') is not null then lpad(trim(upper(prov_classification_code)),2,'0') - when prov_classification_type='4' and - length(trim(prov_classification_code))<3 and length(trim(prov_classification_code))>0 and - nullif(trim(upper(prov_classification_code)),'') is not null then lpad(trim(upper(prov_classification_code)),3,'0') - else nullif(trim(upper(prov_classification_code)),'') + when (prvdr_clsfctn_type_cd='2' or prvdr_clsfctn_type_cd='3') and + length(trim(prvdr_clsfctn_cd))<2 and length(trim(prvdr_clsfctn_cd))>0 and + nullif(trim(upper(prvdr_clsfctn_cd)),'') is not null then lpad(trim(upper(prvdr_clsfctn_cd)),2,'0') + when prvdr_clsfctn_type_cd='4' and + length(trim(prvdr_clsfctn_cd))<3 and length(trim(prvdr_clsfctn_cd))>0 and + nullif(trim(upper(prvdr_clsfctn_cd)),'') is not null then lpad(trim(upper(prvdr_clsfctn_cd)),3,'0') + else nullif(trim(upper(prvdr_clsfctn_cd)),'') end as prov_classification_code""", - 'prov_classification_type', - 'prov_taxonomy_classification_eff_date', - 'prov_taxonomy_classification_end_date'] + 'prvdr_clsfctn_type_cd as prov_classification_type', + 'prvdr_txnmy_clsfctn_efctv_dt as prov_taxonomy_classification_eff_date', + 'prvdr_txnmy_clsfctn_end_dt as prov_taxonomy_classification_end_date'] - whr06 = 'prov_classification_type is not null and upper(prov_classification_code) is not null' + whr06 = 'prvdr_clsfctn_type_cd is not null and upper(prvdr_clsfctn_cd) is not null' self.copy_activerows('Prov06_Taxonomy_Latest1', cols06, diff --git a/taf/PRV/PRV07.py b/taf/PRV/PRV07.py index 7441d812..25492b4b 100644 --- a/taf/PRV/PRV07.py +++ b/taf/PRV/PRV07.py @@ -18,7 +18,7 @@ def process_07_medicaid(self, maintbl, outtbl): 'submitting_state', 'submitting_state_prov_id'] - self.screen_runid('tmsis.Prov_Medicaid_Enrollment', + self.screen_runid('tmsis.Tmsis_Prvdr_Mdcd_Enrlmt', maintbl, runlist, 'Prov07_Medicaid_Latest1') @@ -26,20 +26,20 @@ def process_07_medicaid(self, maintbl, outtbl): # row count # self.prv.countrows(Prov07_Medicaid_Latest1, cnt_latest, PRV07_Latest) - cols07 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%zero_pad(prov_medicaid_enrollment_status_code, 2)', - 'state_plan_enrollment', - 'prov_enrollment_method', - '%fix_old_dates(appl_date)', - '%fix_old_dates(prov_medicaid_eff_date)', - '%set_end_dt(prov_medicaid_end_date) as prov_medicaid_end_date'] - - whr07 = 'prov_medicaid_enrollment_status_code is not null' + cols07 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%zero_pad(prvdr_mdcd_enrlmt_stus_cd, 2, prov_medicaid_enrollment_status_code)', + 'state_plan_enrlmt_cd as state_plan_enrollment', + 'prvdr_mdcd_enrlmt_mthd_cd as prov_enrollment_method', + '%fix_old_dates(appl_dt, appl_date)', + '%fix_old_dates(prvdr_mdcd_efctv_dt, prov_medicaid_eff_date)', + '%set_end_dt(prvdr_mdcd_end_dt) as prov_medicaid_end_date'] + + whr07 = 'prvdr_mdcd_enrlmt_stus_cd is not null' self.copy_activerows('Prov07_Medicaid_Latest1', cols07, diff --git a/taf/PRV/PRV08.py b/taf/PRV/PRV08.py index 73aec677..93b3972b 100644 --- a/taf/PRV/PRV08.py +++ b/taf/PRV/PRV08.py @@ -18,7 +18,7 @@ def process_08_groups(self, maintbl, outtbl): 'submitting_state', 'submitting_state_prov_id'] - self.screen_runid('tmsis.Prov_Affiliated_Groups', + self.screen_runid('tmsis.Tmsis_Prvdr_Afltd_Grp', maintbl, runlist, 'Prov08_AffGrps_Latest1') @@ -26,18 +26,18 @@ def process_08_groups(self, maintbl, outtbl): # row count # self.prv.countrows(Prov08_AffGrps_Latest1, cnt_latest, PRV08_Latest) - cols08 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(submitting_state_prov_id_of_affiliated_entity) as submitting_state_prov_id_of_affiliated_entity', - 'prov_affiliated_group_eff_date', - 'prov_affiliated_group_end_date'] + cols08 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(submtg_state_afltd_prvdr_id) as submitting_state_prov_id_of_affiliated_entity', + 'prvdr_afltd_grp_efctv_dt as prov_affiliated_group_eff_date', + 'prvdr_afltd_grp_end_dt as prov_affiliated_group_end_date'] # copy 08(Affiliated Groups) provider rows - whr08 = 'upper(submitting_state_prov_id_of_affiliated_entity) is not null' + whr08 = 'upper(submtg_state_afltd_prvdr_id) is not null' self.copy_activerows('Prov08_AffGrps_Latest1', cols08, diff --git a/taf/PRV/PRV09.py b/taf/PRV/PRV09.py index 03ef2fdd..edd25b1c 100644 --- a/taf/PRV/PRV09.py +++ b/taf/PRV/PRV09.py @@ -18,7 +18,7 @@ def process_09_affpgms(self, maintbl, outtbl): 'submitting_state', 'submitting_state_prov_id'] - self.screen_runid('tmsis.Prov_Affiliated_Programs', + self.screen_runid('tmsis.Tmsis_Prvdr_Afltd_Pgm', maintbl, runlist, 'Prov09_AffPgms_Latest1') @@ -26,19 +26,19 @@ def process_09_affpgms(self, maintbl, outtbl): # row count # self.prv.countrows(Prov09_AffPgms_Latest1, cnt_latest, PRV09_Latest) - cols09 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(affiliated_program_id) as affiliated_program_id', - 'affiliated_program_type', - 'prov_affiliated_program_eff_date', - 'prov_affiliated_program_end_date'] + cols09 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(afltd_pgm_id) as affiliated_program_id', + 'afltd_pgm_type_cd as affiliated_program_type', + 'prvdr_afltd_pgm_efctv_dt as prov_affiliated_program_eff_date', + 'prvdr_afltd_pgm_end_dt as prov_affiliated_program_end_date'] # copy 09(Affiliated Programs) provider rows - whr09 = 'upper(affiliated_program_id) is not null' + whr09 = 'upper(afltd_pgm_id) is not null' self.copy_activerows('Prov09_AffPgms_Latest1', cols09, diff --git a/taf/PRV/PRV10.py b/taf/PRV/PRV10.py index e75d9b6f..ab7ea726 100644 --- a/taf/PRV/PRV10.py +++ b/taf/PRV/PRV10.py @@ -18,7 +18,7 @@ def process_10_beds(self, loctbl, outtbl): 'submitting_state_prov_id', 'prov_location_id'] - self.screen_runid('tmsis.Prov_Bed_Type_Info', + self.screen_runid('tmsis.Tmsis_Prvdr_Bed_Type', loctbl, runlist, 'Prov10_BedType_Latest1', @@ -27,23 +27,23 @@ def process_10_beds(self, loctbl, outtbl): # row count # self.prv.countrows(Prov10_BedType_Latest1, cnt_latest, PRV10_Latest) - cols10 = ['tms_run_id', - 'tms_reporting_period', - 'record_number', - 'submitting_state', - 'submitting_state as submtg_state_cd', - '%upper_case(submitting_state_prov_id) as submitting_state_prov_id', - '%upper_case(prov_location_id) as prov_location_id', - 'bed_count', + cols10 = ['tmsis_run_id as tms_run_id', + 'tmsis_rptg_prd as tms_reporting_period', + 'rec_num as record_number', + 'submtg_state_cd as submitting_state', + 'submtg_state_cd', + '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', + '%upper_case(prvdr_lctn_id) as prov_location_id', + 'bed_cnt as bed_count', """case - when trim(TRAILING FROM bed_type_code) in ('1','2','3','4','5','6','7') then trim(TRAILING FROM bed_type_code) + when trim(TRAILING FROM bed_type_cd) in ('1','2','3','4','5','6','7') then trim(TRAILING FROM bed_type_cd) else null end as bed_type_code""", - 'bed_type_eff_date', - 'bed_type_end_date'] + 'bed_type_efctv_dt as bed_type_eff_date', + 'bed_type_end_dt as bed_type_end_date'] # copy 10(bed type) provider rows - whr10 = "(trim(TRAILING FROM bed_type_code) in ('1','2','3','4','5','6','7')) or (bed_count is not null and bed_count<>0)" + whr10 = "(trim(TRAILING FROM bed_type_cd) in ('1','2','3','4','5','6','7')) or (bed_cnt is not null and bed_cnt<>0)" self.copy_activerows('Prov10_BedType_Latest1', cols10, diff --git a/taf/TAF_Closure.py b/taf/TAF_Closure.py index 1a009b99..33e5e9bc 100644 --- a/taf/TAF_Closure.py +++ b/taf/TAF_Closure.py @@ -626,12 +626,16 @@ def var_set_taxo( return "\n ".join(result) - def fix_old_dates(date_var): + def fix_old_dates(date_var, out_as=None): """ For dates older than 1600-01-01, default the dates to 1599-12-31. + out_as: specify output column name - default = input column name """ - return f"case when ({date_var} < to_date('1600-01-01')) then to_date('1599-12-31') else {date_var} end as {date_var}" + if out_as is None: + out_as = date_var + + return f"case when ({date_var} < to_date('1600-01-01')) then to_date('1599-12-31') else {date_var} end as {out_as}" def set_end_dt(enddt): """ @@ -663,16 +667,20 @@ def zpad(col): return "lpad(trim(col), 4, '0')" - def zero_pad(var_cd, var_len): + def zero_pad(var_cd, var_len, out_as=None): """ Another zero pad function. + out_as: specify output column name - default = input column name """ + if out_as is None: + out_as = var_cd + return f"""case when length(trim({var_cd}))<{var_len} and length(trim({var_cd}))>0 and {var_cd} is not null then lpad(trim(upper({var_cd})),{var_len},'0') else nullif(trim(upper({var_cd})),'') - end as {var_cd} + end as {out_as} """ typecast = { From 8f301c8632e52fae534ca786690c64db6a578827 Mon Sep 17 00:00:00 2001 From: fumanjie Date: Mon, 15 Sep 2025 14:28:38 -0400 Subject: [PATCH 2/3] DBDAART-15629 Convert MCP build process from using TMSIS tables to using TMSIS views. --- taf/MCP/MCP.py | 26 +++++++++++++++++++------- taf/MCP/MCP01.py | 2 +- taf/MCP/MCP02.py | 36 ++++++++++++++++++------------------ taf/MCP/MCP03.py | 36 ++++++++++++++++++------------------ taf/MCP/MCP04.py | 20 ++++++++++---------- taf/MCP/MCP05.py | 30 +++++++++++++++--------------- taf/MCP/MCP06.py | 20 ++++++++++---------- taf/MCP/MCP07.py | 20 ++++++++++---------- 8 files changed, 101 insertions(+), 89 deletions(-) diff --git a/taf/MCP/MCP.py b/taf/MCP/MCP.py index 41989e9b..99e8ae30 100644 --- a/taf/MCP/MCP.py +++ b/taf/MCP/MCP.py @@ -40,14 +40,25 @@ def screen_runid(self, intbl, runtbl, runvars, outtbl, runtyp="C"): T.submitting_state = R.submitting_state and upper(T.state_plan_id_num) = R.state_plan_id_num""" + # retain tmsis table column names used in subsequent merge and in TAF_Metadata.py + renames = [ + 'tmsis_run_id as tms_run_id', + 'submtg_state_cd as submitting_state', + ] + # diststyle key distkey(state_plan_id_num) # compound sortkey (&&&runvars) as z = f""" create or replace temporary view {outtbl} as select T.* - from - {intbl} T + from ( + select + *, + { ','.join(renames) } + from + {intbl} + ) T inner join {runtbl} R { on.format(runvars) } order by @@ -78,7 +89,7 @@ def copy_activerows(self, intbl, collist, whr, outtbl): from { intbl } where - tms_is_active=1 + tmsis_actv_ind=1 { whr } order by tms_run_id, @@ -97,14 +108,15 @@ def copy_activerows_nts(self, intbl, collist, outtbl): from ( select *, - submitting_state as submtg_state_cd + tmsis_run_id as tms_run_id, + submtg_state_cd as submitting_state from {intbl} where - tms_is_active = 1 and - tms_reporting_period is not null and + tmsis_actv_ind = 1 and + tmsis_rptg_prd is not null and tot_rec_cnt > 0 and - trim(submitting_state) not in ('94','96')) + trim(submtg_state_cd) not in ('94','96')) where 1=1 { self.mcp.ST_FILTER() } order by diff --git a/taf/MCP/MCP01.py b/taf/MCP/MCP01.py index 36b451b8..81270e43 100644 --- a/taf/MCP/MCP01.py +++ b/taf/MCP/MCP01.py @@ -21,7 +21,7 @@ def process_01_header(self, outtbl): # copy managed care header table self.copy_activerows_nts( - "tmsis.file_header_record_managed_care", cols01, "MC01_Header_Copy" + "tmsis.tmsis_fhdr_rec_mc", cols01, "MC01_Header_Copy" ) # self.mcp.countrows('MC01_Header_Copy', 'cnt_active', 'MC01_Active') diff --git a/taf/MCP/MCP02.py b/taf/MCP/MCP02.py index 83a84450..8649abe4 100644 --- a/taf/MCP/MCP02.py +++ b/taf/MCP/MCP02.py @@ -20,31 +20,31 @@ def process_02_mcmain(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state"] self.screen_runid( - "tmsis.Managed_Care_Main", runtbl, runlist, "MC02_Main_Latest1", "M" + "tmsis.Tmsis_Mc_Mn_Data", runtbl, runlist, "MC02_Main_Latest1", "M" ) # row count self.count_rows("MC02_Main_Latest1", "cnt_latest", "MC02_Latest") cols02 = [ - "tms_run_id", - "tms_reporting_period", - "submitting_state", - "submitting_state as submtg_state_cd", - "record_number", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "submtg_state_cd as submitting_state", + "submtg_state_cd", + "rec_num as record_number", "%upper_case(state_plan_id_num) as state_plan_id_num", - "managed_care_main_rec_eff_date", - "managed_care_main_rec_end_date", - "%fix_old_dates(managed_care_contract_eff_date)", - "case when managed_care_contract_eff_date is not null and managed_care_contract_end_date is null then to_date('9999-12-31') when to_date('1600-01-01') > managed_care_contract_end_date then to_date('1599-12-31') else managed_care_contract_end_date end as MC_CNTRCT_END_DT", - "managed_care_name", - "managed_care_program", - "managed_care_plan_type", - "reimbursement_arrangement", - "managed_care_profit_status", - "core_based_statistical_area_code", - "percent_business", - "managed_care_service_area", + "mc_mn_rec_efctv_dt as managed_care_main_rec_eff_date", + "mc_mn_rec_end_dt as managed_care_main_rec_end_date", + "%fix_old_dates(mc_cntrct_efctv_dt, managed_care_contract_eff_date)", + "case when mc_cntrct_efctv_dt is not null and mc_cntrct_end_dt is null then to_date('9999-12-31') when to_date('1600-01-01') > mc_cntrct_end_dt then to_date('1599-12-31') else mc_cntrct_end_dt end as MC_CNTRCT_END_DT", + "mc_name as managed_care_name", + "mc_pgm_cd as managed_care_program", + "mc_plan_type_cd as managed_care_plan_type", + "reimbrsmt_arngmt_cd as reimbursement_arrangement", + "mc_prft_stus_cd as managed_care_profit_status", + "cbsa_cd as core_based_statistical_area_code", + "busns_pct as percent_business", + "mc_sarea_cd as managed_care_service_area", ] whr02 = "upper(state_plan_id_num) is not null" diff --git a/taf/MCP/MCP03.py b/taf/MCP/MCP03.py index eb11162d..b4f9268f 100644 --- a/taf/MCP/MCP03.py +++ b/taf/MCP/MCP03.py @@ -19,7 +19,7 @@ def process_03_location(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state", "state_plan_id_num"] self.screen_runid( - "tmsis.Managed_care_location_and_contact_info", + "tmsis.Tmsis_Mc_Lctn_Cntct", runtbl, runlist, "MC03_Location_Latest1", @@ -30,26 +30,26 @@ def process_03_location(self, runtbl, outtbl): self.count_rows("MC03_Location_Latest1", "cnt_latest", "MC03_Latest") cols03 = [ - "tms_run_id", - "tms_reporting_period", - "record_number", - "submitting_state", - "submitting_state as submtg_state_cd", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "rec_num as record_number", + "submtg_state_cd as submitting_state", + "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", - "%upper_case(managed_care_location_id) as managed_care_location_id", - "%fix_old_dates(managed_care_location_and_contact_info_eff_date)", - "%set_end_dt(managed_care_location_and_contact_info_end_date) as managed_care_location_and_contact_info_end_date", - "%upper_case(managed_care_addr_ln1) as managed_care_addr_ln1", - "managed_care_addr_ln2", - "managed_care_addr_ln3", - "managed_care_addr_type", - "managed_care_city", - "managed_care_county", - "%upper_case(managed_care_state) as managed_care_state", - "managed_care_zip_code", + "%upper_case(mc_lctn_id) as managed_care_location_id", + "%fix_old_dates(mc_lctn_cntct_efctv_dt, managed_care_location_and_contact_info_eff_date)", + "%set_end_dt(mc_lctn_cntct_end_dt) as managed_care_location_and_contact_info_end_date", + "%upper_case(mc_line_1_adr) as managed_care_addr_ln1", + "mc_line_2_adr as managed_care_addr_ln2", + "mc_line_3_adr as managed_care_addr_ln3", + "mc_adr_type_cd as managed_care_addr_type", + "mc_city_name as managed_care_city", + "mc_cnty_cd as managed_care_county", + "%upper_case(mc_state_cd) as managed_care_state", + "mc_zip_cd as managed_care_zip_code", ] - whr03 = "managed_care_addr_type=3" + whr03 = "mc_adr_type_cd=3" self.copy_activerows( "MC03_Location_Latest1", cols03, whr03, "MC03_Location_Copy" diff --git a/taf/MCP/MCP04.py b/taf/MCP/MCP04.py index 6b92395d..2b772dcf 100644 --- a/taf/MCP/MCP04.py +++ b/taf/MCP/MCP04.py @@ -19,7 +19,7 @@ def process_04_service_area(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state", "state_plan_id_num"] self.screen_runid( - "tmsis.Managed_care_service_area", + "tmsis.Tmsis_Mc_Sarea", runtbl, runlist, "MC04_Service_Area_Latest1", @@ -29,18 +29,18 @@ def process_04_service_area(self, runtbl, outtbl): self.count_rows("MC04_Service_Area_Latest1", "cnt_latest", "MC04_Latest") cols04 = [ - "tms_run_id", - "tms_reporting_period", - "record_number", - "submitting_state", - "submitting_state as submtg_state_cd", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "rec_num as record_number", + "submtg_state_cd as submitting_state", + "submtg_state_cd", f"""{ TAF_Closure.upper_case('state_plan_id_num')} as state_plan_id_num""", - f"""{ TAF_Closure.upper_case('managed_care_service_area_name')} as managed_care_service_area_name""", - f"""{ TAF_Closure.fix_old_dates('managed_care_service_area_eff_date')}""", - f"""{ TAF_Closure.set_end_dt('managed_care_service_area_end_date')} as managed_care_service_area_end_date""" + f"""{ TAF_Closure.upper_case('mc_sarea_name')} as managed_care_service_area_name""", + f"""{ TAF_Closure.fix_old_dates('mc_sarea_efctv_dt','managed_care_service_area_eff_date')}""", + f"""{ TAF_Closure.set_end_dt('mc_sarea_end_dt')} as managed_care_service_area_end_date""" ] - whr04 = "upper(managed_care_service_area_name) is not null" + whr04 = "upper(mc_sarea_name) is not null" self.copy_activerows( "MC04_Service_Area_Latest1", cols04, whr04, "MC04_Service_Area_Copy" diff --git a/taf/MCP/MCP05.py b/taf/MCP/MCP05.py index 504ae847..0b388bb8 100644 --- a/taf/MCP/MCP05.py +++ b/taf/MCP/MCP05.py @@ -19,7 +19,7 @@ def process_05_operating_authority(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state", "state_plan_id_num"] self.screen_runid( - "tmsis.Managed_care_operating_authority", + "tmsis.Tmsis_Mc_Oprtg_Authrty", runtbl, runlist, "MC05_Operating_Authority_Latest1", @@ -29,26 +29,26 @@ def process_05_operating_authority(self, runtbl, outtbl): self.count_rows("MC05_Operating_Authority_Latest1", "cnt_latest", "MC05_Latest") cols05 = [ - "tms_run_id", - "tms_reporting_period", - "record_number", - "submitting_state", - "submitting_state as submtg_state_cd", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "rec_num as record_number", + "submtg_state_cd as submitting_state", + "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", """case - when length(trim(TRAILING FROM operating_authority))<2 and length(trim(TRAILING FROM operating_authority))>0 and operating_authority in ('1','2','3','4','5','6','7','8','9') - then lpad(trim(TRAILING FROM operating_authority),2,'0') - when trim(TRAILING FROM operating_authority) in ('01','02','03','04','05','06','07','08','09') or - trim(TRAILING FROM operating_authority) in ('10','11','12','13','14','15','16','17','18','19','20','21','22','23') - then trim(TRAILING FROM operating_authority) + when length(trim(TRAILING FROM oprtg_authrty_cd))<2 and length(trim(TRAILING FROM oprtg_authrty_cd))>0 and oprtg_authrty_cd in ('1','2','3','4','5','6','7','8','9') + then lpad(trim(TRAILING FROM oprtg_authrty_cd),2,'0') + when trim(TRAILING FROM oprtg_authrty_cd) in ('01','02','03','04','05','06','07','08','09') or + trim(TRAILING FROM oprtg_authrty_cd) in ('10','11','12','13','14','15','16','17','18','19','20','21','22','23') + then trim(TRAILING FROM oprtg_authrty_cd) else null end as operating_authority""", - "%upper_case(waiver_id) as waiver_id", - "managed_care_op_authority_eff_date", - "managed_care_op_authority_end_date", + "%upper_case(wvr_id) as waiver_id", + "mc_op_authrty_efctv_dt as managed_care_op_authority_eff_date", + "mc_op_authrty_end_dt as managed_care_op_authority_end_date", ] - whr05 = "(trim(TRAILING FROM operating_authority) in ('1','2','3','4','5','6','7','8','9') or trim(TRAILING FROM operating_authority) in ('01','02','03','04','05','06','07','08','09') or trim(TRAILING FROM operating_authority) in ('10','11','12','13','14','15','16','17','18','19','20','21','22','23')) or (upper(waiver_id) is not null)" + whr05 = "(trim(TRAILING FROM oprtg_authrty_cd) in ('1','2','3','4','5','6','7','8','9') or trim(TRAILING FROM oprtg_authrty_cd) in ('01','02','03','04','05','06','07','08','09') or trim(TRAILING FROM oprtg_authrty_cd) in ('10','11','12','13','14','15','16','17','18','19','20','21','22','23')) or (upper(wvr_id) is not null)" self.copy_activerows( "MC05_Operating_Authority_Latest1", diff --git a/taf/MCP/MCP06.py b/taf/MCP/MCP06.py index 094f8c86..f842a00c 100644 --- a/taf/MCP/MCP06.py +++ b/taf/MCP/MCP06.py @@ -20,7 +20,7 @@ def process_06_population(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state", "state_plan_id_num"] self.screen_runid( - "tmsis.Managed_care_plan_population_enrolled", + "tmsis.Tmsis_Mc_Plan_Pop_Enrld", runtbl, runlist, "MC06_Population_Latest1", @@ -30,18 +30,18 @@ def process_06_population(self, runtbl, outtbl): self.count_rows("MC06_Population_Latest1", "cnt_latest", "MC06_Latest") cols06 = [ - "tms_run_id", - "tms_reporting_period", - "record_number", - "submitting_state", - "submitting_state as submtg_state_cd", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "rec_num as record_number", + "submtg_state_cd as submitting_state", + "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", - "%zero_pad(managed_care_plan_pop, 2)", - "%fix_old_dates(managed_care_plan_pop_eff_date)", - "%set_end_dt(managed_care_plan_pop_end_date) as managed_care_plan_pop_end_date", + "%zero_pad(mc_plan_pop_cnt, 2, managed_care_plan_pop)", + "%fix_old_dates(mc_plan_pop_efctv_dt, managed_care_plan_pop_eff_date)", + "%set_end_dt(mc_plan_pop_end_dt) as managed_care_plan_pop_end_date", ] - whr06 = "managed_care_plan_pop is not null" + whr06 = "mc_plan_pop_cnt is not null" self.copy_activerows( "MC06_Population_Latest1", cols06, whr06, "MC06_Population_Copy" diff --git a/taf/MCP/MCP07.py b/taf/MCP/MCP07.py index 16cee173..5a332166 100644 --- a/taf/MCP/MCP07.py +++ b/taf/MCP/MCP07.py @@ -20,7 +20,7 @@ def process_07_accreditation(self, runtbl, outtbl): runlist = ["tms_run_id", "submitting_state", "state_plan_id_num"] self.screen_runid( - "tmsis.Managed_care_accreditation_organization", + "tmsis.Tmsis_Mc_Acrdtn_Org", runtbl, runlist, "MC07_Accreditation_Latest1", @@ -31,18 +31,18 @@ def process_07_accreditation(self, runtbl, outtbl): self.count_rows("MC07_Accreditation_Latest1", "cnt_latest", "MC07_Latest") cols07 = [ - "tms_run_id", - "tms_reporting_period", - "record_number", - "submitting_state", - "submitting_state as submtg_state_cd", + "tmsis_run_id as tms_run_id", + "tmsis_rptg_prd as tms_reporting_period", + "rec_num as record_number", + "submtg_state_cd as submitting_state", + "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", - "%zero_pad(accreditation_organization, 2)", - "%fix_old_dates(date_accreditation_achieved)", - "%set_end_dt(date_accreditation_end) as date_accreditation_end", + "%zero_pad(acrdtn_org_cd, 2, accreditation_organization)", + "%fix_old_dates(acrdtn_achvmt_dt, date_accreditation_achieved)", + "%set_end_dt(acrdtn_end_dt) as date_accreditation_end", ] - whr07 = "accreditation_organization is not null" + whr07 = "acrdtn_org_cd is not null" self.copy_activerows( "MC07_Accreditation_Latest1", cols07, whr07, "MC07_Accreditation_Copy" From cbb0ebc6e2c2e448e7a2a2c3eb176566a643bf6e Mon Sep 17 00:00:00 2001 From: fumanjie Date: Fri, 19 Sep 2025 14:06:53 -0400 Subject: [PATCH 3/3] DBDAART-15627 + DBDAART-15629 Create separate fix_old_dates and zero_pad functions in TAF_Closure to permit column renames for MCP/PRV, rather than adding a conditional to functions shared by claims, which would unnecessarily increase run time. --- taf/MCP/MCP.py | 2 ++ taf/MCP/MCP02.py | 4 +++- taf/MCP/MCP03.py | 4 +++- taf/MCP/MCP04.py | 4 +++- taf/MCP/MCP05.py | 2 ++ taf/MCP/MCP06.py | 6 ++++-- taf/MCP/MCP07.py | 6 ++++-- taf/PRV/PRV.py | 2 ++ taf/PRV/PRV02.py | 6 ++++-- taf/PRV/PRV03.py | 2 ++ taf/PRV/PRV04.py | 2 ++ taf/PRV/PRV05.py | 2 ++ taf/PRV/PRV06.py | 2 ++ taf/PRV/PRV07.py | 8 +++++--- taf/PRV/PRV08.py | 2 ++ taf/PRV/PRV09.py | 2 ++ taf/PRV/PRV10.py | 2 ++ taf/TAF_Closure.py | 29 +++++++++++++++++++++++++++-- 18 files changed, 73 insertions(+), 14 deletions(-) diff --git a/taf/MCP/MCP.py b/taf/MCP/MCP.py index 99e8ae30..3c4dd224 100644 --- a/taf/MCP/MCP.py +++ b/taf/MCP/MCP.py @@ -101,6 +101,8 @@ def copy_activerows(self, intbl, collist, whr, outtbl): def copy_activerows_nts(self, intbl, collist, outtbl): # diststyle even compound sortkey(tms_run_id, submitting_state) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing z = f""" create or replace temporary view {outtbl} as select diff --git a/taf/MCP/MCP02.py b/taf/MCP/MCP02.py index 8649abe4..ccfd542c 100644 --- a/taf/MCP/MCP02.py +++ b/taf/MCP/MCP02.py @@ -26,6 +26,8 @@ def process_02_mcmain(self, runtbl, outtbl): # row count self.count_rows("MC02_Main_Latest1", "cnt_latest", "MC02_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols02 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", @@ -35,7 +37,7 @@ def process_02_mcmain(self, runtbl, outtbl): "%upper_case(state_plan_id_num) as state_plan_id_num", "mc_mn_rec_efctv_dt as managed_care_main_rec_eff_date", "mc_mn_rec_end_dt as managed_care_main_rec_end_date", - "%fix_old_dates(mc_cntrct_efctv_dt, managed_care_contract_eff_date)", + "%fix_old_dates_rename(mc_cntrct_efctv_dt, managed_care_contract_eff_date)", "case when mc_cntrct_efctv_dt is not null and mc_cntrct_end_dt is null then to_date('9999-12-31') when to_date('1600-01-01') > mc_cntrct_end_dt then to_date('1599-12-31') else mc_cntrct_end_dt end as MC_CNTRCT_END_DT", "mc_name as managed_care_name", "mc_pgm_cd as managed_care_program", diff --git a/taf/MCP/MCP03.py b/taf/MCP/MCP03.py index b4f9268f..19ae87c8 100644 --- a/taf/MCP/MCP03.py +++ b/taf/MCP/MCP03.py @@ -29,6 +29,8 @@ def process_03_location(self, runtbl, outtbl): # row count self.count_rows("MC03_Location_Latest1", "cnt_latest", "MC03_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols03 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", @@ -37,7 +39,7 @@ def process_03_location(self, runtbl, outtbl): "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", "%upper_case(mc_lctn_id) as managed_care_location_id", - "%fix_old_dates(mc_lctn_cntct_efctv_dt, managed_care_location_and_contact_info_eff_date)", + "%fix_old_dates_rename(mc_lctn_cntct_efctv_dt, managed_care_location_and_contact_info_eff_date)", "%set_end_dt(mc_lctn_cntct_end_dt) as managed_care_location_and_contact_info_end_date", "%upper_case(mc_line_1_adr) as managed_care_addr_ln1", "mc_line_2_adr as managed_care_addr_ln2", diff --git a/taf/MCP/MCP04.py b/taf/MCP/MCP04.py index 2b772dcf..1f536a5f 100644 --- a/taf/MCP/MCP04.py +++ b/taf/MCP/MCP04.py @@ -28,6 +28,8 @@ def process_04_service_area(self, runtbl, outtbl): # row count self.count_rows("MC04_Service_Area_Latest1", "cnt_latest", "MC04_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols04 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", @@ -36,7 +38,7 @@ def process_04_service_area(self, runtbl, outtbl): "submtg_state_cd", f"""{ TAF_Closure.upper_case('state_plan_id_num')} as state_plan_id_num""", f"""{ TAF_Closure.upper_case('mc_sarea_name')} as managed_care_service_area_name""", - f"""{ TAF_Closure.fix_old_dates('mc_sarea_efctv_dt','managed_care_service_area_eff_date')}""", + f"""{ TAF_Closure.fix_old_dates_rename('mc_sarea_efctv_dt','managed_care_service_area_eff_date')}""", f"""{ TAF_Closure.set_end_dt('mc_sarea_end_dt')} as managed_care_service_area_end_date""" ] diff --git a/taf/MCP/MCP05.py b/taf/MCP/MCP05.py index 0b388bb8..9ead7be4 100644 --- a/taf/MCP/MCP05.py +++ b/taf/MCP/MCP05.py @@ -28,6 +28,8 @@ def process_05_operating_authority(self, runtbl, outtbl): # row count self.count_rows("MC05_Operating_Authority_Latest1", "cnt_latest", "MC05_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols05 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", diff --git a/taf/MCP/MCP06.py b/taf/MCP/MCP06.py index f842a00c..21940b44 100644 --- a/taf/MCP/MCP06.py +++ b/taf/MCP/MCP06.py @@ -29,6 +29,8 @@ def process_06_population(self, runtbl, outtbl): # row count self.count_rows("MC06_Population_Latest1", "cnt_latest", "MC06_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols06 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", @@ -36,8 +38,8 @@ def process_06_population(self, runtbl, outtbl): "submtg_state_cd as submitting_state", "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", - "%zero_pad(mc_plan_pop_cnt, 2, managed_care_plan_pop)", - "%fix_old_dates(mc_plan_pop_efctv_dt, managed_care_plan_pop_eff_date)", + "%zero_pad_rename(mc_plan_pop_cnt, 2, managed_care_plan_pop)", + "%fix_old_dates_rename(mc_plan_pop_efctv_dt, managed_care_plan_pop_eff_date)", "%set_end_dt(mc_plan_pop_end_dt) as managed_care_plan_pop_end_date", ] diff --git a/taf/MCP/MCP07.py b/taf/MCP/MCP07.py index 5a332166..4fd6986b 100644 --- a/taf/MCP/MCP07.py +++ b/taf/MCP/MCP07.py @@ -30,6 +30,8 @@ def process_07_accreditation(self, runtbl, outtbl): # row count self.count_rows("MC07_Accreditation_Latest1", "cnt_latest", "MC07_Latest") + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols07 = [ "tmsis_run_id as tms_run_id", "tmsis_rptg_prd as tms_reporting_period", @@ -37,8 +39,8 @@ def process_07_accreditation(self, runtbl, outtbl): "submtg_state_cd as submitting_state", "submtg_state_cd", "%upper_case(state_plan_id_num) as state_plan_id_num", - "%zero_pad(acrdtn_org_cd, 2, accreditation_organization)", - "%fix_old_dates(acrdtn_achvmt_dt, date_accreditation_achieved)", + "%zero_pad_rename(acrdtn_org_cd, 2, accreditation_organization)", + "%fix_old_dates_rename(acrdtn_achvmt_dt, date_accreditation_achieved)", "%set_end_dt(acrdtn_end_dt) as date_accreditation_end", ] diff --git a/taf/PRV/PRV.py b/taf/PRV/PRV.py index 68ae32e4..26072a89 100644 --- a/taf/PRV/PRV.py +++ b/taf/PRV/PRV.py @@ -133,6 +133,8 @@ def copy_activerows(self, intbl, collist, whr, outtbl): def copy_activerows_nts(self, intbl, collist, outtbl): # diststyle even compound sortkey(tms_run_id, submitting_state) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing z = f""" create or replace temporary view {outtbl} as select diff --git a/taf/PRV/PRV02.py b/taf/PRV/PRV02.py index 1be957e8..a0bd7a49 100644 --- a/taf/PRV/PRV02.py +++ b/taf/PRV/PRV02.py @@ -31,6 +31,8 @@ def process_02_main(self, runtbl, outtbl): 'cnt_latest', 'PRV02_Latest') + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols02 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'submtg_state_cd as submitting_state', @@ -51,8 +53,8 @@ def process_02_main(self, runtbl, outtbl): 'sex_cd as sex', 'ownrshp_cd as ownership_code', 'prvdr_prft_stus_cd as prov_profit_status', - '%fix_old_dates(birth_dt, date_of_birth)', - '%fix_old_dates(death_dt, date_of_death)', + '%fix_old_dates_rename(birth_dt, date_of_birth)', + '%fix_old_dates_rename(death_dt, date_of_death)', 'acpt_new_ptnts_ind as accepting_new_patients_ind', 'atyp_prov_ind as atypical_prov_ind', ] diff --git a/taf/PRV/PRV03.py b/taf/PRV/PRV03.py index 1398d656..f914c8bc 100644 --- a/taf/PRV/PRV03.py +++ b/taf/PRV/PRV03.py @@ -26,6 +26,8 @@ def process_03_locations(self, maintbl, outtbl): # row count # self.prv.countrows(Prov03_Locations_Latest1, cnt_latest, PRV03_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols03 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV04.py b/taf/PRV/PRV04.py index 10b60011..ce2af364 100644 --- a/taf/PRV/PRV04.py +++ b/taf/PRV/PRV04.py @@ -28,6 +28,8 @@ def process_04_licensing(self, loctbl, outtbl): # row count # self.prv.countrows(Prov04_Licensing_Latest1, cnt_latest, PRV04_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols04 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV05.py b/taf/PRV/PRV05.py index 3bdd9e84..0ef25340 100644 --- a/taf/PRV/PRV05.py +++ b/taf/PRV/PRV05.py @@ -28,6 +28,8 @@ def process_05_identifiers(self, loctbl, outtbl): # row count # self.prv.countrows(Prov05_Identifiers_Latest1, cnt_latest, PRV05_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols05 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV06.py b/taf/PRV/PRV06.py index 760ba5f7..e154724c 100644 --- a/taf/PRV/PRV06.py +++ b/taf/PRV/PRV06.py @@ -26,6 +26,8 @@ def process_06_taxonomy(self, maintbl, outtbl): # row count # self.prv.countrows(Prov06_Taxonomy_Latest1, cnt_latest, PRV06_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols06 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV07.py b/taf/PRV/PRV07.py index 25492b4b..2dd6d658 100644 --- a/taf/PRV/PRV07.py +++ b/taf/PRV/PRV07.py @@ -26,17 +26,19 @@ def process_07_medicaid(self, maintbl, outtbl): # row count # self.prv.countrows(Prov07_Medicaid_Latest1, cnt_latest, PRV07_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols07 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', 'submtg_state_cd as submitting_state', 'submtg_state_cd', '%upper_case(submtg_state_prvdr_id) as submitting_state_prov_id', - '%zero_pad(prvdr_mdcd_enrlmt_stus_cd, 2, prov_medicaid_enrollment_status_code)', + '%zero_pad_rename(prvdr_mdcd_enrlmt_stus_cd, 2, prov_medicaid_enrollment_status_code)', 'state_plan_enrlmt_cd as state_plan_enrollment', 'prvdr_mdcd_enrlmt_mthd_cd as prov_enrollment_method', - '%fix_old_dates(appl_dt, appl_date)', - '%fix_old_dates(prvdr_mdcd_efctv_dt, prov_medicaid_eff_date)', + '%fix_old_dates_rename(appl_dt, appl_date)', + '%fix_old_dates_rename(prvdr_mdcd_efctv_dt, prov_medicaid_eff_date)', '%set_end_dt(prvdr_mdcd_end_dt) as prov_medicaid_end_date'] whr07 = 'prvdr_mdcd_enrlmt_stus_cd is not null' diff --git a/taf/PRV/PRV08.py b/taf/PRV/PRV08.py index 93b3972b..844d62e1 100644 --- a/taf/PRV/PRV08.py +++ b/taf/PRV/PRV08.py @@ -26,6 +26,8 @@ def process_08_groups(self, maintbl, outtbl): # row count # self.prv.countrows(Prov08_AffGrps_Latest1, cnt_latest, PRV08_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols08 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV09.py b/taf/PRV/PRV09.py index edd25b1c..5d123b23 100644 --- a/taf/PRV/PRV09.py +++ b/taf/PRV/PRV09.py @@ -26,6 +26,8 @@ def process_09_affpgms(self, maintbl, outtbl): # row count # self.prv.countrows(Prov09_AffPgms_Latest1, cnt_latest, PRV09_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols09 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/PRV/PRV10.py b/taf/PRV/PRV10.py index ab7ea726..cec96305 100644 --- a/taf/PRV/PRV10.py +++ b/taf/PRV/PRV10.py @@ -27,6 +27,8 @@ def process_10_beds(self, loctbl, outtbl): # row count # self.prv.countrows(Prov10_BedType_Latest1, cnt_latest, PRV10_Latest) + # upon conversion from using TMSIS tables to using TMSIS views + # retain TMSIS table column names to preserve downstream processing cols10 = ['tmsis_run_id as tms_run_id', 'tmsis_rptg_prd as tms_reporting_period', 'rec_num as record_number', diff --git a/taf/TAF_Closure.py b/taf/TAF_Closure.py index 33e5e9bc..7dcbef2e 100644 --- a/taf/TAF_Closure.py +++ b/taf/TAF_Closure.py @@ -626,9 +626,18 @@ def var_set_taxo( return "\n ".join(result) - def fix_old_dates(date_var, out_as=None): + def fix_old_dates(date_var): """ For dates older than 1600-01-01, default the dates to 1599-12-31. + """ + + return f"case when ({date_var} < to_date('1600-01-01')) then to_date('1599-12-31') else {date_var} end as {date_var}" + + def fix_old_dates_rename(date_var, out_as=None): + """ + For dates older than 1600-01-01, default the dates to 1599-12-31. + Add ability to rename output column - for use with MCP/PRV after + converting from using TMSIS tables to using TMSIS views. out_as: specify output column name - default = input column name """ @@ -667,9 +676,23 @@ def zpad(col): return "lpad(trim(col), 4, '0')" - def zero_pad(var_cd, var_len, out_as=None): + def zero_pad(var_cd, var_len): + """ + Another zero pad function. + """ + + return f"""case + when length(trim({var_cd}))<{var_len} and length(trim({var_cd}))>0 and {var_cd} is not null + then lpad(trim(upper({var_cd})),{var_len},'0') + else nullif(trim(upper({var_cd})),'') + end as {var_cd} + """ + + def zero_pad_rename(var_cd, var_len, out_as=None): """ Another zero pad function. + Add ability to rename output column - for use with MCP/PRV after + converting from using TMSIS tables to using TMSIS views. out_as: specify output column name - default = input column name """ @@ -1048,6 +1071,7 @@ def misslogic(var, length): "%count_rec": count_rec, "%ever_year": ever_year, "%fix_old_dates": fix_old_dates, + "%fix_old_dates_rename": fix_old_dates_rename, "%getmax": getmax, "%last_best": last_best, "%monthly_array": monthly_array, @@ -1056,6 +1080,7 @@ def misslogic(var, length): "%sumrecs": sumrecs, "%upper_case": upper_case, "%zero_pad": zero_pad, + "%zero_pad_rename": zero_pad_rename, } @staticmethod