diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2cf650f8..8d558d15 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -24,7 +24,7 @@ vars: ## The vars directly below enable all models related to the type of healthcare data being used clinical_enabled: true - claims_enabled: true + # claims_enabled: true ## The vars directly below enable a single data mart. See the Quickstart diff --git a/integration_tests/docs_generate/dbt_project.yml b/integration_tests/docs_generate/dbt_project.yml index b64b8d20..18f0a5df 100644 --- a/integration_tests/docs_generate/dbt_project.yml +++ b/integration_tests/docs_generate/dbt_project.yml @@ -19,9 +19,9 @@ vars: # Use the vars below to enabled or disable sections of The Tuva Project. ## The vars directly below enable all models related to the type of healthcare data being used - #clinical_enabled: true - #claims_enabled: true - tuva_marts_enabled: true + clinical_enabled: true + claims_enabled: true + #tuva_marts_enabled: true dispatch: diff --git a/models/core/core_models.yml b/models/core/core_models.yml index 7464d302..10256463 100644 --- a/models/core/core_models.yml +++ b/models/core/core_models.yml @@ -18,7 +18,9 @@ models: - name: condition_id description: Unique identifier for each condition in the table. tests: - - unique + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for each patient in the dataset. - name: encounter_id @@ -98,6 +100,12 @@ models: tags: core materialized: table columns: + - name: eligibility_id + description: Unique identifier for each eligibility row in the table. + tests: + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for each patient in the dataset. - name: member_id @@ -164,6 +172,10 @@ models: columns: - name: encounter_id description: Unique identifier for each encounter. + tests: + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for a patient. - name: encounter_type @@ -270,7 +282,11 @@ models: materialized: table columns: - name: lab_result_id - description: Unique identifier for the lab test. + description: Unique identifier for each lab result. + tests: + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for each patient. - name: encounter_id @@ -368,6 +384,10 @@ models: columns: - name: location_id description: Unique identifier for each location. + tests: + - unique: + config: + severity: warn - name: npi description: > The national provider identifier associated with the location e.g. @@ -401,6 +421,7 @@ models: `dbt_utils.pretty_time` as the local time of the `dbt run` environment. Timezone is configurable via the `tuva_last_run` var. + - name: core__medical_claim description: > The medical claim table contains information on services rendered to @@ -412,10 +433,14 @@ models: alias: medical_claim tags: core materialized: table - tests: - - unique: - column_name: "(claim_id||'-'||claim_line_number)" columns: + - name: medical_claim_id + description: Unique identifier for each row in the table. + tests: + - unique: + config: + severity: error + - name: claim_id description: Unique identifier for each claim. - name: claim_line_number @@ -592,6 +617,10 @@ models: columns: - name: medication_id description: Unique identifier for each medication in the table. + tests: + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for each patient in the dataset. - name: encounter_id @@ -672,6 +701,10 @@ models: columns: - name: observation_id description: Unique identifier for each observation in the dataset. + tests: + - unique: + config: + severity: warn - name: patient_id description: Unique identifier for each patient in the dataset. - name: encounter_id @@ -734,6 +767,10 @@ models: columns: - name: patient_id description: Unique identifier for each person across all datasets. + tests: + - unique: + config: + severity: error - name: sex description: The gender of the patient. meta: @@ -793,6 +830,12 @@ models: tags: core materialized: table columns: + - name: pharmacy_claim_id + description: Unique identifier for each row in the table. + tests: + - unique: + config: + severity: error - name: claim_id description: Unique identifier for each claim. - name: claim_line_number @@ -874,6 +917,10 @@ models: columns: - name: practitioner_id description: Unique ID for the provider. + tests: + - unique: + config: + severity: warn - name: npi description: NPI for the provider. meta: @@ -914,7 +961,9 @@ models: - name: procedure_id description: The unique identifier for the performed procedure. tests: - - unique + - unique: + config: + severity: warn - name: encounter_id description: > The encounter_id for the encounter where this procedure was performed. diff --git a/models/core/staging/core__stg_claims_condition.sql b/models/core/staging/core__stg_claims_condition.sql index 2d7128f9..1b6423f7 100644 --- a/models/core/staging/core__stg_claims_condition.sql +++ b/models/core/staging/core__stg_claims_condition.sql @@ -13,6 +13,7 @@ with unpivot_cte as ( select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -32,6 +33,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -51,6 +53,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -70,6 +73,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -89,6 +93,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -108,6 +113,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -127,6 +133,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -146,6 +153,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -165,6 +173,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -184,6 +193,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -203,6 +213,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -222,6 +233,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -241,6 +253,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -260,6 +273,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -279,6 +293,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -298,6 +313,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -317,6 +333,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -336,6 +353,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -355,6 +373,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -374,6 +393,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -393,6 +413,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -412,6 +433,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -431,6 +453,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -450,6 +473,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -469,6 +493,7 @@ union all select claim_id + , claim_line_number , patient_id , coalesce(admission_date , claim_start_date @@ -487,7 +512,7 @@ where diagnosis_code_25 is not null ) select distinct - cast(unpivot_cte.data_source||'_'||unpivot_cte.claim_id||'_'||unpivot_cte.diagnosis_rank||'_'||unpivot_cte.source_code as {{ dbt.type_string() }} ) as condition_id + cast(unpivot_cte.claim_id||'_'||unpivot_cte.claim_line_number||'_'||unpivot_cte.diagnosis_rank||'_'||unpivot_cte.source_code as {{ dbt.type_string() }} ) as condition_id , cast(unpivot_cte.patient_id as {{ dbt.type_string() }} ) as patient_id , cast(coalesce(ap.encounter_id, ed.encounter_id) as {{ dbt.type_string() }} ) as encounter_id , cast(unpivot_cte.claim_id as {{ dbt.type_string() }} ) as claim_id diff --git a/models/core/staging/core__stg_claims_eligibility.sql b/models/core/staging/core__stg_claims_eligibility.sql index 1a604162..77e1245e 100644 --- a/models/core/staging/core__stg_claims_eligibility.sql +++ b/models/core/staging/core__stg_claims_eligibility.sql @@ -13,7 +13,9 @@ select - cast(patient_id as {{ dbt.type_string() }} ) as patient_id + cast(patient_id as {{ dbt.type_string() }} ) || '-' || cast(enrollment_start_date as {{ dbt.type_string() }} ) || '-' || cast(enrollment_end_date as {{ dbt.type_string() }} ) + || '-' || cast(payer as {{ dbt.type_string() }} ) || '-' || cast(plan as {{ dbt.type_string() }} ) as eligibility_id + , cast(patient_id as {{ dbt.type_string() }} ) as patient_id , cast(member_id as {{ dbt.type_string() }} ) as member_id , cast(subscriber_id as {{ dbt.type_string() }} ) as subscriber_id , cast(birth_date as date) as birth_date diff --git a/models/core/staging/core__stg_claims_location.sql b/models/core/staging/core__stg_claims_location.sql index 97bad9d8..34d99c5e 100644 --- a/models/core/staging/core__stg_claims_location.sql +++ b/models/core/staging/core__stg_claims_location.sql @@ -1,48 +1,49 @@ --- depends_on: {{ ref('data_quality__claims_preprocessing_summary') }} - -{{ config( - enabled = var('claims_enabled',var('tuva_marts_enabled',False)) | as_bool - ) -}} - -with all_providers_in_claims_dataset as ( -select distinct facility_npi as npi, data_source -from {{ ref('core__stg_claims_medical_claim') }} - -union all - -select distinct rendering_npi as npi, data_source -from {{ ref('core__stg_claims_medical_claim') }} - -union all - -select distinct billing_npi as npi, data_source -from {{ ref('core__stg_claims_medical_claim') }} -), - - -provider as ( -select aa.*, bb.data_source -from {{ ref('terminology__provider') }} aa -inner join all_providers_in_claims_dataset bb -on aa.npi = bb.npi -where lower(aa.entity_type_description) = 'organization' -) - - - -select - cast(npi as {{ dbt.type_string() }} ) as location_id - , cast(npi as {{ dbt.type_string() }} ) as npi - , cast(provider_organization_name as {{ dbt.type_string() }} ) as name - , cast(null as {{ dbt.type_string() }} ) as facility_type - , cast(parent_organization_name as {{ dbt.type_string() }} ) as parent_organization - , cast(practice_address_line_1 as {{ dbt.type_string() }} ) as address - , cast(practice_city as {{ dbt.type_string() }} ) as city - , cast(practice_state as {{ dbt.type_string() }} ) as state - , cast(practice_zip_code as {{ dbt.type_string() }} ) as zip_code - , cast(null as {{ dbt.type_float() }} ) as latitude - , cast(null as {{ dbt.type_float() }} ) as longitude - , cast(data_source as {{ dbt.type_string() }} ) as data_source - , cast( '{{ var('tuva_last_run')}}' as {{ dbt.type_timestamp() }} ) as tuva_last_run -from provider +-- depends_on: {{ ref('data_quality__claims_preprocessing_summary') }} + +{{ config( + enabled = var('claims_enabled',var('tuva_marts_enabled',False)) + | as_bool + ) +}} + +with all_providers_in_claims_dataset as ( +select distinct facility_npi as npi, data_source +from {{ ref('core__stg_claims_medical_claim') }} + +union all + +select distinct rendering_npi as npi, data_source +from {{ ref('core__stg_claims_medical_claim') }} + +union all + +select distinct billing_npi as npi, data_source +from {{ ref('core__stg_claims_medical_claim') }} +), + + +provider as ( +select aa.*, bb.data_source +from {{ ref('terminology__provider') }} aa +inner join all_providers_in_claims_dataset bb +on aa.npi = bb.npi +where lower(aa.entity_type_description) = 'organization' +) + + + +select + cast(npi as {{ dbt.type_string() }} ) as location_id + , cast(npi as {{ dbt.type_string() }} ) as npi + , cast(provider_organization_name as {{ dbt.type_string() }} ) as name + , cast(null as {{ dbt.type_string() }} ) as facility_type + , cast(parent_organization_name as {{ dbt.type_string() }} ) as parent_organization + , cast(practice_address_line_1 as {{ dbt.type_string() }} ) as address + , cast(practice_city as {{ dbt.type_string() }} ) as city + , cast(practice_state as {{ dbt.type_string() }} ) as state + , cast(practice_zip_code as {{ dbt.type_string() }} ) as zip_code + , cast(null as {{ dbt.type_float() }} ) as latitude + , cast(null as {{ dbt.type_float() }} ) as longitude + , cast(data_source as {{ dbt.type_string() }} ) as data_source + , cast( '{{ var('tuva_last_run')}}' as {{ dbt.type_timestamp() }} ) as tuva_last_run +from provider diff --git a/models/core/staging/core__stg_claims_medical_claim.sql b/models/core/staging/core__stg_claims_medical_claim.sql index 74f35959..97ac5c25 100644 --- a/models/core/staging/core__stg_claims_medical_claim.sql +++ b/models/core/staging/core__stg_claims_medical_claim.sql @@ -16,7 +16,8 @@ select - cast(med.claim_id as {{ dbt.type_string() }} ) as claim_id + cast(med.claim_id as {{ dbt.type_string() }} )|| '-' ||cast(med.claim_line_number as {{ dbt.type_string() }} ) as medical_claim_id + , cast(med.claim_id as {{ dbt.type_string() }} ) as claim_id , cast(med.claim_line_number as {{ dbt.type_int() }} ) as claim_line_number , cast(coalesce(ap.encounter_id,ed.encounter_id) as {{ dbt.type_string() }} ) as encounter_id , cast(med.claim_type as {{ dbt.type_string() }} ) as claim_type diff --git a/models/core/staging/core__stg_claims_pharmacy_claim.sql b/models/core/staging/core__stg_claims_pharmacy_claim.sql index 54b2ffe7..b11da0f8 100644 --- a/models/core/staging/core__stg_claims_pharmacy_claim.sql +++ b/models/core/staging/core__stg_claims_pharmacy_claim.sql @@ -14,7 +14,8 @@ select - cast(claim_id as {{ dbt.type_string() }} ) as claim_id + cast(claim_id as {{ dbt.type_string() }} ) || '-' || cast(claim_line_number as {{ dbt.type_string() }} ) as pharmacy_claim_id + , cast(claim_id as {{ dbt.type_string() }} ) as claim_id , cast(claim_line_number as integer ) as claim_line_number , cast(patient_id as {{ dbt.type_string() }} ) as patient_id , cast(member_id as {{ dbt.type_string() }} ) as member_id