Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
2d2d91d
new tables for HL7 results import
spamhurts Jun 2, 2022
e3e5578
Added missing remark column to labwork pivot queries (#452) (#459)
spamhurts Jun 14, 2022
21ddb6a
Added missing columns
spamhurts Jun 14, 2022
f9f61eb
corrected spelling in comment
spamhurts Jun 15, 2022
a7a6209
- Added pivot tables for Clinical Path Lab data
spamhurts Jul 6, 2022
0b508d6
- Still working on pivot queries
spamhurts Jul 8, 2022
3cdc9c6
- Added SQL scripts in source_queries to migrate old clinpath data to…
spamhurts Jul 13, 2022
0f7d153
Added files missing from last commit
spamhurts Jul 13, 2022
e7b5bb6
Added study.labworkResults dataset to labworkResultsAll.sql query; TO…
spamhurts Jul 13, 2022
0f49852
Added missing metadata for HL7_MSH table
spamhurts Jul 13, 2022
808d29c
copy Sqlmed Msh to LK.sql - fixed column order and removed CAST from…
spamhurts Jul 14, 2022
1044833
- Added HL7_GroupNTE view to concatenate NTE records into a single fi…
spamhurts Jul 21, 2022
3f11e81
Migrated feature branch from 22.3 release branch.
spamhurts Jul 22, 2022
767e6b8
Merge branch '22.3_fb_HL7Integration' of https://github.com/LabKey/sn…
spamhurts Jul 25, 2022
7c2e43b
Integrating new ClinPath data source with existing datasource - refac…
spamhurts Aug 10, 2022
5dc30c9
-Integrating new ClinPath data source with existing datasource
spamhurts Aug 15, 2022
8fc7918
-Updated clinpathRunsAll.sql to use type column from clinpathRuns if …
spamhurts Aug 16, 2022
6d24ea2
Added missing column to snprc_ehr.xml schema
spamhurts Aug 16, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions snprc_ehr/resources/etls/SqlmedMsh.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<etl xmlns="http://labkey.org/etl/xml">
<name>Sqlmed MSH data</name>
<description>Sqlmed MSH Data (Synthetic)</description>
<transforms>
<transform id="step1" type="org.labkey.di.pipeline.TransformTask">
<description>Copy to target</description>
<source schemaName="snprcSrc" queryName="v_Sqlmed_msh" />
<destination schemaName="snprc_ehr" queryName="HL7_MSH" bulkLoad="true" batchSize="10000" targetOption="merge">
<alternateKeys>
<column name="object_id"/>
</alternateKeys>
</destination>
</transform>
</transforms>
<incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="timestamp">
<deletedRowsSource schemaName="snprcSrc" queryName="v_delete_clinpathruns" timestampColumnName="entry_date_tm" deletedSourceKeyColumnName="objectid" targetKeyColumnName="OBJECT_ID"/>
</incrementalFilter>
<schedule>
<poll interval="60m" />
</schedule>
</etl>
22 changes: 22 additions & 0 deletions snprc_ehr/resources/etls/SqlmedObr.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<etl xmlns="http://labkey.org/etl/xml">
<name>Sqlmed OBR data</name>
<description>Clinical Pathology OBR data</description>
<transforms>
<transform id="step1" type="org.labkey.di.pipeline.TransformTask">
<description>Copy to target</description>
<source schemaName="snprcSrc" queryName="v_Sqlmed_obr" />
<destination schemaName="snprc_ehr" queryName="HL7_OBR" bulkLoad="true" batchSize="10000" targetOption="merge">
<alternateKeys>
<column name="OBJECT_ID"/>
</alternateKeys>
</destination>
</transform>
</transforms>
<incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="timestamp">
<deletedRowsSource schemaName="snprcSrc" queryName="v_delete_clinpathruns" timestampColumnName="entry_date_tm" deletedSourceKeyColumnName="objectid" targetKeyColumnName="OBJECT_ID"/>
</incrementalFilter>
<schedule>
<poll interval="60m" />
</schedule>
</etl>
18 changes: 1 addition & 17 deletions snprc_ehr/resources/queries/pdl_workflow/pdlResults.query.xml
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,6 @@
<columnTitle>Ref Range</columnTitle>
<isUnselectable>true</isUnselectable>
<isHidden>false</isHidden>
<!-- <fk>
<fkDbSchema>study</fkDbSchema>
<fkTable>labRefRange</fkTable>
<fkColumnName>lsid</fkColumnName>
</fk> -->
</column>
<column columnName="abnormal_flags">
<columnTitle>Abnormal Flags</columnTitle>
Expand All @@ -74,27 +69,16 @@
<column columnName="method">
<nullable>true</nullable>
<columnTitle>Method</columnTitle>
<!--
<fk>
<fkDbSchema>ehr_lookups</fkDbSchema>
<fkTable>chemistry_method</fkTable>
<fkColumnName>value</fkColumnName>
</fk> -->
</column>

<column columnName="parentid">
<!--<fk>-->
<!--<fkDbSchema>study</fkDbSchema>-->
<!--<fkTable>chemistryRuns</fkTable>-->
<!--<fkColumnName>objectid</fkColumnName>-->
<!--</fk>-->
<nullable>true</nullable>
</column>
<column columnName="runid">
<isHidden>true</isHidden>
<fk>
<fkDbSchema>study</fkDbSchema>
<fkTable>ClinpathRuns</fkTable>
<fkTable>ClinpathRunsAll</fkTable>
<fkColumnName>objectid</fkColumnName>
</fk>
</column>
Expand Down
12 changes: 1 addition & 11 deletions snprc_ehr/resources/queries/pdl_workflow/pdlResults.sql
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,18 @@
Purpose: ETL from SPF assay tables to assay_labworkResults and ultimately SurveillancePivot
Author: Charles Peterson, Texas Biomedical Research Institute, October 17, 2018
*/
-- noinspection SqlNoDataSourceInspection
select distinct
case when d.snprc_id is NULL then
d.pdl_animal_id
else d.snprc_id end as Id,
COALESCE(d.snprc_id, d.pdl_animal_id) as Id,
d.sample_date as date,
d.procedure_name as panelName,
NULL as serviceTestId,
d.procedure_id as serviceId,
d.test_id as testid,
d.test_name as test_name,
/* d.Run as runId, */
NULL as runId,
NULL as result,
'ST' as value_type,
d.results as qualresult,
d.file_name as remark,
d.sample as sampleId

/* ServiceName debugging hack
,d.test_id as refRange,
d.procedure_id as abnormal_flags,
d.test_name as resultOORIndicator */

from pdl_workflow.data as d
4 changes: 1 addition & 3 deletions snprc_ehr/resources/queries/pdl_workflow/pdlWorkflow.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@
Author: Charles Peterson, Texas Biomedical Research Institute, September 20, 2018
*/
select distinct
case when d.snprc_id is NULL then
d.pdl_animal_id
else d.snprc_id end as Id,
COALESCE(d.snprc_id, d.pdl_animal_id) as Id,
d.sample_date as date,
d.report_date as datefinalized,
d.procedure_id as serviceId,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7BiochemistryPivot" tableDbType="NOT_IN_DB">
<tableTitle>Biochemistry Results Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
25 changes: 25 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7BiochemistryPivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
SELECT
obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
WHERE obr.PROCEDURE_ID.Dataset = 'Biochemistry'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.SET_ID, obr.PROCEDURE_NAME, obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName as TEST_NAME from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Biochemistry'
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<customView xmlns="http://labkey.org/data/xml/queryCustomView">
<sorts>
<sort column="Id"/>
<sort column="Date" descending="true"/>
</sorts>
</customView>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7CulturePivot" tableDbType="NOT_IN_DB">
<tableTitle>Culture and Sensitivity Results Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
25 changes: 25 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7CulturePivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
SELECT
obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
WHERE obr.PROCEDURE_ID.Dataset = 'Culture'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.SET_ID, obr.PROCEDURE_NAME, obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName as TEST_NAME from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Culture'
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<customView xmlns="http://labkey.org/data/xml/queryCustomView">
<sorts>
<sort column="Id"/>
<sort column="Date" descending="true"/>
</sorts>
</customView>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7HematologyPivot" tableDbType="NOT_IN_DB">
<tableTitle>Hematology Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
25 changes: 25 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7HematologyPivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
SELECT
obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
WHERE obr.PROCEDURE_ID.Dataset = 'Hematology'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.SET_ID, obr.PROCEDURE_NAME, obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName as TEST_NAME from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Hematology'
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<customView xmlns="http://labkey.org/data/xml/queryCustomView">
<sorts>
<sort column="Id"/>
<sort column="Date" descending="true"/>
</sorts>
</customView>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7HistologyPivot" tableDbType="NOT_IN_DB">
<tableTitle>Histology Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
24 changes: 24 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7HistologyPivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
SELECT obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
WHERE obr.PROCEDURE_ID.Dataset = 'Histology'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, obr.PROCEDURE_NAME, obr.SET_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Histology'
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<customView xmlns="http://labkey.org/data/xml/queryCustomView">
<sorts>
<sort column="Id"/>
<sort column="Date" descending="true"/>
</sorts>
</customView>
9 changes: 9 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7MiscPivot.query.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7MiscPivot" tableDbType="NOT_IN_DB">
<tableTitle>Misc Tests Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
24 changes: 24 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7MiscPivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
SELECT obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
WHERE obr.PROCEDURE_ID.Dataset = 'Misc Tests'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, obr.PROCEDURE_NAME, obr.SET_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Misc Tests'
)
6 changes: 6 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7MiscPivot/.qview.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<customView xmlns="http://labkey.org/data/xml/queryCustomView">
<sorts>
<sort column="Id"/>
<sort column="Date" descending="true"/>
</sorts>
</customView>
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<query xmlns="http://labkey.org/data/xml/query">
<metadata>
<tables xmlns="http://labkey.org/data/xml">
<table tableName="HL7ParasitologyPivot" tableDbType="NOT_IN_DB">
<tableTitle>Parasitology Pivot</tableTitle>
</table>
</tables>
</metadata>
</query>
24 changes: 24 additions & 0 deletions snprc_ehr/resources/queries/snprc_ehr/HL7ParasitologyPivot.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
SELECT obr.ANIMAL_ID as id,
obr.OBSERVATION_DATE_TM as date,
obr.MESSAGE_ID,
COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME) as PROCEDURE_NAME,
obr.PROCEDURE_ID,
nte.COMMENT,
COALESCE (lp.TestName, obx.TEST_NAME) as TestName,
MAX(obx.ABNORMAL_FLAGS) AS ABNORMAL_FLAGS,
COALESCE(MAX(obx.RESULT), MAX(obx.QUALITATIVE_RESULT)) as RESULT

FROM snprc_ehr.HL7_OBR obr
LEFT OUTER JOIN snprc_ehr.HL7_OBX obx ON obr.OBJECT_ID = obx.OBR_OBJECT_ID AND obr.SET_ID = obx.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.HL7_GroupNTE AS nte ON obr.OBJECT_ID = nte.OBR_OBJECT_ID AND obr.SET_ID = nte.OBR_SET_ID
LEFT OUTER JOIN snprc_ehr.labwork_Panels AS lp on obx.TEST_ID = lp.TestId AND obr.PROCEDURE_ID = lp.ServiceId
WHERE obr.PROCEDURE_ID.Dataset = 'Parasitology'

GROUP BY obr.ANIMAL_ID, obr.OBSERVATION_DATE_TM, obr.MESSAGE_ID, obr.PROCEDURE_NAME, obr.SET_ID, COALESCE (lp.ServiceId.ServiceName, obr.PROCEDURE_NAME), obr.PROCEDURE_ID, nte.COMMENT, COALESCE (lp.TestName, obx.TEST_NAME)
PIVOT RESULT, ABNORMAL_FLAGS BY TestName

IN
(
select TestName from snprc_ehr.labwork_panels t
where t.includeInPanel = true AND t.ServiceId.Dataset='Parasitology'
)
Loading