Skip to content

Commit

Permalink
Add new extractor engine - :serialized-objects-array
Browse files Browse the repository at this point in the history
  • Loading branch information
ApricotLace committed Dec 16, 2022
1 parent 1cc305a commit 4a5146d
Show file tree
Hide file tree
Showing 7 changed files with 253 additions and 5 deletions.
3 changes: 2 additions & 1 deletion deps.edn
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
com.rpl/specter {:mvn/version "1.1.3"}
cli-matic/cli-matic {:mvn/version "0.4.3"}
http-kit/http-kit {:mvn/version "2.6.0"}
enlive/enlive {:mvn/version "1.1.6"}}
enlive/enlive {:mvn/version "1.1.6"}
clj-commons/clj-yaml {:mvn/version "1.0.26"}}

:aliases
{:nrepl
Expand Down
11 changes: 11 additions & 0 deletions src/ftr/core.clj
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,14 @@
::write-tag-index-hash])


(def serialized-objects-array-pipeline
[::extract-terminology
::write-terminology-file
::shape-ftr-layout
:ftr.post-write-coordination.core/coordinate
::write-tag-index-hash])


(defmethod u/*fn ::select-ftr-pipeline [{:as _ctx,
::keys [commit-type]
{:keys [source-type]} :cfg}]
Expand All @@ -118,6 +126,9 @@
[:append :snomed]
snomed-pipeline

[:append :serialized-objects-array]
serialized-objects-array-pipeline

[:tag-merge nil]
tag-merge-pipeline)})

Expand Down
6 changes: 4 additions & 2 deletions src/ftr/extraction/core.clj
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
(ns ftr.extraction.core
(:require [ftr.extraction.flat-table :as flat-table]
[ftr.extraction.ig.core]
[ftr.extraction.snomed]))
[ftr.extraction.snomed]
[ftr.extraction.serialized-objects-array]))

(defn extract [cfg]
(let [{:keys [source-type source-url extractor-options]} cfg
Expand All @@ -12,4 +13,5 @@
:flat-table (flat-table/import-from-cfg extractor-cfg)
:ig (ftr.extraction.ig.core/import-from-cfg extractor-cfg)
:igs (ftr.extraction.ig.core/import-from-cfg extractor-cfg)
:snomed (ftr.extraction.snomed/import-from-cfg extractor-cfg))))
:snomed (ftr.extraction.snomed/import-from-cfg extractor-cfg)
:serialized-objects-array (ftr.extraction.serialized-objects-array/import-from-cfg extractor-cfg))))
3 changes: 1 addition & 2 deletions src/ftr/extraction/flat_table.clj
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
[ftr.utils.unifn.core :as u]
[ftr.utils.core]
[clojure.data.csv :as csv]
[clojure.string :as str]
[cheshire.core]))
[clojure.string :as str]))

(defmethod u/*fn ::create-value-set [cfg]
{::result {:value-set
Expand Down
99 changes: 99 additions & 0 deletions src/ftr/extraction/serialized_objects_array.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
(ns ftr.extraction.serialized-objects-array
(:require [clojure.java.io :as io]
[ftr.utils.unifn.core :as u]
[ftr.utils.core]
[clojure.string :as str]
[clj-yaml.core]))


(defmethod u/*fn ::create-value-set [cfg]
{::result {:value-set
(-> (:value-set cfg)
(assoc :resourceType "ValueSet")
(->> (merge {:status "unknown"
:compose {:include [{:system (get-in cfg [:code-system :url])}]}})))}})


(defmethod u/*fn ::create-code-system [cfg]
{::result {:code-system (-> (:code-system cfg)
(assoc :resourceType "CodeSystem")
(->> (merge {:status "unknown"
:content "not-present"
:valueSet (get-in cfg [:value-set :url])})
(conj #{})))}})


(defn extract-mapped-value [mapping source]
(update-vals mapping
(fn [el]
(let [v (get-in source (:path el))]
(cond-> el
(not (str/blank? v))
(assoc :value v))))))


(defn extract-mapping [deserialized-object mapping]
(update-vals mapping #(extract-mapped-value % deserialized-object)))

(ftr.utils.core/strip-nils {})

(defn format-concept-resource [extracted-mapping {:keys [system valueset]}]
(let [code (get-in extracted-mapping [:concept :code :value])
display (get-in extracted-mapping [:concept :display :value])
status (get-in extracted-mapping [:concept :deprecated? :value])
deprecation-mark (get-in extracted-mapping [:concept :deprecated? :true-values])
parent-id (get-in extracted-mapping [:concept :parent-id :value])
hierarchy-id (get-in extracted-mapping [:concept :hierarchy-id :value])
concept (ftr.utils.core/strip-nils
{:resourceType "Concept"
:system system
:valueset [valueset]
:code code
:deprecated (if (some #(= status %) deprecation-mark) true nil)
:display display
:parent-id parent-id
:hierarchy-id hierarchy-id
:property (-> (:property extracted-mapping)
(update-vals :value)
ftr.utils.core/strip-nils
not-empty)})]
(when (:code concept)
concept)))


(defn process-deserialized-objects [deserialized-objects
{:as cfg, :keys [mapping]}]
(->> deserialized-objects
(map (fn process-deserialized-object [do]
(-> do
(extract-mapping mapping)
(format-concept-resource cfg))))
(filter identity)))


(defmethod u/*fn ::import [{:as cfg,
:keys [source-url]}]
(let [deserialized-objects
(clj-yaml.core/parse-stream (io/reader source-url))

deserialized-objects-seq
(process-deserialized-objects
deserialized-objects
(merge cfg
{:system (get-in cfg [:code-system :url])
:valueset (get-in cfg [:value-set :url])
:mapping (:mapping cfg)}))]
{::result {:concepts deserialized-objects-seq}}))


(defn import-from-cfg [cfg]
(::result (u/*apply [::create-value-set
::create-code-system
::import] cfg)))

(comment

(clj-yaml.core/parse-stream (io/reader "/Users/ghrp/Downloads/external-care-team-relationship.yaml") )


)
24 changes: 24 additions & 0 deletions test/fixture/yaml/external-care-team-relationship.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
- code: "SPS"
display: "Spouse"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- code: "CAREGIVER"
display: "Caregiver"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- code: "CHILD"
display: "Children"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- code: "NMTH"
display: "Parents"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- code: "SIGOTHR"
display: "Significant Other"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- code: "HOMPART"
display: "Partner(s)"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- Healthcare Proxy
- Relative
- code: "STPPRN"
display: "Step-Parent(s)"
system: "http://terminology.hl7.org/CodeSystem/v3-RoleCode"
- Other
112 changes: 112 additions & 0 deletions test/ftr/core_test.clj
Original file line number Diff line number Diff line change
Expand Up @@ -970,3 +970,115 @@
(clean-up-test-env!))

)


(t/deftest serialized-objects-array-source-type-test
(def soa-test-env-cfg {:ftr-path "/tmp/soa/ftr"
:soa-source "/Users/ghrp/dev/ftr/test/fixture/yaml/external-care-team-relationship.yaml"})


(def soa-user-cfg {:module "soa"
:source-url (:soa-source soa-test-env-cfg)
:ftr-path (:ftr-path soa-test-env-cfg)
:tag "prod"
:source-type :serialized-objects-array
:extractor-options {:format "yaml"
:mapping {:concept {:code {:path [:code]}
:display {:path [:display]}}}
:code-system {:id "soa"
:url "http://soa/cs"}
:value-set {:id "soa"
:url "http://soa/vs"}}})
(t/testing "Clean up test-env"
(ftr.utils.core/rmrf (:ftr-path soa-test-env-cfg)))

(t/testing "User provides config for SOA"


(let [{:as user-cfg, :keys [module ftr-path tag]
{{value-set-name :url} :value-set} :extractor-options}
soa-user-cfg

value-set-name (ftr.utils.core/escape-url value-set-name)

_
(sut/apply-cfg {:cfg user-cfg})

ftr-tree
(get-in (fs-tree->tree-map ftr-path) (str/split (subs ftr-path 1) #"/"))]

(t/testing "sees generated repository layout, tf sha is correct"
(matcho/match
ftr-tree
{"soa"
{"vs"
{"http:--soa-vs"
{"tf.92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b.ndjson.gz"
{},
"tag.prod.ndjson.gz" {}}},
"tags" {"prod.ndjson.gz" {}, "prod.hash" {}}}}))

(t/testing "sees terminology tag file"
(matcho/match
(ftr.utils.core/parse-ndjson-gz
"/tmp/soa/ftr/soa/tags/prod.ndjson.gz")
[{:hash
"92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b"
:name "soa.http:--soa-vs"}
nil?]))

(t/testing "sees terminology file"
(matcho/match
(ftr.utils.core/parse-ndjson-gz
"/tmp/soa/ftr/soa/vs/http:--soa-vs/tf.92be9f413299bc56f798f6855a3f018b73f676e353e5faae6b51ebe8d946777b.ndjson.gz")
[{:resourceType "CodeSystem"
:url "http://soa/cs"
:valueSet "http://soa/vs"}
{:resourceType "ValueSet"
:url "http://soa/vs"}
{:code "CAREGIVER"
:display "Caregiver"
:id "http:--soa-cs-http:--soa-vs-CAREGIVER"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "CHILD"
:display "Children"
:id "http:--soa-cs-http:--soa-vs-CHILD"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "HOMPART"
:display "Partner(s)"
:id "http:--soa-cs-http:--soa-vs-HOMPART"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "NMTH"
:display "Parents"
:id "http:--soa-cs-http:--soa-vs-NMTH"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "SIGOTHR"
:display "Significant Other"
:id "http:--soa-cs-http:--soa-vs-SIGOTHR"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "SPS"
:display "Spouse"
:id "http:--soa-cs-http:--soa-vs-SPS"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
{:code "STPPRN"
:display "Step-Parent(s)"
:id "http:--soa-cs-http:--soa-vs-STPPRN"
:resourceType "Concept"
:system "http://soa/cs"
:valueset ["http://soa/vs"]}
nil?]))))

(t/testing "Clean up test-env"
(ftr.utils.core/rmrf (:ftr-path soa-test-env-cfg))))

0 comments on commit 4a5146d

Please sign in to comment.