Merge branch 'develop' of https://github.com/ClarityNLP/ClarityNLP in…

…to develop

env.sh

@@ -150,4 +150,14 @@ export CLARITYNLPAAS_CUSTOM_S3_URL=
 export CLARITYNLPAAS_CUSTOM_DIR=custom_s3
 export CLARITYNLPAAS_CLARITYNLP_URL=${NLP_API_URL}
 export CLARITYNLPAAS_TOKEN_URL=
-export CLARITYNLPAAS_SECRET=
+export CLARITYNLPAAS_SECRET=
+
+# CQL / FHIR
+export FHIR_CQL_EVAL_URL=https://gt-apps.hdap.gatech.edu/cql/evaluate
+export FHIR_DATA_SERVICE_URI=https://apps.hdap.gatech.edu/gt-fhir/fhir/
+export FHIR_AUTH_TYPE=
+export FHIR_AUTH_TOKEN=
+export FHIR_TERMINOLOGY_SERVICE_URI=https://cts.nlm.nih.gov/fhir/
+export FHIR_TERMINOLOGY_SERVICE_ENDPOINT=Terminology Service Endpoint
+export FHIR_TERMINOLOGY_USER_NAME=username
+export FHIR_TERMINOLOGY_USER_PASSWORD=password

nlp/custom_tasks/CQLExecutionTask.py

@@ -353,7 +353,7 @@ def _extract_condition_resource(obj, mongo_obj):
     result_display_obj = {
         'date': mongo_obj['datetime'],
         'result_content':'Condition: {0}, onset: {1}, abatement: {2}'.format(
-            condition_name, onset_date_time, abatement_date_time),
+            condition_name, mongo_obj['datetime'], mongo_obj['end_datetime']),
         'sentence':'',
         'highlights':[condition_name]
     }
@@ -428,13 +428,17 @@ def _extract_observation_resource(obj, mongo_obj):
 def _get_custom_arg(str_key, str_variable_name, job_id, custom_arg_dict):
     """
     Extract a value at the given key from the given dict, or return None
-    if not found.
+    if not found. Attempt to read from environmental variables, if known.
     """
 
     value = None
     if str_key in custom_arg_dict:
         value = custom_arg_dict[str_key]
 
+    if not value:
+        if str_key in util.properties:
+            value = util.properties[str_key]
+
     # echo in job status and in log file
     msg = 'CQLExecutionTask: {0} == {1}'.format(str_variable_name, value)
     data_access.update_job_status(job_id,

nlp/data_access/results.py

@@ -89,7 +89,6 @@ def phenotype_performance_results(jobs: list):
     return metrics
 
 
-
 def phenotype_feedback_results(job: str):
     job_type = 'annotations'
     client = util.mongo_client()
@@ -253,7 +252,6 @@ def lookup_phenotype_result_by_id(id: str):
         traceback.print_exc(file=sys.stdout)
         obj['success'] = False
 
-
     return obj
 
 

nlp/example.cfg

@@ -65,9 +65,11 @@ use_redis_caching=false
 [local]
 debug=false
 evaluator=mongo
-fhir_patient_id=38587
-fhir_url=https://gt-apps.hdap.gatech.edu/cql/evaluate
-fhir_service_uri=Terminology Service Endpoint
+cql_eval_url=https://gt-apps.hdap.gatech.edu/cql/evaluate
+fhir_patient_id=
+fhir_auth_type=
+fhir_auth_token=
+fhir_terminology_service_endpoint=Terminology Service Endpoint
 fhir_data_service_uri=https://apps.hdap.gatech.edu/gt-fhir/fhir/
 fhir_terminology_service_uri=https://cts.nlm.nih.gov/fhir/
 fhir_terminology_user=username

nlp/luigi_tools/phenotype_helper.py

@@ -129,6 +129,7 @@ def get_item_by_key(dictionary, keys: list):
 manually_mapped_keys = ['owner', 'limit', 'name', 'config_type', 'terms', 'cohort', 'job_results', 'concept_code',
                         'concept_code_system', 'cql', 'cql_source', 'named_arguments', 'display_name']
 
+
 def map_arguments(pipeline: PipelineConfig, e, all_terms):
     for k in e.keys():
         if k not in manually_mapped_keys:

nlp/util.py

@@ -1,5 +1,7 @@
 import configparser
 from os import getenv, environ, path
+
+import pymongo
 import redis
 from pymongo import MongoClient
 
@@ -12,7 +14,10 @@
 EXPIRE_TIME_SECONDS = 604800
 
 
-def read_property(env_name, config_tuple, default=''):
+def read_property(env_name, config_tuple, default='', key_name=None):
+    if not key_name:
+        key_name = env_name
+    global properties
     property_name = default
     try:
         if getenv(env_name):
@@ -21,11 +26,11 @@ def read_property(env_name, config_tuple, default=''):
             property_name = config.get(config_tuple[0], config_tuple[1])
         if not property_name:
             property_name = default
-        if len(env_name) > 0 and 'PASSWORD' not in env_name and 'KEY' not in env_name and 'USERNAME' not in env_name:
-            properties[env_name] = property_name
+        if len(key_name) > 0 and 'PASSWORD' not in key_name and 'KEY' not in key_name and 'USERNAME' not in key_name:
+            properties[key_name] = property_name
     except Exception as ex:
         print(ex)
-        properties[env_name] = default
+        properties[key_name] = default
     return property_name
 
 
@@ -111,6 +116,21 @@ def read_boolean_property(prop, default=False):
                                   ('optimizations', 'use_redis_caching'),
                                   default='true')
 
+cql_eval_url = read_property('FHIR_CQL_EVAL_URL', ('local', 'cql_eval_url'), key_name='cql_eval_url')
+fhir_data_service_uri = read_property('FHIR_DATA_SERVICE_URI', ('local', 'fhir_data_service_uri'),
+                                      key_name='fhir_data_service_uri')
+fhir_auth_type = read_property('FHIR_AUTH_TYPE', ('local', 'fhir_auth_type'), key_name='fhir_auth_type')
+fhir_auth_token = read_property('FHIR_AUTH_TOKEN', ('local', 'fhir_auth_token'), key_name='fhir_auth_token')
+fhir_terminology_service_uri = read_property('FHIR_TERMINOLOGY_SERVICE_URI', ('local', 'fhir_terminology_service_uri'),
+                                             key_name='fhir_terminology_service_uri')
+fhir_terminology_service_endpoint = read_property('FHIR_TERMINOLOGY_SERVICE_ENDPOINT',
+                                                  ('local', 'fhir_terminology_service_endpoint'),
+                                                  key_name='fhir_terminology_service_endpoint')
+fhir_terminology_user_name = read_property('FHIR_TERMINOLOGY_USER_NAME', ('local', 'fhir_terminology_user'),
+                                           key_name='fhir_terminology_user_name')
+fhir_terminology_user_password = read_property('FHIR_TERMINOLOGY_USER_PASSWORD', ('local', 'fhir_terminology_password'),
+                                               key_name='fhir_terminology_user_password')
+
 # TODO this out a bit more, this is more for experimental evaluation
 cache_counts = {
     'compute': 0,
@@ -184,8 +204,10 @@ def __cmp__(self, other):
 
     return K
 
+
 _mongo_client = None
 
+
 def mongo_client(host=None, port=None, username=None, password=None):
     global _mongo_client
     if _mongo_client:
@@ -212,7 +234,7 @@ def mongo_client(host=None, port=None, username=None, password=None):
     if username and len(username) > 0 and password and len(password) > 0:
         # print('authenticated mongo')
         _mongo_client = MongoClient(host=host, port=port, username=username,
-                           password=password, socketTimeoutMS=15000, maxPoolSize=500,
+                                    password=password, socketTimeoutMS=15000, maxPoolSize=500,
                                     maxIdleTimeMS=30000)
     else:
         # print('unauthenticated mongo')