From c11e08f02b584359384a01afd130c1a5331d94ce Mon Sep 17 00:00:00 2001 From: Charity Hilton Date: Tue, 6 Aug 2019 14:48:51 -0400 Subject: [PATCH 1/2] add CQL/FHIR properties as environment properties --- env.sh | 12 +++++++++++- nlp/custom_tasks/CQLExecutionTask.py | 6 +++++- nlp/data_access/results.py | 2 -- nlp/example.cfg | 8 +++++--- nlp/luigi_tools/phenotype_helper.py | 1 + nlp/util.py | 26 ++++++++++++++++++++++++-- 6 files changed, 46 insertions(+), 9 deletions(-) diff --git a/env.sh b/env.sh index 65d53158..d6b3b81d 100644 --- a/env.sh +++ b/env.sh @@ -150,4 +150,14 @@ export CLARITYNLPAAS_CUSTOM_S3_URL= export CLARITYNLPAAS_CUSTOM_DIR=custom_s3 export CLARITYNLPAAS_CLARITYNLP_URL=${NLP_API_URL} export CLARITYNLPAAS_TOKEN_URL= -export CLARITYNLPAAS_SECRET= \ No newline at end of file +export CLARITYNLPAAS_SECRET= + +# CQL / FHIR +export FHIR_CQL_EVAL_URL=https://gt-apps.hdap.gatech.edu/cql/evaluate +export FHIR_DATA_SERVICE_URI=https://apps.hdap.gatech.edu/gt-fhir/fhir/ +export FHIR_AUTH_TYPE= +export FHIR_AUTH_TOKEN= +export FHIR_TERMINOLOGY_SERVICE_URI=https://cts.nlm.nih.gov/fhir/ +export FHIR_TERMINOLOGY_SERVICE_ENDPOINT=Terminology Service Endpoint +export FHIR_TERMINOLOGY_USER_NAME=username +export FHIR_TERMINOLOGY_USER_PASSWORD=password diff --git a/nlp/custom_tasks/CQLExecutionTask.py b/nlp/custom_tasks/CQLExecutionTask.py index 213a2cc1..b9269be3 100644 --- a/nlp/custom_tasks/CQLExecutionTask.py +++ b/nlp/custom_tasks/CQLExecutionTask.py @@ -353,7 +353,7 @@ def _extract_condition_resource(obj, mongo_obj): result_display_obj = { 'date': mongo_obj['datetime'], 'result_content':'Condition: {0}, onset: {1}, abatement: {2}'.format( - condition_name, onset_date_time, abatement_date_time), + condition_name, mongo_obj['datetime'], mongo_obj['end_datetime']), 'sentence':'', 'highlights':[condition_name] } @@ -435,6 +435,10 @@ def _get_custom_arg(str_key, str_variable_name, job_id, custom_arg_dict): if str_key in custom_arg_dict: value = custom_arg_dict[str_key] + if not value: + if str_key in util.properties: + value = util.properties[str_key] + # echo in job status and in log file msg = 'CQLExecutionTask: {0} == {1}'.format(str_variable_name, value) data_access.update_job_status(job_id, diff --git a/nlp/data_access/results.py b/nlp/data_access/results.py index 8eaa2b02..f3bc288f 100644 --- a/nlp/data_access/results.py +++ b/nlp/data_access/results.py @@ -89,7 +89,6 @@ def phenotype_performance_results(jobs: list): return metrics - def phenotype_feedback_results(job: str): job_type = 'annotations' client = util.mongo_client() @@ -253,7 +252,6 @@ def lookup_phenotype_result_by_id(id: str): traceback.print_exc(file=sys.stdout) obj['success'] = False - return obj diff --git a/nlp/example.cfg b/nlp/example.cfg index 1aa29721..df8a2f90 100644 --- a/nlp/example.cfg +++ b/nlp/example.cfg @@ -65,9 +65,11 @@ use_redis_caching=false [local] debug=false evaluator=mongo -fhir_patient_id=38587 -fhir_url=https://gt-apps.hdap.gatech.edu/cql/evaluate -fhir_service_uri=Terminology Service Endpoint +cql_eval_url=https://gt-apps.hdap.gatech.edu/cql/evaluate +fhir_patient_id= +fhir_auth_type= +fhir_auth_token= +fhir_terminology_service_endpoint=Terminology Service Endpoint fhir_data_service_uri=https://apps.hdap.gatech.edu/gt-fhir/fhir/ fhir_terminology_service_uri=https://cts.nlm.nih.gov/fhir/ fhir_terminology_user=username diff --git a/nlp/luigi_tools/phenotype_helper.py b/nlp/luigi_tools/phenotype_helper.py index 879c53d3..7c850d5d 100644 --- a/nlp/luigi_tools/phenotype_helper.py +++ b/nlp/luigi_tools/phenotype_helper.py @@ -129,6 +129,7 @@ def get_item_by_key(dictionary, keys: list): manually_mapped_keys = ['owner', 'limit', 'name', 'config_type', 'terms', 'cohort', 'job_results', 'concept_code', 'concept_code_system', 'cql', 'cql_source', 'named_arguments', 'display_name'] + def map_arguments(pipeline: PipelineConfig, e, all_terms): for k in e.keys(): if k not in manually_mapped_keys: diff --git a/nlp/util.py b/nlp/util.py index 177ed809..82abfa85 100644 --- a/nlp/util.py +++ b/nlp/util.py @@ -1,5 +1,7 @@ import configparser from os import getenv, environ, path + +import pymongo import redis from pymongo import MongoClient @@ -12,7 +14,10 @@ EXPIRE_TIME_SECONDS = 604800 -def read_property(env_name, config_tuple, default=''): +def read_property(env_name, config_tuple, default='', key_name=None): + if not key_name: + key_name = env_name + global properties property_name = default try: if getenv(env_name): @@ -111,6 +116,21 @@ def read_boolean_property(prop, default=False): ('optimizations', 'use_redis_caching'), default='true') +cql_eval_url = read_property('FHIR_CQL_EVAL_URL', ('local', 'cql_eval_url'), key_name='cql_eval_url') +fhir_data_service_uri = read_property('FHIR_DATA_SERVICE_URI', ('local', 'fhir_data_service_uri'), + key_name='fhir_data_service_uri') +fhir_auth_type = read_property('FHIR_AUTH_TYPE', ('local', 'fhir_auth_type'), key_name='fhir_auth_type') +fhir_auth_token = read_property('FHIR_AUTH_TOKEN', ('local', 'fhir_auth_token'), key_name='fhir_auth_token') +fhir_terminology_service_uri = read_property('FHIR_TERMINOLOGY_SERVICE_URI', ('local', 'fhir_terminology_service_uri'), + key_name='fhir_terminology_service_uri') +fhir_terminology_service_endpoint = read_property('FHIR_TERMINOLOGY_SERVICE_ENDPOINT', + ('local', 'fhir_terminology_service_endpoint'), + key_name='fhir_terminology_service_endpoint') +fhir_terminology_user_name = read_property('FHIR_TERMINOLOGY_USER_NAME', ('local', 'fhir_terminology_user'), + key_name='fhir_terminology_user_name') +fhir_terminology_user_password = read_property('FHIR_TERMINOLOGY_USER_PASSWORD', ('local', 'fhir_terminology_password'), + key_name='fhir_terminology_user_password') + # TODO this out a bit more, this is more for experimental evaluation cache_counts = { 'compute': 0, @@ -184,8 +204,10 @@ def __cmp__(self, other): return K + _mongo_client = None + def mongo_client(host=None, port=None, username=None, password=None): global _mongo_client if _mongo_client: @@ -212,7 +234,7 @@ def mongo_client(host=None, port=None, username=None, password=None): if username and len(username) > 0 and password and len(password) > 0: # print('authenticated mongo') _mongo_client = MongoClient(host=host, port=port, username=username, - password=password, socketTimeoutMS=15000, maxPoolSize=500, + password=password, socketTimeoutMS=15000, maxPoolSize=500, maxIdleTimeMS=30000) else: # print('unauthenticated mongo') From 0fa5159c2b7c5c55a023898bf7eb199e1f1cf3f3 Mon Sep 17 00:00:00 2001 From: Charity Hilton Date: Tue, 6 Aug 2019 14:54:11 -0400 Subject: [PATCH 2/2] FHIR / CQL config : fix property lookup key --- nlp/custom_tasks/CQLExecutionTask.py | 2 +- nlp/util.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nlp/custom_tasks/CQLExecutionTask.py b/nlp/custom_tasks/CQLExecutionTask.py index b9269be3..df7b3708 100644 --- a/nlp/custom_tasks/CQLExecutionTask.py +++ b/nlp/custom_tasks/CQLExecutionTask.py @@ -428,7 +428,7 @@ def _extract_observation_resource(obj, mongo_obj): def _get_custom_arg(str_key, str_variable_name, job_id, custom_arg_dict): """ Extract a value at the given key from the given dict, or return None - if not found. + if not found. Attempt to read from environmental variables, if known. """ value = None diff --git a/nlp/util.py b/nlp/util.py index 82abfa85..144800a9 100644 --- a/nlp/util.py +++ b/nlp/util.py @@ -26,11 +26,11 @@ def read_property(env_name, config_tuple, default='', key_name=None): property_name = config.get(config_tuple[0], config_tuple[1]) if not property_name: property_name = default - if len(env_name) > 0 and 'PASSWORD' not in env_name and 'KEY' not in env_name and 'USERNAME' not in env_name: - properties[env_name] = property_name + if len(key_name) > 0 and 'PASSWORD' not in key_name and 'KEY' not in key_name and 'USERNAME' not in key_name: + properties[key_name] = property_name except Exception as ex: print(ex) - properties[env_name] = default + properties[key_name] = default return property_name