Skip to content

Commit

Permalink
Fixed some bugs.
Browse files Browse the repository at this point in the history 
- Notebook 0: create folder './res' if it does not exist
- Notebook 5: import re
- Notebook 8: build indices to speedup
- Notebook 10: use list to initialize OrderedDict instead of using dict
  • Loading branch information
@mengcz13
mengcz13 committed Oct 31, 2018
1 parent a92bd76 commit 7b34965
Show file tree
Hide file tree
Showing 15 changed files with 27,192 additions and 6,467 deletions.
6 changes: 4 additions & 2 deletions Codes/mimic3_mvcv/0_createAdmissionList.ipynb
View file
Expand Up @@ -111,6 +111,8 @@
},
"outputs": [],
"source": [
"if not os.path.exists('./res'):\n",
" os.makedirs('./res')\n",
"# save to admission_ids.npy\n",
"tosave = {'admission_ids':admission_ids, 'admission_ids_txt': admission_ids_txt}\n",
"np.save('res/admission_ids.npy',tosave)"
Expand Down Expand Up @@ -151,7 +153,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -193,7 +195,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.2"
"version": "3.6.1"
}
},
"nbformat": 4,
Expand Down
72 changes: 26 additions & 46 deletions Codes/mimic3_mvcv/10_get_17-features-processed(fromdb).ipynb
View file
Expand Up @@ -76,26 +76,6 @@
"print(len(valid_aids), valid_aids[:10])"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# raw features: features directly from database or only need simple pre-processing like sum/mean\n",
"# TODO...\n",
"raw_features = OrderedDict({\n",
" 'ts': OrderedDict({\n",
" \n",
" }),\n",
" 'static': OrderedDict({\n",
" \n",
" })\n",
"})"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -118,31 +98,31 @@
"source": [
"# 17 features: features used in calculating SAPS II score\n",
"# Here mean/max/min is done for values with the same aid and the same timestamp, only for solving conflict\n",
"features = OrderedDict({\n",
" 'ts': OrderedDict({\n",
" 'glasgow_coma_scale': ['mengcz_glasgow_coma_scale_ts', 'GCS'],\n",
" 'systolic_blood_pressure': ['mengcz_vital_ts', 'SysBP_Mean'],\n",
" 'heart_rate': ['mengcz_vital_ts', 'HeartRate_Mean'],\n",
" 'body_temperature': ['mengcz_vital_ts', 'TempC_Mean'],\n",
"# 'pao2_fio2_ratio': ['mengcz_pao2fio2_ts', 'PaO2FiO2'],\n",
" 'pao2': ['mengcz_pao2fio2_ts', 'PO2'],\n",
" 'fio2': ['mengcz_pao2fio2_ts', 'FIO2'],\n",
" 'urinary_output': ['mengcz_urine_output_ts', 'UrineOutput'],\n",
" 'serum_urea_nitrogen_level': ['mengcz_labs_ts', 'BUN_min'],\n",
" 'white_blood_cells_count': ['mengcz_labs_ts', 'WBC_min'],\n",
" 'serum_bicarbonate_level': ['mengcz_labs_ts', 'BICARBONATE_min'],\n",
" 'sodium_level': ['mengcz_labs_ts', 'SODIUM_min'],\n",
" 'potassium_level': ['mengcz_labs_ts', 'POTASSIUM_min'],\n",
" 'bilirubin_level': ['mengcz_labs_ts', 'BILIRUBIN_min'],\n",
" }),\n",
" 'static': OrderedDict({\n",
" 'age': ['mengcz_17features_first24h', 'age'],\n",
" 'aids': ['mengcz_17features_first24h', 'AIDS'],\n",
" 'hem': ['mengcz_17features_first24h', 'HEM'],\n",
" 'mets': ['mengcz_17features_first24h', 'METS'],\n",
" 'admission_type': ['mengcz_17features_first24h', 'AdmissionType'],\n",
" })\n",
"})"
"features = OrderedDict([\n",
" ['ts', OrderedDict([\n",
" ['glasgow_coma_scale', ['mengcz_glasgow_coma_scale_ts', 'GCS']],\n",
" ['systolic_blood_pressure', ['mengcz_vital_ts', 'SysBP_Mean']],\n",
" ['heart_rate', ['mengcz_vital_ts', 'HeartRate_Mean']],\n",
" ['body_temperature', ['mengcz_vital_ts', 'TempC_Mean']],\n",
"# ['pao2_fio2_ratio', ['mengcz_pao2fio2_ts', 'PaO2FiO2']],\n",
" ['pao2', ['mengcz_pao2fio2_ts', 'PO2']],\n",
" ['fio2', ['mengcz_pao2fio2_ts', 'FIO2']],\n",
" ['urinary_output', ['mengcz_urine_output_ts', 'UrineOutput']],\n",
" ['serum_urea_nitrogen_level', ['mengcz_labs_ts', 'BUN_min']],\n",
" ['white_blood_cells_count', ['mengcz_labs_ts', 'WBC_min']],\n",
" ['serum_bicarbonate_level', ['mengcz_labs_ts', 'BICARBONATE_min']],\n",
" ['sodium_level', ['mengcz_labs_ts', 'SODIUM_min']],\n",
" ['potassium_level', ['mengcz_labs_ts', 'POTASSIUM_min']],\n",
" ['bilirubin_level', ['mengcz_labs_ts', 'BILIRUBIN_min']],\n",
" ])],\n",
" ['static', OrderedDict([\n",
" ['age', ['mengcz_17features_first24h', 'age']],\n",
" ['aids', ['mengcz_17features_first24h', 'AIDS']],\n",
" ['hem', ['mengcz_17features_first24h', 'HEM']],\n",
" ['mets', ['mengcz_17features_first24h', 'METS']],\n",
" ['admission_type', ['mengcz_17features_first24h', 'AdmissionType']],\n",
" ])]\n",
"])"
]
},
{
Expand Down Expand Up @@ -1013,7 +993,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.2"
"version": "3.6.1"
}
},
"nbformat": 4,
Expand Down
108 changes: 54 additions & 54 deletions Codes/mimic3_mvcv/10_get_17-features-raw.ipynb
View file
Expand Up @@ -171,30 +171,30 @@
"source": [
"# derive 17 features from manually selected itemids\n",
"# https://docs.google.com/spreadsheets/d/1e2KqLn3LTvcUwpSe5oE2ADwIEmUH9Xh54VADYVQ9mEQ/edit?ts=5960262a#gid=750248768\n",
"feature_itemids = OrderedDict({\n",
" 'gcsverbal': [723, 223900],\n",
" 'gcsmotor': [454, 223901],\n",
" 'gcseyes': [184, 220739],\n",
"# 'glasgow_coma_scale': [454, 223900],\n",
"# 'systolic_blood_pressure_abp_high_6': [6, 220050],\n",
"# 'systolic_blood_pressure_abp_high_51': [51, 220050],\n",
"# 'systolic_blood_pressure_abp_high_6701': [6701, 220050],\n",
" 'systolic_blood_pressure_abp_mean': [51, 442, 455, 6701, 220050, 220179],\n",
"# 'systolic_blood_pressure_abp_high_mean': [6, 51, 6701, 220050],\n",
"# 'systolic_blood_pressure_abp_high_max': [6, 51, 6701, 220050],\n",
"# 'systolic_blood_pressure_abp_high_min': [6, 51, 6701, 220050],\n",
"# 'systolic_blood_pressure_abp_low': [6],\n",
"# 'systolic_blood_pressure_nbp_high': [455, 220179],\n",
"# 'systolic_blood_pressure_nbp_low': [],\n",
" 'heart_rate': [211, 220045],\n",
" 'body_temperature': {\n",
"feature_itemids = OrderedDict([\n",
" ['gcsverbal', [723, 223900]],\n",
" ['gcsmotor', [454, 223901]],\n",
" ['gcseyes', [184, 220739]],\n",
"# ['glasgow_coma_scale', [454, 223900]],\n",
"# ['systolic_blood_pressure_abp_high_6', [6, 220050]],\n",
"# ['systolic_blood_pressure_abp_high_51', [51, 220050]],\n",
"# ['systolic_blood_pressure_abp_high_6701', [6701, 220050]],\n",
" ['systolic_blood_pressure_abp_mean', [51, 442, 455, 6701, 220050, 220179]],\n",
"# ['systolic_blood_pressure_abp_high_mean', [6, 51, 6701, 220050]],\n",
"# ['systolic_blood_pressure_abp_high_max', [6, 51, 6701, 220050]],\n",
"# ['systolic_blood_pressure_abp_high_min', [6, 51, 6701, 220050]],\n",
"# ['systolic_blood_pressure_abp_low', [6]],\n",
"# ['systolic_blood_pressure_nbp_high', [455, 220179]],\n",
"# ['systolic_blood_pressure_nbp_low', []],\n",
" ['heart_rate', [211, 220045]],\n",
" ['body_temperature', {\n",
" 'f': [678, 223761],\n",
" 'c': [676, 223762]\n",
" },\n",
" 'pao2': [50821],\n",
" 'fio2': [50816, 223835, 3420, 3422, 190],\n",
"# 'pao2_fio2_ratio': [50821, 50816],\n",
" 'urinary_output': [40055,\n",
" }],\n",
" ['pao2', [50821]],\n",
" ['fio2', [50816, 223835, 3420, 3422, 190]],\n",
"# ['pao2_fio2_ratio', [50821, 50816]],\n",
" ['urinary_output', [40055,\n",
"43175,\n",
"40069,\n",
"40094,\n",
Expand All @@ -219,37 +219,37 @@
"226557,\n",
"226558,\n",
"227488,\n",
"227489],\n",
" 'serum_urea_nitrogen_level': [51006],\n",
"# 'white_blood_cells_count_51300': [51300],\n",
"# 'white_blood_cells_count_51301': [51301],\n",
" 'white_blood_cells_count_mean': [51300, 51301],\n",
"# 'white_blood_cells_count_max': [51300, 51301],\n",
"# 'white_blood_cells_count_min': [51300, 51301],\n",
"# 'serum_bicarbonate_level_50803': [50803],\n",
"# 'serum_bicarbonate_level_50804': [50804],\n",
"# 'serum_bicarbonate_level_50802': [50802],\n",
" 'serum_bicarbonate_level_mean': [50882],\n",
"# 'serum_bicarbonate_level_max': [50803, 50804, 50802],\n",
"# 'serum_bicarbonate_level_min': [50803, 50804, 50802],\n",
"# 'sodium_level_50824': [50824],\n",
"# 'sodium_level_50983': [50983],\n",
" 'sodium_level_mean': [50824, 50983],\n",
"# 'sodium_level_max': [50824, 50983],\n",
"# 'sodium_level_min': [50824, 50983],\n",
"# 'potassium_level_50822': [50822],\n",
"# 'potassium_level_50971': [50971],\n",
" 'potassium_level_mean': [50822, 50971],\n",
"# 'potassium_level_max': [50822, 50971],\n",
"# 'potassium_level_min': [50822, 50971],\n",
" 'bilirubin_level': [50885],\n",
"# 'type_of_admission': [],\n",
"# 'acquired_immunodeficiency_syndrome': [],\n",
"# 'metastatic_cancer': [],\n",
"# 'hematologic_malignancy': []\n",
" 'timestamp': [],\n",
" 'aid': []\n",
"})\n",
"227489]],\n",
" ['serum_urea_nitrogen_level', [51006]],\n",
"# ['white_blood_cells_count_51300', [51300]],\n",
"# ['white_blood_cells_count_51301', [51301]],\n",
" ['white_blood_cells_count_mean', [51300, 51301]],\n",
"# ['white_blood_cells_count_max', [51300, 51301]],\n",
"# ['white_blood_cells_count_min', [51300, 51301]],\n",
"# ['serum_bicarbonate_level_50803', [50803]],\n",
"# ['serum_bicarbonate_level_50804', [50804]],\n",
"# ['serum_bicarbonate_level_50802', [50802]],\n",
" ['serum_bicarbonate_level_mean', [50882]],\n",
"# ['serum_bicarbonate_level_max', [50803, 50804, 50802]],\n",
"# ['serum_bicarbonate_level_min', [50803, 50804, 50802]],\n",
"# ['sodium_level_50824', [50824]],\n",
"# ['sodium_level_50983', [50983]],\n",
" ['sodium_level_mean', [50824, 50983]],\n",
"# ['sodium_level_max', [50824, 50983]],\n",
"# ['sodium_level_min', [50824, 50983]],\n",
"# ['potassium_level_50822', [50822]],\n",
"# ['potassium_level_50971', [50971]],\n",
" ['potassium_level_mean', [50822, 50971]],\n",
"# ['potassium_level_max', [50822, 50971]],\n",
"# ['potassium_level_min', [50822, 50971]],\n",
" ['bilirubin_level', [50885]],\n",
"# ['type_of_admission', []],\n",
"# ['acquired_immunodeficiency_syndrome', []],\n",
"# ['metastatic_cancer', []],\n",
"# ['hematologic_malignancy', []],\n",
" ['timestamp', []],\n",
" ['aid', []]\n",
"])\n",
"\n",
"merge_funcs = {\n",
" 'mean': np.mean,\n",
Expand Down Expand Up @@ -39601,7 +39601,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.2"
"version": "3.6.1"
}
},
"nbformat": 4,
Expand Down
92 changes: 46 additions & 46 deletions Codes/mimic3_mvcv/10_get_99plus-features-raw.ipynb
View file
Expand Up @@ -172,20 +172,20 @@
" feature_itemids[feature_name] = inmapitemids\n",
" \n",
"# features from previous 17 raw features\n",
"feature_itemids_17raw = OrderedDict({\n",
" 'gcsverbal': [723, 223900],\n",
" 'gcsmotor': [454, 223901],\n",
" 'gcseyes': [184, 220739],\n",
" 'systolic_blood_pressure_abp_mean': [51, 442, 455, 6701, 220050, 220179],\n",
" 'heart_rate': [211, 220045],\n",
" 'body_temperature': [\n",
"feature_itemids_17raw = OrderedDict([\n",
" ['gcsverbal', [723, 223900]],\n",
" ['gcsmotor', [454, 223901]],\n",
" ['gcseyes', [184, 220739]],\n",
" ['systolic_blood_pressure_abp_mean', [51, 442, 455, 6701, 220050, 220179]],\n",
" ['heart_rate', [211, 220045]],\n",
" ['body_temperature', [\n",
" ('_f2c', [678, 223761]),\n",
" 676,\n",
" 223762\n",
" ],\n",
" 'pao2': [50821],\n",
" 'fio2': [50816, 223835, 3420, 3422, 190],\n",
" 'urinary_output_sum': [40055,\n",
" ]],\n",
" ['pao2', [50821]],\n",
" ['fio2', [50816, 223835, 3420, 3422, 190]],\n",
" ['urinary_output_sum', [40055,\n",
"43175,\n",
"40069,\n",
"40094,\n",
Expand All @@ -210,14 +210,14 @@
"226557,\n",
"226558,\n",
"227488,\n",
"227489],\n",
" 'serum_urea_nitrogen_level': [51006],\n",
" 'white_blood_cells_count_mean': [51300, 51301],\n",
" 'serum_bicarbonate_level_mean': [50882],\n",
" 'sodium_level_mean': [50824, 50983],\n",
" 'potassium_level_mean': [50822, 50971],\n",
" 'bilirubin_level': [50885],\n",
"})\n",
"227489]],\n",
" ['serum_urea_nitrogen_level', [51006]],\n",
" ['white_blood_cells_count_mean', [51300, 51301]],\n",
" ['serum_bicarbonate_level_mean', [50882]],\n",
" ['sodium_level_mean', [50824, 50983]],\n",
" ['potassium_level_mean', [50822, 50971]],\n",
" ['bilirubin_level', [50885]],\n",
"])\n",
"\n",
"for key, value in feature_itemids_17raw.items():\n",
" feature_itemids[key] = value\n",
Expand All @@ -229,37 +229,37 @@
"]\n",
"\n",
"# add other features\n",
"extra_features = OrderedDict({\n",
" 'diastolic_blood_pressure_mean': [8368, 8440, 8441, 8555, 220180, 220051],\n",
" 'arterial_pressure_mean': [456, 52, 6702, 443, 220052, 220181, 225312],\n",
" 'respiratory_rate': [618, 615, 220210, 224690],\n",
" 'spo2_peripheral': [646, 220277],\n",
" 'glucose': [807, 811, 1529, 3745, 3744, 225664, 220621, 226537],\n",
" 'weight': [\n",
"extra_features = OrderedDict([\n",
" ['diastolic_blood_pressure_mean', [8368, 8440, 8441, 8555, 220180, 220051]],\n",
" ['arterial_pressure_mean', [456, 52, 6702, 443, 220052, 220181, 225312]],\n",
" ['respiratory_rate', [618, 615, 220210, 224690]],\n",
" ['spo2_peripheral', [646, 220277]],\n",
" ['glucose', [807, 811, 1529, 3745, 3744, 225664, 220621, 226537]],\n",
" ['weight', [\n",
" 762, 763, 3723, 3580, 226512,\n",
" ('_lb2kg', [3581]),\n",
" ('_oz2kg', [3582])\n",
" ],\n",
" 'height': [\n",
" ]],\n",
" ['height', [\n",
" ('_inches2cm', [920, 1394, 4187, 3486,]),\n",
" 3485, 4188, 226707\n",
" ],\n",
" 'hgb': [50811, 51222],\n",
" 'platelet': [51265],\n",
" 'chloride': [50806, 50902],\n",
" 'creatinine': [50912],\n",
" 'norepinephrine': [30047, 30120, 221906],\n",
" 'epinephrine': [30044,30119,30309,221289],\n",
" 'phenylephrine': [30127,30128,221749],\n",
" 'vasopressin': [30051,222315],\n",
" 'dopamine': [30043,30307,221662],\n",
" 'isuprel': [30046,227692],\n",
" 'midazolam': [30124, 221668],\n",
" 'fentanyl': [30150, 30308, 30118, 30149, 221744, 225972, 225942],\n",
" 'propofol': [30131, 222168],\n",
" 'peep': [50819],\n",
" 'ph': [50820],\n",
"})\n",
" ]],\n",
" ['hgb', [50811, 51222]],\n",
" ['platelet', [51265]],\n",
" ['chloride', [50806, 50902]],\n",
" ['creatinine', [50912]],\n",
" ['norepinephrine', [30047, 30120, 221906]],\n",
" ['epinephrine', [30044,30119,30309,221289]],\n",
" ['phenylephrine', [30127,30128,221749]],\n",
" ['vasopressin', [30051,222315]],\n",
" ['dopamine', [30043,30307,221662]],\n",
" ['isuprel', [30046,227692]],\n",
" ['midazolam', [30124, 221668]],\n",
" ['fentanyl', [30150, 30308, 30118, 30149, 221744, 225972, 225942]],\n",
" ['propofol', [30131, 222168]],\n",
" ['peep', [50819]],\n",
" ['ph', [50820]],\n",
"])\n",
"\n",
"for key, value in extra_features.items():\n",
" feature_itemids[key] = value\n",
Expand Down Expand Up @@ -22725,7 +22725,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.2"
"version": "3.6.1"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 7b34965

Please sign in to comment.