Automated building of materialized views

.gitignore

@@ -1,3 +1,9 @@
+# Data files
+*.csv
+
+# Picture files
+*.png
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.travis.yml

@@ -18,19 +18,20 @@ services:
 # Set the language
 language: python
 python:
-    - "2.7"
+  - "2.7"
 
 before_script:
+  - sudo apt-get install python-dev
 #  - psql -c 'create database mimic_test;' -U postgres
 #  - psql -c 'create schema mimiciii;' -d mimic_test -U postgres
 #  - psql target-db -U <admin user> -p <port> -h <DB instance name> -c "\copy source-table from 'source-table.csv' with DELIMITER ','" 
 #  - psql -f './buildmimic/postgres/postgres_create_tables.sql' -U postgres
 
 # command to install dependencies
 install:
-   - pip install psycopg2 --quiet
-   - pip install pandas --quiet
-   - pip install MySQL-python --quiet
+  - pip install psycopg2 --quiet
+  - pip install pandas --quiet
+  - pip install MySQL-python --quiet
 #  - pip install .
 #  - pip install -r requirements.txt
 

Makefile

@@ -0,0 +1,75 @@
+## ------------------------------------------------------------------
+## Title: Top-level build file
+## Description: Automated import of data and SQL scripts
+## ------------------------------------------------------------------
+
+## Parameters ##
+
+DBNAME=mimic
+DBUSER=mimic
+SCHEMA=mimiciii
+DATADIR=
+
+# path of this makefile - used to locate concepts subfolder
+ROOT_DIR:=$(shell "dirname" $(realpath $(lastword $(MAKEFILE_LIST))))
+
+## Commands ##
+
+PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
+
+
+## Export ##
+# Parameters given in this Makefile take precedence over those defined in each
+# individual Makefile (due to specifying the -e option and the export command
+# here)
+export
+
+## Build targets ##
+help:
+	@echo '---------------------------------------------------------------------------'
+	@echo 'mimic-download: Download data from PhysioNet'
+	@echo 'mimic-gz: Import data into a local PostgreSQL database using .csv.gz files'
+	@echo 'mimic: Import data into a local PostgreSQL database using .csv files'
+	@echo 'concepts: Create community contributed materialized views'
+	@echo '--------------------------------------------------------------------------- '
+	@echo '   Download MIMIC-III from PhysioNet to the /path/to/data/ directory -      '
+	@echo '                                                                            '
+	@echo '       make mimic-download physionetuser=USERNAME datadir="/path/to/data/"  '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic-download physionetuser=me@email.com datadir="/data/" '
+	@echo '                                                                            '
+	@echo '   Build MIMIC-III using the CSV files in the /path/to/data directory -     '
+	@echo '                                                                            '
+	@echo '       make mimic datadir="/path/to/data/"                                  '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic datadir="/data/mimic/v1_4/"                          '
+	@echo '                                                                            '
+	@echo '   Build MIMIC-III using the .csv.gz files in the /path/to/data directory - '
+	@echo '                                                                            '
+	@echo '       make mimic-gz datadir="/path/to/data/"                               '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic-gz datadir="/data/mimic/v1_4/"                       '
+	@echo '--------------------------------------------------------------------------- '
+
+mimic: mimic-build mimic-check
+mimic-gz: mimic-build-gz mimic-check-gz
+
+mimic-download:
+	@$(MAKE) -e -C buildmimic/postgres mimic-download
+
+mimic-build:
+	@$(MAKE) -e -C buildmimic/postgres mimic-build
+
+mimic-check:
+	@$(MAKE) -e -C buildmimic/postgres mimic-check
+
+mimic-build-gz:
+	@$(MAKE) -e -C buildmimic/postgres mimic-build-gz
+
+mimic-check-gz:
+	@$(MAKE) -e -C buildmimic/postgres mimic-check-gz
+
+concepts:
+	@cd $(ROOT_DIR)/concepts && $(PSQL) -f make-concepts.sql
+
+.PHONY: help mimic mimic-build mimic-download mimic-check mimic-gz mimic-build-gz mimic-check-gz concepts

Makefile.md

@@ -0,0 +1,40 @@
+
+## Introduction to the build system
+
+The build system for mimic code uses the GNU Makefile system. From a user's point of view this makes the whole process very straightforward.
+Starting from a fresh system which has both GNU Make installed, PostgreSQL installed, and a local copy of this repository, an instance of the MIMIC database can be imported from PhysioNet by running the following:
+
+```
+make mimic-download
+make mimic-gz DATADIR=/path/to/data
+```
+
+Note that if you have already downloaded the data, you can skip the `make mimic-download`. If you have already decompressed the data into `.csv` files, then you can run `make mimic DATADIR=/path/to/data`.
+
+Optionally, additional contributed materialized views can be created afterward by running:
+
+```
+make concepts
+```
+
+Note that you may want to modify parameters at the top of the Makefile - e.g. the username (see below "non-standard username or database name").
+
+### Authentication
+In order to avoid the prompts for your database password each time, you may create a file in your home directory called .pgpass containing the following:
+
+```
+localhost:5432:*:mimic:password
+```
+
+Replace ```mimic``` with your username and ```password``` with your password. Note that this is storage of your database password in the clear and so we would recommend only doing this for installation.
+
+### Non-standard username or database name
+If you need to use a username or database name other than ```mimic```, then you will need to specify this by modifying the top-level Makefile:
+
+```
+DBNAME=mimic
+DBUSER=mimic
+```
+
+## Contributing
+If you would like to contribute code to create a materialized view to the `concepts` folder, the existence of this makefile places an additional (and hopefully very minor) burden to ensure that your views are included in this build system. The top-level Makefile (i.e. the one in the root of this repository) is a wrapper that calls `concepts/make-concepts.sql`: simply add a command which calls the script to this file. The format is fairly straightforward: e.g. adding the `\i sepsis/angus.sql` line informs the script to call the `concepts/sepsis/angus.sql` file.

README.md

@@ -2,6 +2,10 @@
 
 This is a repository of code shared by the research community. The repository is intended to be a central hub for sharing, refining, and reusing code used for analysis of the [MIMIC critical care database](https://mimic.physionet.org). To find out more about MIMIC, please see: https://mimic.physionet.org
 
+## Acknowledgement
+
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.821872.svg)](https://doi.org/10.5281/zenodo.821872)
+
 ## How to contribute
 
 Our team has worked hard to create and share the MIMIC dataset. We encourage you to share the code that you use for data processing and analysis. Sharing code helps to make studies reproducible and promotes collaborative research. To contribute, please:
@@ -10,10 +14,16 @@ Our team has worked hard to create and share the MIMIC dataset. We encourage you
 - Commit your changes to the forked repository.
 - Submit a pull request to the [MIMIC code repository](https://github.com/MIT-LCP/mimic-code), using the method described at: https://help.github.com/articles/using-pull-requests/
 
-## Coding style
+We encourage users to share concepts they have extracted by writing code which generates a materialized view. These materialized views can then be used by researchers around the world to speed up data extraction. For example, ventilation durations can be acquired by creating the ventdurations view in [etc/ventilation-durations.sql](https://github.com/MIT-LCP/mimic-code/blob/master/concepts/ventilation-durations.sql).
 
-Please refer to the [style guide](https://github.com/MIT-LCP/mimic-code/blob/master/styleguide.md) for guidelines on formatting your code for the repository. 
+## License
 
+By committing your code to the [MIMIC Code Repository](https://github.com/mit-lcp/mimic-code) you agree to release the code under the [MIT License attached to the repository](https://github.com/mit-lcp/mimic-code/blob/master/LICENSE).
+
+## Coding style
 
+Please refer to the [style guide](https://github.com/MIT-LCP/mimic-code/blob/master/styleguide.md) for guidelines on formatting your code for the repository.
 
+## Building MIMIC automatically
 
+A Makefile build system has been created to facilitate the building of the MIMIC database, and optionally contributed views from the community. Please refer to the [Makefile guide](https://github.com/MIT-LCP/mimic-code/blob/master/Makefile.md) for more details.

buildmimic/apache-drill/drill_create_data.sql

@@ -1,9 +1,7 @@
--- ------------------------------------------------------------------  
+-- ------------------------------------------------------------------
 -- Title: Create the MIMIC-III tables
--- Description: More detailed description explaining the purpose.  
--- MIMIC version: MIMIC-III v1.3  
--- Created by: paris nicolas 
--- ------------------------------------------------------------------  
+-- Description: More detailed description explaining the purpose.
+-- ------------------------------------------------------------------
 
 
 
@@ -35,7 +33,6 @@ case when EDREGTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(EDREGTIME as
 case when EDOUTTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(EDOUTTIME as TIMESTAMP(0)) end as EDOUTTIME,
 DIAGNOSIS,
 case when HOSPITAL_EXPIRE_FLAG = '' then cast(NULL as INT) else cast(HOSPITAL_EXPIRE_FLAG as INT) end as HOSPITAL_EXPIRE_FLAG,
-case when HAS_IOEVENTS_DATA = '' then cast(NULL as INT ) else cast(HAS_IOEVENTS_DATA as INT) end as HAS_IOEVENTS_DATA,
 case when HAS_CHARTEVENTS_DATA = '' then cast(NULL as INT ) else cast(HAS_CHARTEVENTS_DATA as INT) end as HAS_CHARTEVENTS_DATA
 FROM dfs.tmp.`ADMISSIONS.csv` ;
 
@@ -76,7 +73,7 @@ FROM dfs.tmp.`CALLOUT.csv`;
 --------------------------------------------------------
 
 CREATE TABLE dfs.mimiciii.`CAREGIVERS` AS
-SELECT 
+SELECT
 case when ROW_ID = '' then cast(NULL as INT ) else cast(ROW_ID as INT) end as ROW_ID,
 case when CGID = '' then cast(NULL as INT ) else cast(CGID as INT) end as CGID,
 LABEL,
@@ -270,7 +267,7 @@ case when LAST_WARDID = '' then cast(NULL as INT ) else cast(LAST_WARDID as INT)
 case when INTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(INTIME as TIMESTAMP(0)) end as INTIME,
 case when OUTTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(OUTTIME as TIMESTAMP(0)) end as OUTTIME,
 case when LOS = '' then cast(NULL as DOUBLE ) else cast(LOS as DOUBLE) end as LOS
-FROM dfs.tmp.`ICUSTAYS.csv`;										
+FROM dfs.tmp.`ICUSTAYS.csv`;
 
 --------------------------------------------------------
 --  DDL for Table INPUTEVENTS_CV
@@ -461,7 +458,7 @@ DOSE_VAL_RX,
 DOSE_UNIT_RX,
 FORM_VAL_DISP,
 FORM_UNIT_DISP,
-ROUTE                                               
+ROUTE
 FROM dfs.tmp.`PRESCRIPTIONS.csv`;
 
 
@@ -545,4 +542,3 @@ case when INTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(INTIME as TIMEST
 case when OUTTIME = '' then cast(NULL as TIMESTAMP(0)) else cast(OUTTIME as TIMESTAMP(0)) end as OUTTIME,
 case when LOS = '' then cast(NULL as DOUBLE ) else cast(LOS as DOUBLE) end as LOS
 FROM dfs.tmp.`TRANSFERS.csv`;
-

buildmimic/docker/Dockerfile

@@ -1,16 +1,24 @@
 FROM postgres:latest
 
-MAINTAINER Aaron J. Masino
-
+# in the docker initialization, we do not build the data
 ENV BUILD_MIMIC 0
 
 RUN apt-get update
+RUN apt-get install -y vim git
+
+# clone the postgres build scripts into a local folder
+RUN mkdir mimic-code
+RUN cd mimic-code
+RUN git init
+RUN git remote add -f origin https://github.com/MIT-lcp/mimic-code
+RUN git config core.sparseCheckout true
+RUN echo "buildmimic/postgres/" >> .git/info/sparse-checkout
+RUN git pull origin master
 
-RUN apt-get install -y vim
+# copy the build scripts into a different folder
+RUN cp -r buildmimic /docker-entrypoint-initdb.d/
 
+# make a directory for the data
 RUN mkdir /mimic_data
 
 ADD setup.sh /docker-entrypoint-initdb.d/
-
-ADD mimic_build_files /docker-entrypoint-initdb.d/mimic_build_files
-