Added requirements.txt to tests/

.gitignore

@@ -1,3 +1,9 @@
+# Data files
+*.csv
+
+# Picture files
+*.png
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -59,3 +65,9 @@ target/
 # OSX .DS_Store
 .DS_Store
 
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+
+
+

.travis.yml

@@ -18,18 +18,20 @@ services:
 # Set the language
 language: python
 python:
-    - "2.7"
+  - "2.7"
 
 before_script:
+  - sudo apt-get install python-dev
 #  - psql -c 'create database mimic_test;' -U postgres
 #  - psql -c 'create schema mimiciii;' -d mimic_test -U postgres
 #  - psql target-db -U <admin user> -p <port> -h <DB instance name> -c "\copy source-table from 'source-table.csv' with DELIMITER ','" 
 #  - psql -f './buildmimic/postgres/postgres_create_tables.sql' -U postgres
 
 # command to install dependencies
 install:
-   - pip install psycopg2 --quiet
-   - pip install pandas --quiet
+  - pip install psycopg2 --quiet
+  - pip install pandas --quiet
+  - pip install MySQL-python --quiet
 #  - pip install .
 #  - pip install -r requirements.txt
 

Makefile

@@ -0,0 +1,75 @@
+## ------------------------------------------------------------------
+## Title: Top-level build file
+## Description: Automated import of data and SQL scripts
+## ------------------------------------------------------------------
+
+## Parameters ##
+
+DBNAME=mimic
+DBUSER=mimic
+SCHEMA=mimiciii
+DATADIR=
+
+# path of this makefile - used to locate concepts subfolder
+ROOT_DIR:=$(shell "dirname" $(realpath $(lastword $(MAKEFILE_LIST))))
+
+## Commands ##
+
+PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
+
+
+## Export ##
+# Parameters given in this Makefile take precedence over those defined in each
+# individual Makefile (due to specifying the -e option and the export command
+# here)
+export
+
+## Build targets ##
+help:
+	@echo '---------------------------------------------------------------------------'
+	@echo 'mimic-download: Download data from PhysioNet'
+	@echo 'mimic-gz: Import data into a local PostgreSQL database using .csv.gz files'
+	@echo 'mimic: Import data into a local PostgreSQL database using .csv files'
+	@echo 'concepts: Create community contributed materialized views'
+	@echo '--------------------------------------------------------------------------- '
+	@echo '   Download MIMIC-III from PhysioNet to the /path/to/data/ directory -      '
+	@echo '                                                                            '
+	@echo '       make mimic-download physionetuser=USERNAME datadir="/path/to/data/"  '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic-download physionetuser=me@email.com datadir="/data/" '
+	@echo '                                                                            '
+	@echo '   Build MIMIC-III using the CSV files in the /path/to/data directory -     '
+	@echo '                                                                            '
+	@echo '       make mimic datadir="/path/to/data/"                                  '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic datadir="/data/mimic/v1_4/"                          '
+	@echo '                                                                            '
+	@echo '   Build MIMIC-III using the .csv.gz files in the /path/to/data directory - '
+	@echo '                                                                            '
+	@echo '       make mimic-gz datadir="/path/to/data/"                               '
+	@echo '                                                                            '
+	@echo '       e.g. make mimic-gz datadir="/data/mimic/v1_4/"                       '
+	@echo '--------------------------------------------------------------------------- '
+
+mimic: mimic-build mimic-check
+mimic-gz: mimic-build-gz mimic-check-gz
+
+mimic-download:
+	@$(MAKE) -e -C buildmimic/postgres mimic-download
+
+mimic-build:
+	@$(MAKE) -e -C buildmimic/postgres mimic-build
+
+mimic-check:
+	@$(MAKE) -e -C buildmimic/postgres mimic-check
+
+mimic-build-gz:
+	@$(MAKE) -e -C buildmimic/postgres mimic-build-gz
+
+mimic-check-gz:
+	@$(MAKE) -e -C buildmimic/postgres mimic-check-gz
+
+concepts:
+	@cd $(ROOT_DIR)/concepts && $(PSQL) -f make-concepts.sql
+
+.PHONY: help mimic mimic-build mimic-download mimic-check mimic-gz mimic-build-gz mimic-check-gz concepts

Makefile.md

@@ -0,0 +1,40 @@
+
+## Introduction to the build system
+
+The build system for mimic code uses the GNU Makefile system. From a user's point of view this makes the whole process very straightforward.
+Starting from a fresh system which has both GNU Make installed, PostgreSQL installed, and a local copy of this repository, an instance of the MIMIC database can be imported from PhysioNet by running the following:
+
+```
+make mimic-download
+make mimic-gz DATADIR=/path/to/data
+```
+
+Note that if you have already downloaded the data, you can skip the `make mimic-download`. If you have already decompressed the data into `.csv` files, then you can run `make mimic DATADIR=/path/to/data`.
+
+Optionally, additional contributed materialized views can be created afterward by running:
+
+```
+make concepts
+```
+
+Note that you may want to modify parameters at the top of the Makefile - e.g. the username (see below "non-standard username or database name").
+
+### Authentication
+In order to avoid the prompts for your database password each time, you may create a file in your home directory called .pgpass containing the following:
+
+```
+localhost:5432:*:mimic:password
+```
+
+Replace ```mimic``` with your username and ```password``` with your password. Note that this is storage of your database password in the clear and so we would recommend only doing this for installation.
+
+### Non-standard username or database name
+If you need to use a username or database name other than ```mimic```, then you will need to specify this by modifying the top-level Makefile:
+
+```
+DBNAME=mimic
+DBUSER=mimic
+```
+
+## Contributing
+If you would like to contribute code to create a materialized view to the `concepts` folder, the existence of this makefile places an additional (and hopefully very minor) burden to ensure that your views are included in this build system. The top-level Makefile (i.e. the one in the root of this repository) is a wrapper that calls `concepts/make-concepts.sql`: simply add a command which calls the script to this file. The format is fairly straightforward: e.g. adding the `\i sepsis/angus.sql` line informs the script to call the `concepts/sepsis/angus.sql` file.

README.md

@@ -2,6 +2,10 @@
 
 This is a repository of code shared by the research community. The repository is intended to be a central hub for sharing, refining, and reusing code used for analysis of the [MIMIC critical care database](https://mimic.physionet.org). To find out more about MIMIC, please see: https://mimic.physionet.org
 
+## Acknowledgement
+
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.821872.svg)](https://doi.org/10.5281/zenodo.821872)
+
 ## How to contribute
 
 Our team has worked hard to create and share the MIMIC dataset. We encourage you to share the code that you use for data processing and analysis. Sharing code helps to make studies reproducible and promotes collaborative research. To contribute, please:
@@ -10,19 +14,16 @@ Our team has worked hard to create and share the MIMIC dataset. We encourage you
 - Commit your changes to the forked repository.
 - Submit a pull request to the [MIMIC code repository](https://github.com/MIT-LCP/mimic-code), using the method described at: https://help.github.com/articles/using-pull-requests/
 
-## Required header information
+We encourage users to share concepts they have extracted by writing code which generates a materialized view. These materialized views can then be used by researchers around the world to speed up data extraction. For example, ventilation durations can be acquired by creating the ventdurations view in [etc/ventilation-durations.sql](https://github.com/MIT-LCP/mimic-code/blob/master/concepts/ventilation-durations.sql).
+
+## License
 
-Please copy and paste the following header information into the top of your code:
+By committing your code to the [MIMIC Code Repository](https://github.com/mit-lcp/mimic-code) you agree to release the code under the [MIT License attached to the repository](https://github.com/mit-lcp/mimic-code/blob/master/LICENSE).
 
-```
--- ------------------------------------------------------------------  
--- Title: Short descriptive title.  
--- Description: More detailed description explaining the purpose.  
--- MIMIC version: Version of MIMIC (e.g. MIMIC-III v1.1).  
--- Created by: List of people who created the code (optional).  
--- References: References to relevant academic papers etc (optional).  
--- ------------------------------------------------------------------  
-```
+## Coding style
 
+Please refer to the [style guide](https://github.com/MIT-LCP/mimic-code/blob/master/styleguide.md) for guidelines on formatting your code for the repository.
 
+## Building MIMIC automatically
 
+A Makefile build system has been created to facilitate the building of the MIMIC database, and optionally contributed views from the community. Please refer to the [Makefile guide](https://github.com/MIT-LCP/mimic-code/blob/master/Makefile.md) for more details.