reworked make to call a single sql file to create concepts

Makefile

@@ -10,6 +10,8 @@ DBUSER=mimic
 SCHEMA=mimiciii
 DATADIR=
 
+# path of this makefile - used to locate concepts subfolder
+ROOT_DIR:=$(shell "dirname" $(realpath $(lastword $(MAKEFILE_LIST))))
 
 ## Commands ##
 
@@ -23,19 +25,12 @@ PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER
 export
 
 ## Build targets ##
-
 help:
 	@echo '---------------------------------------------------------------------------'
-	@echo 'mimic: Import data'
-	@echo 'extra: Create community contributed materialzed views'
-	@echo '                                                                           '
-	@echo 'extra includes:'
-	@echo '  etc: Miscellaneous staging scripts for useful clinical concepts'
-	@echo '    firstday: Miscellaneous scripts for concepts on day 1 of an admission'
-	@echo '  comorbidity: Comorbidity scores'
-	@echo '  sepsis: Sepsis scores'
-	@echo '  severityscores: Severity scores'
-	@echo '                                                                            '
 	@echo '--------------------------------------------------------------------------- '
 	@echo '   Download MIMIC-III from PhysioNet to the /path/to/data/ directory -      '
 	@echo '                                                                            '
@@ -74,34 +69,7 @@ mimic-build-gz:
 mimic-check-gz:
 	@$(MAKE) -e -C buildmimic/postgres mimic-check-gz
 
-extra: comorbidity demographics sepsis severityscores
-
-
-## Individual build targets ##
-
-etc:
-	@$(MAKE) -e -C etc extra
-
-comorbidity: etc
-	@$(MAKE) -e -C comorbidity/postgres extra
-
-demographics: etc
-	@$(MAKE) -e -C demographics/postgres extra
-
-sepsis: etc
-	@$(MAKE) -e -C sepsis extra
-
-severityscores: etc
-	@$(MAKE) -e -C severityscores extra
-
-## Clean ##
-
-clean:
-	@$(MAKE) -e -C buildmimic/postgres clean
-	@$(MAKE) -e -C etc clean
-	@$(MAKE) -e -C comorbidity/postgres clean
-	@$(MAKE) -e -C demographics/postgres clean
-	@$(MAKE) -e -C sepsis clean
-	@$(MAKE) -e -C severityscores clean
 
-.PHONY: help mimic mimic-build mimic-download mimic-check mimic-gz mimic-build-gz mimic-check-gz extra etc comorbidity demographics sepsis severityscores

Makefile.md

@@ -1,19 +1,24 @@
 
 ## Introduction to the build system
 
-The build system for mimic code uses the GNU Makefile system. From a user's point of view this makes the whole process very straightforward: in order to import data they may run "make mimic DATADIR=/path" and in order to build all of the community contributed views and tables they may simply run "make extra".
+Starting from a fresh system which has both GNU Make installed, PostgreSQL installed, and a local copy of this repository, an instance of the MIMIC database can be imported from PhysioNet by running the following:
 
 ```
-# Import data from .csv files
-make mimic DATADIR=/path/to/data
-
-# Alternative to import data using .csv.gz files
 make mimic-gz DATADIR=/path/to/data
+```
+
+Note that if you have already downloaded the data, you can skip the `make mimic-download`. If you have already decompressed the data into `.csv` files, then you can run `make mimic DATADIR=/path/to/data`.
 
-# Import user contributed scripts
-make extra
+```
+make concepts
 ```
 
+Note that you may want to modify parameters at the top of the Makefile - e.g. the username (see below "non-standard username or database name").
+
 ### Authentication
 In order to avoid the prompts for your database password each time, you may create a file in your home directory called .pgpass containing the following:
 
@@ -32,10 +37,4 @@ DBUSER=mimic
 ```
 
 ## Contributing
-From a contributor's point of view this places an additional (and hopefully very minor) burden to ensure that their views are included in this build system. The top-level Makefile (i.e. the one in the root of this repository) is mostly a wrapper that calls each of the various subdirectorys' own Makefiles; running "make extra" will change directory to etc, run "make etc", which changes directory to "etc/firstday" and runs "make firstday" and so on. This also tells you which Makefile you need to modify: it's the one in the same directory as your script.
-
-You do not have to specifically tell the Makefile how to build your SQL script: this is handled by an "implicit rule" that covers all SQL files. You do have to tell Makefile that there's an SQL script there to be built though. This is achieved by adding the name of the file (without the .sql extension) to the "extra" build target.
-
-### clean.sql
-Do not call "DROP MATERIALIZED VIEW" or "DROP TABLE" from within your contributed SQL script. This is a specific requirement to ensure that make can build its dependencies quickly. Instead add a "DROP ..." command to the shared clean.sql in that subdirectory. This is there for when "make clean" is run.
-

concepts/Makefile

@@ -1,42 +0,0 @@
-## ------------------------------------------------------------------
-## Title: Build file for etc and etc/firstday
-## Description: Automated import of SQL scripts for etc and etc/firstday
-## ------------------------------------------------------------------
-
-## Parameters ##
-# The top-level Makefile settings take precedence over this
-DBNAME=mimic
-DBUSER=mimic
-SCHEMA=mimiciii
-
-
-## Commands ##
-# The top-level Makefile settings take precedence over this
-PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
-
-
-## Build targets ##
-
-help:
-	@echo 'extra: Miscellaneous staging scripts for useful clinical concepts'
-	@echo '  firstday: Miscellaneous scripts for concepts on day 1 of an admission'
-
-extra: echo-data firstday rrt ventilation-durations
-
-firstday:
-	@$(MAKE) -e -C firstday extra
-
-clean:
-	@$(PSQL) -f clean.sql
-	@$(MAKE) -e -C firstday clean
-
-.PHONY: extra firstday help clean
-
-
-## Implicit rules ##
-
-%: %.sql
-	@echo
-	@echo '--- Building' $< '---'
-	@echo
-	@$(PSQL) -f $<

concepts/clean.sql

@@ -1,10 +0,0 @@
--- ------------------------------------------------------------------
--- Title: SQL clean script called by "make clean"
--- Description: Drops all materialized views re: misc clinical concepts
--- ------------------------------------------------------------------
-
-DROP MATERIALIZED VIEW IF EXISTS ECHODATA CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS rrt CASCADE;
--- Tables for ventdurations
-DROP TABLE IF EXISTS ventsettings CASCADE;
-DROP TABLE IF EXISTS ventdurations CASCADE;

concepts/comorbidity/clean.sql

@@ -1,7 +0,0 @@
--- ------------------------------------------------------------------
--- Title: SQL clean script called by "make clean"
--- Description: Drops all materialized views re: comorbidity scoring
--- ------------------------------------------------------------------
-
-DROP MATERIALIZED VIEW IF EXISTS ELIXHAUSER_AHRQ CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS ELIXHAUSER_QUAN CASCADE;

concepts/comorbidity/postgres/Makefile

@@ -1,38 +0,0 @@
-## ------------------------------------------------------------------
-## Title: Build file for comorbidity
-## Description: Automated import of SQL scripts for comorbidity
-## ------------------------------------------------------------------
-
-## Parameters ##
-# The top-level Makefile settings take precedence over this
-DBNAME=mimic
-DBUSER=mimic
-SCHEMA=mimiciii
-
-
-## Commands ##
-# The top-level Makefile settings take precedence over this
-PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
-
-
-## Build targets ##
-
-help:
-	@echo 'extra: Comorbidity scores'
-	@echo 'clean: Drop all comorbidity score materialized views'
-
-extra: elixhauser-ahrq-v37-with-drg elixhauser-quan
-
-clean:
-	@$(PSQL) -f clean.sql
-
-.PHONY: extra clean help
-
-
-## Implicit rules ##
-
-%: %.sql
-	@echo
-	@echo '--- Building' $< '---'
-	@echo
-	@$(PSQL) -f $<

concepts/demographics/Makefile

@@ -1,37 +0,0 @@
-## ------------------------------------------------------------------
-## Title: Build file for demographics
-## Description: Automated import of SQL scripts for demographics
-## ------------------------------------------------------------------
-
-## Parameters ##
-# The top-level Makefile settings take precedence over this
-DBNAME=mimic
-DBUSER=mimic
-SCHEMA=mimiciii
-
-
-## Commands ##
-# The top-level Makefile settings take precedence over this
-PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
-
-
-## Build targets ##
-
-help:
-	@echo 'demographics: tables and views related to demographics'
-
-extra: HeightWeightQuery icustay_detail
-
-clean:
-	@$(PSQL) -f clean.sql
-
-.PHONY: help clean extra
-
-
-## Implicit rules ##
-
-%: %.sql
-	@echo
-	@echo '--- Building' $< '---'
-	@echo
-	@$(PSQL) -f $<

concepts/demographics/clean.sql

@@ -1,7 +0,0 @@
--- ------------------------------------------------------------------
--- Title: SQL clean script called by "make clean"
--- Description: Drops all materialized views re: demographics
--- ------------------------------------------------------------------
-
-DROP MATERIALIZED VIEW IF EXISTS icustay_detail CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS heightweight CASCADE;

concepts/firstday/Makefile

@@ -1,35 +0,0 @@
-## ------------------------------------------------------------------
-## Title: Build file for etc/firstday
-## Description: Automated import of SQL scripts for firstday
-## ------------------------------------------------------------------
-
-## Parameters ##
-# The top-level Makefile settings take precedence over this
-DBNAME=mimic
-DBUSER=mimic
-SCHEMA=mimiciii
-
-
-## Commands ##
-# The top-level Makefile settings take precedence over this
-PSQL=psql "dbname=$(DBNAME) options=--search_path=$(SCHEMA)" --username=$(DBUSER)
-
-
-## Build targets ##
-
-#blood-gas-first-day-arterial: blood-gas-first-day
-extra: blood-gas-first-day blood-gas-first-day-arterial gcs-first-day height-first-day labs-first-day rrt-first-day urine-output-first-day ventilation-first-day vitals-first-day weight-first-day
-
-clean:
-	@$(PSQL) -f clean.sql
-
-.PHONY: extra clean
-
-
-## Implicit rules ##
-
-%: %.sql
-	@echo
-	@echo '--- Building' $< '---'
-	@echo
-	@$(PSQL) -f $<

concepts/firstday/clean.sql

@@ -1,17 +0,0 @@
--- ------------------------------------------------------------------
--- Title: SQL clean script called by "make clean"
--- Description: Drops all materialized views re: first day concepts
--- ------------------------------------------------------------------
-
-DROP MATERIALIZED VIEW IF EXISTS bloodgasfirstdayarterial CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS bloodgasfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS gcsfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS heightfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS labsfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS rrtfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS uofirstday CASCADE;
--- Need to drop table as well for legacy purposes
-DROP TABLE IF EXISTS ventfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS ventfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS vitalsfirstday CASCADE;
-DROP MATERIALIZED VIEW IF EXISTS weightfirstday CASCADE;

concepts/make-concepts.sql

@@ -0,0 +1,60 @@
+-- This file makes all materialized views in this subfolder
+-- Note that this may take a large amount of time and hard drive space
+
+\echo 'Beginning to create materialized views for MIMIC database.'
+BEGIN;
+\echo 'Top level files..'
+\i code-status.sql
+\i echo-data.sql
+\i ventilation-durations.sql
+
+\echo 'Directory 1 of 6: comorbidity'
+\i elixhauser-ahrq-v37-with-drg.sql
+\i elixhauser-quan.sql
+\i elixhauser-score-ahrq.sql
+\i elixhauser-score-quan.sql
+
+\echo 'Directory 2 of 6: demographics'
+\i HeightWeightQuery.sql
+\i icustay_detail.sql
+
+\echo 'Directory 3 of 6: firstday'
+-- data which is extracted from a patient's first ICU stay
+\i firstday/blood-gas-first-day.sql
+\i firstday/blood-gas-first-day-arterial.sql
+\i firstday/gcs-first-day.sql
+\i firstday/height-first-day.sql
+\i firstday/labs-first-day.sql
+\i firstday/rrt-first-day.sql
+\i firstday/urine-output-first-day.sql
+\i firstday/ventilation-first-day.sql
+\i firstday/vitals-first-day.sql
+\i firstday/weight-first-day.sql
+
+\echo 'Directory 4 of 6: sepsis'
+\i sepsis/angus.sql
+
+-- vasopressor durations
+\echo 'Directory 5 of 6: vasopressor-durations'
+\i vasopressor-durations/adenosine-durations.sql
+\i vasopressor-durations/dobutamine-durations.sql
+\i vasopressor-durations/dopamine-durations.sql
+\i vasopressor-durations/epinephrine-durations.sql
+\i vasopressor-durations/isuprel-durations.sql
+\i vasopressor-durations/milrinone-durations.sql
+\i vasopressor-durations/norepinephrine-durations.sql
+\i vasopressor-durations/phenylephrine-durations.sql
+\i vasopressor-durations/vasopressin-durations.sql
+\i vasopressor-durations/vasopressor-durations.sql
+
+-- Severity of illness scores (requires many views from above)
+\echo 'Directory 6 of 6: severityscores'
+\i severityscores/oasis.sql
+\i severityscores/sofa.sql
+\i severityscores/saps.sql
+\i severityscores/sapsii.sql
+\i severityscores/apsiii.sql
+\i severityscores/lods.sql
+
+COMMIT;
+\echo 'Finished loading materialized views.'