Merge branch '0.3.0-prerelease'

Dockerfile

@@ -9,7 +9,8 @@ RUN apt-get update && apt-get install -y \
     && CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \
     && echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee /etc/apt/sources.list.d/google-cloud-sdk.list \
     && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
-    && apt-get update && apt-get install -y google-cloud-sdk \
+    && apt-get update && apt-get install -y \
+    google-cloud-sdk \
     && apt-get autoremove \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
@@ -21,7 +22,10 @@ RUN apt-get update && apt-get install -y \
     && apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \
     && echo 'deb https://apt.dockerproject.org/repo debian-jessie main' > /etc/apt/sources.list.d/docker.list
 RUN apt-get update && apt-get install -y \
-    docker-engine
+    docker-engine \
+    && apt-get autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Loom's OS dependencies.
 RUN apt-get update && apt-get install -y \
@@ -30,21 +34,18 @@ RUN apt-get update && apt-get install -y \
     libmysqlclient-dev \
     libssl-dev \
     python-dev \
-    python-pip \ 
-    && pip install virtualenv \
+    python-pip \
     && apt-get autoremove \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Loom and its Python dependencies.
-COPY . /opt/loom/
-RUN virtualenv /opt/loom-env \
-    && . /opt/loom-env/bin/activate \
-    && cd /opt/loom \
-    && pip install -r requirements.txt
-
-# Add Loom to the path.
-ENV PATH /opt/loom-env/bin:$PATH
-
-# Accept connections on port 8000.
-EXPOSE 8000
+WORKDIR /loomengine/
+ADD ./requirements.txt /loomengine/
+ADD ./setup.py /loomengine/
+ADD ./README.rst /loomengine/
+ADD ./loomengine/utils/ /loomengine/loomengine/utils/
+ADD ./loomengine/__init__.py /loomengine/loomengine/
+ADD ./loomengine/VERSION /loomengine/loomengine/
+RUN pip install -r requirements.txt
+ADD . /loomengine/

bin/run_master.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+DEFAULT_LOG_LEVEL=info
+DEFAULT_IP=0.0.0.0
+DEFAULT_PORT=8000
+DEFAULT_GUNICORN_WORKERS_COUNT=5
+
+LOOM_LOG_LEVEL=${LOOM_LOG_LEVEL:-$DEFAULT_LOG_LEVEL}
+LOOM_MASTER_INTERNAL_IP=${LOOM_MASTER_INTERNAL_IP:-$DEFAULT_IP}
+LOOM_MASTER_INTERNAL_PORT=${LOOM_MASTER_INTERNAL_PORT:-$DEFAULT_PORT}
+LOOM_MASTER_GUNICORN_WORKERS_COUNT=${LOOM_MASTER_GUNICORN_WORKERS_COUNT:-$DEFAULT_GUNICORN_WORKERS_COUNT}
+
+
+BIN_PATH="`dirname \"$0\"`"
+
+# Wait for database to become available
+RETRIES=30
+n=0
+while :
+do
+    # break if db connection is successful
+    $BIN_PATH/../loomengine/master/manage.py inspectdb > /dev/null 2>%1 && break
+
+    # exit if retries exceeded
+    if [ $n -ge $RETRIES ]
+    then
+	>&2 echo "Timeout while waiting for database"
+	exit 1;
+    fi
+
+    sleep 1
+    n=$[$n+1]
+done
+
+$BIN_PATH/../loomengine/master/manage.py migrate
+$BIN_PATH/../loomengine/master/manage.py collectstatic --noinput
+
+gunicorn loomengine.master.master.wsgi --bind ${LOOM_MASTER_INTERNAL_IP}:${LOOM_MASTER_INTERNAL_PORT} --log-level ${LOOM_LOG_LEVEL} --capture-output -w ${LOOM_MASTER_GUNICORN_WORKERS_COUNT}

bin/run_scheduler.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+BIN_PATH="`dirname \"$0\"`"
+
+DEFAULT_LOG_LEVEL=info
+
+LOOM_LOG_LEVEL=${LOOM_LOG_LEVEL:-$DEFAULT_LOG_LEVEL}
+
+# omitting --without-gossip causes missed heartbeat errors
+celery beat -A loomengine.master.master -l ${LOOM_LOG_LEVEL} --workdir=${BIN_PATH}/../loomengine/master
+
+

bin/run_tests.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+loom test unit

bin/run_worker.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+BIN_PATH="`dirname \"$0\"`"
+
+DEFAULT_LOG_LEVEL=info
+DEFAULT_CELERY_CONCURRENCY=30
+
+LOOM_LOG_LEVEL=${LOOM_LOG_LEVEL:-$DEFAULT_LOG_LEVEL}
+LOOM_WORKER_CELERY_CONCURRENCY=${LOOM_WORKER_CELERY_CONCURRENCY:-$DEFAULT_CELERY_CONCURRENCY}
+
+# omitting --without-gossip causes missed heartbeat errors
+celery -c ${LOOM_WORKER_CELERY_CONCURRENCY} -A loomengine.master.master -l ${LOOM_LOG_LEVEL} -P eventlet worker --workdir=${BIN_PATH}/../loomengine/master --without-gossip

doc/about.rst

@@ -26,7 +26,7 @@ Loom remembers anything you ever run and can tell you exactly how each result wa
 Portability between platforms
 -----------------------------
 
-Exactly the same workflow can be run on your laptop or on a public cloud service.
+Exactly the same workflow can be run on your laptop or on a cloud service.
 
 Open architecture
 -----------------
@@ -40,17 +40,10 @@ Not only is Loom open source and free to use, it uses an inside-out architecture
 - Connect Loom to multiple file stores without creating redundant copies
 - Efficient re-use of results for redundant analysis steps
 
-How many times do you really need to run the same analysis on the same inputs? Loom knows which steps in your workflow have already been run and seamlessly integrates previous results with the current run, while still maintaining data provenance and traceability.
-
-Graphical user interface
+Browser interface
 ------------------------
 
-While you may want to automate your analysis from the command line, a graphical user interface is invaluable for browsing workflows and results and keeping an eye on current runs.
-
-Security and compliance
------------------------
-
-Loom is designed with clinical compliance in mind.
+While you may want to automate your analysis from the command line, a browser interface is invaluable for exploring your workflow templates and keeping an eyeon current analysis runs.
 
 Who needs Loom?
 ===============
@@ -59,13 +52,6 @@ Loom is built for the kind of workflows that bioinformaticians run -- multi-step
 
 Loom is scalable and supports individual analysts or large institutions.
 
-Get started
-===========
-
-Check out our Getting Started Guide and give Loom a try.
-
-https://github.com/StanfordBioinformatics/loom/wiki/Getting-Started
-
 What is the current status?
 ===========================
 

doc/conf.py

@@ -55,9 +55,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'0.2.1'
+version = u'0.3.0'
 # The full version, including alpha/beta/rc tags.
-release = u'0.2.1'
+release = u'0.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/examples/hello_world/README.txt

@@ -0,0 +1,30 @@
+To run this workflow:
+
+1. If running for the first time, start the server
+
+      loom server start --settings-file local.conf
+
+2. Import the input files
+
+      loom import files hello.txt world.txt
+
+3. Import the template
+
+      loom import template hello_world.yaml
+
+4. Select inputs and execute the run
+
+      loom run hello_world hello=hello.txt world=world.txt
+
+5. Monitor the run from the commandline
+
+      loom show run hello_world --detail
+
+6. Monitor the run in the browser
+
+      loom browser
+
+7. If you want to delete the Loom server
+
+      loom server delete
+

doc/examples/hello_world/hello_world.yaml

@@ -10,7 +10,8 @@ inputs:
 fixed_inputs:
 - type: string
   channel: final_punc
-  data: "!!"
+  data:
+    contents: "!!"
 outputs:
 - type: string
   channel: hello_world

doc/examples/word_scoring/README.txt

@@ -0,0 +1,33 @@
+NOTE: This example does not work currently! The parallel feature is not yet available.
+This template demonstrates how parallel steps WILL BE implemented using
+"scatter" and "gather" modes on inputs and outputs.
+
+To run this workflow:
+
+1. If running for the first time, start the server
+
+      loom server start --settings-file local.conf
+
+2. Import the input file
+
+      loom import file wordfile.txt
+
+3. Import the template
+
+      loom import template word_scoring.yaml
+
+4. Select inputs and execute the run
+
+      loom run word_scoring wordfile=wordfile.txt
+
+5. Monitor the run from the commandline
+
+      loom show run word_scoring --detail
+
+6. Monitor the run in the browser
+
+      loom browser
+
+7. If you want to delete the Loom server
+
+      loom server delete