Mccalluc/yet more tests (#95)

* Assertions for contents of /data

* Sort before diff, and two column format so it is easier to read.

* Checkpoint: trying to PUT data. Failing.

* Checkpoint: create superuser. Not working

* checkpoint

* Checkpoint. Uploading to server, but not successfully.

* Checkpoint. Try aother port. Only seeing "GET" in the logs.

* logs util

* Start up django http

* Extra quote marks must have confused things?

build.sh

@@ -4,6 +4,7 @@ set -e
 
 error_report() {
   docker logs container-$STAMP
+  docker exec -it container-$STAMP /home/higlass/projects/logs.sh
 }
 
 trap 'error_report' ERR
@@ -74,7 +75,6 @@ docker build --cache-from $REPO:latest \
              --tag image-$STAMP \
              web-context
 
-mkdir -p $VOLUME
 docker run --name container-$STAMP \
            --network network-$STAMP \
            --publish $PORT:80 \
@@ -94,9 +94,13 @@ docker ps -a
 if [ $PORT == 0 ]; then
   # If we are running with -l ("latest"), we also want to be sure
   # that the built image can run on its own, without any extra configuration.
+  # (Test is expecting 'redis-data', even if we do not use it.)
+  for DIR in redis-data hg-data/log hg-tmp; do
+    mkdir -p $VOLUME-single/$DIR || echo "$VOLUME/$DIR already exists"
+  done
   docker run --name container-$STAMP-single \
-             --volume $VOLUME/hg-data:/data \
-             --volume $VOLUME/hg-tmp:/tmp \
+             --volume $VOLUME-single/hg-data:/data \
+             --volume $VOLUME-single/hg-tmp:/tmp \
              --detach \
              --publish-all \
              image-$STAMP

expected-data-dir.txt

@@ -0,0 +1,13 @@
+.
+./hg-data
+./hg-data/db.sqlite3
+./hg-data/log
+./hg-data/log/access.log
+./hg-data/log/error.log
+./hg-data/log/hgs.log
+./hg-tmp
+./hg-tmp/nginx-stderr---supervisor-XXXXXX.log
+./hg-tmp/nginx-stdout---supervisor-XXXXXX.log
+./hg-tmp/uwsgi-stderr---supervisor-XXXXXX.log
+./hg-tmp/uwsgi-stdout---supervisor-XXXXXX.log
+./redis-data

test.sh

@@ -51,6 +51,31 @@ echo $HTML | grep -o 'Department of Biomedical Informatics'
 [ -z `echo $NGINX_LOG | grep -v '/api/v1/tilesets/'` ] || false
 #echo $PING_REDIS_INSIDE | grep -o 'PONG'
 echo $VERSION_TXT | grep -o 'WEBSITE_VERSION'
+diff -y expected-data-dir.txt <(
+        pushd /tmp/higlass-docker/volume-$STAMP > /dev/null \
+        && find . | sort | perl -pne 's/-\w+\.log/-XXXXXX.log/' \
+        && popd > /dev/null )
+
+USERNAME=username
+PASSWORD=password
+
+# We need an authorized user in order to upload.
+# TODO: make this less ugly!
+
+docker exec -it container-$STAMP sh -c \
+  "cd /home/higlass/projects/higlass-server/;
+   echo \"import django.contrib.auth; django.contrib.auth.models.User.objects.create_user('$USERNAME', password='$PASSWORD')\" \
+     | python manage.py shell"
+
+S3=https://s3.amazonaws.com/pkerp/public
+COOLER=dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool
+HITILE=wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile
+
+docker exec -it container-$STAMP sh -c \
+  "/home/higlass/projects/upload.sh -c $USERNAME:$PASSWORD -u $S3/$COOLER"
+docker exec -it container-$STAMP sh -c \
+  "/home/higlass/projects/upload.sh -c $USERNAME:$PASSWORD -u $S3/$HITILE"
+
 
 if [[ "$STAMP" != *-single ]]; then
     # Only run these tests if we've started up a separate redis container.

web-context/Dockerfile.template

@@ -46,9 +46,6 @@ USER higlass
 RUN git clone --depth 1 https://github.com/hms-dbmi/higlass-server.git --branch v<SERVER_VERSION>
 USER root
 
-ENV HIGLASS_SERVER_BASE_DIR /data
-VOLUME /data
-
 RUN pip install -r higlass-server/requirements.txt
 USER higlass
 
@@ -75,13 +72,20 @@ RUN rm /etc/nginx/sites-*/default && grep 'listen' /etc/nginx/sites-*/*
 # Without this, two config files are trying to grab port 80:
 # nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
 
+
+# Helper scripts:
+COPY upload.sh .
+COPY logs.sh .
+
 EXPOSE 80
 
+ENV HIGLASS_SERVER_BASE_DIR /data
+VOLUME /data
+
 ARG WORKERS
 ENV WORKERS ${WORKERS}
 RUN echo "WORKERS: $WORKERS"
 
-
 # TODO: Needs to write to logs, but running as root is risky
 # Given as list so that an extra shell does not need to be started.
 CMD ["supervisord", "-n"]

web-context/logs.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+for LOG in /tmp/*-std*.log /data/log/*.log; do
+  echo; echo
+  echo "############"
+  echo $LOG
+  echo "############"
+  cat $LOG
+done

web-context/upload.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -e
+
+while getopts 'c:u:' OPT; do
+  case $OPT in
+    c)
+      CREDENTIALS=$OPTARG
+      ;;
+    u)
+      URL=$OPTARG
+      ;;
+  esac
+done
+
+if [ -z $CREDENTIALS ] || [ -z $URL ]; then
+  echo "USAGE: $0 -c CREDENTIALS -u URL" >&2
+  exit 1
+fi
+
+set -o verbose
+
+PORT=8888
+python /home/higlass/projects/higlass-server/manage.py runserver localhost:$PORT &
+
+# TODO: explicitly wait for server to start
+
+DOWNLOADS=/tmp/downloads
+mkdir -p $DOWNLOADS
+NAME=`basename $URL`
+wget -O $DOWNLOADS/$NAME $URL
+
+# debug...
+whoami
+ls -l $DOWNLOADS
+
+if [[ "$NAME" == *.cool ]]; then
+    CMD="curl -F datafile=@$DOWNLOADS/$NAME -u $CREDENTIALS
+              -F filetype=cooler -F datatype=matrix -F uid=cooler
+              -F coordSystem=hg19
+              http://localhost:$PORT/api/v1/tilesets/"
+elif [[ "$NAME" == *.hitile ]]; then
+    CMD="curl -F datafile=@$DOWNLOADS/$NAME -u $CREDENTIALS
+              -F filetype=hitile -F datatype=vector -F uid=hitile
+              -F coordSystem=hg19
+              http://localhost:$PORT/api/v1/tilesets/"
+else
+    echo "Unrecognized file type: $NAME" >&2
+    exit 1
+fi
+
+echo $CMD
+$CMD
+
+# TODO: stop server?

web-context/uwsgi.ini

@@ -11,6 +11,8 @@ processes = 4
 [local]
 ini = :base
 http = :8000
+# TODO: hgserver_nginx.conf says 8001: Is this one ignored?
+
 # set the virtual env to use
 # home=/Users/you/envs/env