From 6ca6c2db8472f9481f77f0491b3439debe92270b Mon Sep 17 00:00:00 2001 From: Zebula Sampedro Date: Thu, 13 Dec 2018 17:56:10 -0700 Subject: [PATCH 1/4] Restructured compose layers. --- .dockerignore | 1 + .travis.yml | 12 ++++---- docker-compose.dev.yml | 6 +--- docker-compose.prod.yml | 19 ------------- docker-compose.test-backends.yml | 28 +++++++++---------- docker-compose.yml | 21 +++++++++++++- .../install-rcamp-services.sh | 0 .../rcamp.dev.service | 0 .../rcamp.prod.service | 0 scripts/rebuild-rcamp-dev.sh | 11 ++++++++ 10 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 .dockerignore rename install-rcamp-services.sh => scripts/install-rcamp-services.sh (100%) mode change 100644 => 100755 rename rcamp.dev.service => scripts/rcamp.dev.service (100%) rename rcamp.prod.service => scripts/rcamp.prod.service (100%) create mode 100755 scripts/rebuild-rcamp-dev.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d5910a1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +**/*_functional.py[cod] diff --git a/.travis.yml b/.travis.yml index e8a0ac3..2ee3536 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,13 @@ services: before_install: - pip install --upgrade docker-compose -- cd rcamp -- docker build -t dev/rcamp --build-arg UWSGI_UID=$(id -u) --build-arg UWSGI_GID=$(id -u) . -- cd .. +- export UWSGI_UID=$(id -u) +- export UWSGI_GID=$(id -u) - export RCAMP_PORT=9000 -- docker-compose -f docker-compose.yml -f docker-compose.test-backends.yml -f docker-compose.dev.yml build -- docker-compose -f docker-compose.yml -f docker-compose.test-backends.yml -f docker-compose.dev.yml run --rm --service-ports rcamp-uwsgi bash -c 'sleep 30s && python manage.py migrate' -- docker-compose -f docker-compose.yml -f docker-compose.test-backends.yml -f docker-compose.dev.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' +- export HOSTNAME-localhost +- docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build +- docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'sleep 30s && python manage.py migrate' +- docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' install: true script: true diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 820f818..320c46f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,11 +2,7 @@ version: "3.6" services: rcamp-uwsgi: - build: - context: rcamp - cache_from: - - dev/rcamp - dockerfile: Dockerfile.test + command: ["python", "manage.py", "runserver", "0.0.0.0:9000"] environment: - RCAMP_DEBUG=True volumes: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 578aea0..ac4d443 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,22 +18,3 @@ services: - /var/lib/sss/pipes:/var/lib/sss/pipes:rw - /etc/nsswitch.conf:/etc/nsswitch.conf - /etc/pam.d:/etc/pam.d - - nginx: - hostname: ${HOSTNAME} - image: nginx:stable - volumes: - - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro - - ./nginx/uwsgi_params:/etc/nginx/uwsgi_params:ro - - ./certs:/etc/nginx/certs:ro - - static-content:/var/www/static:ro - - media-uploads:/var/www/media:ro - - nginx-logs:/var/logs/nginx - ports: - - "80:80" - - "443:443" - depends_on: - - rcamp-uwsgi - -volumes: - nginx-logs: diff --git a/docker-compose.test-backends.yml b/docker-compose.test-backends.yml index c1a2c85..65370a1 100644 --- a/docker-compose.test-backends.yml +++ b/docker-compose.test-backends.yml @@ -1,11 +1,17 @@ version: "3.6" services: - nginx: + test-mysql: + image: mysql:5.7 + environment: + - MYSQL_ROOT_PASSWORD=password + - MYSQL_DATABASE=rcamp1712 volumes: - - certs:/etc/nginx/certs:ro - depends_on: - - cert-gen + - ./db:/var/lib/mysql + + test-ldap: + image: researchcomputing/rc-test-ldap + # build: ldap cert-gen: build: @@ -16,17 +22,11 @@ services: volumes: - certs:/home/root/certs - test-mysql: - image: mysql:5.7 - environment: - - MYSQL_ROOT_PASSWORD=password - - MYSQL_DATABASE=rcamp1712 + nginx: volumes: - - ./db:/var/lib/mysql - - test-ldap: - image: researchcomputing/rc-test-ldap - # build: ldap + - certs:/etc/nginx/certs:ro + depends_on: + - cert-gen volumes: certs: diff --git a/docker-compose.yml b/docker-compose.yml index 8ca1838..2bc8894 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,11 @@ version: "3.6" services: rcamp-uwsgi: build: - context: rcamp + context: ./rcamp args: - UWSGI_UID=${UWSGI_UID} - UWSGI_GID=${UWSGI_GID} + container_name: rcamp-uwsgi environment: - RCAMP_UCB_LDAP_URI=${RCAMP_UCB_LDAP_URI} - RCAMP_UCB_LDAP_USER=${RCAMP_UCB_LDAP_USER} @@ -19,7 +20,25 @@ services: - media-uploads:/home/uwsgi/rcamp/media - rcamp-logs:/home/uwsgi/rcamp/logs + nginx: + hostname: ${HOSTNAME} + image: nginx:stable + container_name: nginx + volumes: + - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./nginx/uwsgi_params:/etc/nginx/uwsgi_params:ro + - ./certs:/etc/nginx/certs:ro + - static-content:/var/www/static:ro + - media-uploads:/var/www/media:ro + - nginx-logs:/var/logs/nginx + ports: + - "80:80" + - "443:443" + depends_on: + - rcamp-uwsgi + volumes: static-content: media-uploads: rcamp-logs: + nginx-logs: diff --git a/install-rcamp-services.sh b/scripts/install-rcamp-services.sh old mode 100644 new mode 100755 similarity index 100% rename from install-rcamp-services.sh rename to scripts/install-rcamp-services.sh diff --git a/rcamp.dev.service b/scripts/rcamp.dev.service similarity index 100% rename from rcamp.dev.service rename to scripts/rcamp.dev.service diff --git a/rcamp.prod.service b/scripts/rcamp.prod.service similarity index 100% rename from rcamp.prod.service rename to scripts/rcamp.prod.service diff --git a/scripts/rebuild-rcamp-dev.sh b/scripts/rebuild-rcamp-dev.sh new file mode 100755 index 0000000..db8e03a --- /dev/null +++ b/scripts/rebuild-rcamp-dev.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export RCAMP_PORT=9000 +export HOSTNAME=localhost +export UWSGI_UID=$(id -u) +export UWSGI_GID=$(id -u) + +cd $1 +docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml down --remove-orphans +cd $1 +docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build From 0cf084d08578cd9c5ef823d368b0d182307d0f1e Mon Sep 17 00:00:00 2001 From: Zebula Sampedro Date: Fri, 14 Dec 2018 10:16:43 -0700 Subject: [PATCH 2/4] Fixed sysntax error in travis file. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ee3536..18565f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: - export UWSGI_UID=$(id -u) - export UWSGI_GID=$(id -u) - export RCAMP_PORT=9000 -- export HOSTNAME-localhost +- export HOSTNAME=localhost - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'sleep 30s && python manage.py migrate' - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' From d17ef5eb4b1ef7bbf1678c2da127745bf87c35f9 Mon Sep 17 00:00:00 2001 From: Zebula Sampedro Date: Fri, 14 Dec 2018 11:04:04 -0700 Subject: [PATCH 3/4] Added functional suite back in. --- .dockerignore | 1 - .travis.yml | 1 + docker-compose.dev.yml | 3 +++ rcamp/Dockerfile.test | 2 -- scripts/rebuild-rcamp-dev.sh | 1 + 5 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index d5910a1..0000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -**/*_functional.py[cod] diff --git a/.travis.yml b/.travis.yml index 18565f7..9945a08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ before_install: - export UWSGI_GID=$(id -u) - export RCAMP_PORT=9000 - export HOSTNAME=localhost +- docker build -t dev/rcamp --build-arg UWSGI_UID=$UWSGI_UID --build-arg UWSGI_GID=$UWSGI_GID rcamp/ - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'sleep 30s && python manage.py migrate' - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 320c46f..80b1aa4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,6 +2,9 @@ version: "3.6" services: rcamp-uwsgi: + build: + context: ./rcamp + dockerfile: Dockerfile.test command: ["python", "manage.py", "runserver", "0.0.0.0:9000"] environment: - RCAMP_DEBUG=True diff --git a/rcamp/Dockerfile.test b/rcamp/Dockerfile.test index f91db7c..94823e0 100644 --- a/rcamp/Dockerfile.test +++ b/rcamp/Dockerfile.test @@ -14,5 +14,3 @@ RUN useradd -M -p $(echo password | openssl passwd -1 -stdin) testuser1 && \ useradd -M -p $(echo password | openssl passwd -1 -stdin) testuser2 WORKDIR /home/uwsgi/rcamp - -CMD ["python","manage.py","runserver","0.0.0.0:9000"] diff --git a/scripts/rebuild-rcamp-dev.sh b/scripts/rebuild-rcamp-dev.sh index db8e03a..70acde9 100755 --- a/scripts/rebuild-rcamp-dev.sh +++ b/scripts/rebuild-rcamp-dev.sh @@ -6,6 +6,7 @@ export UWSGI_UID=$(id -u) export UWSGI_GID=$(id -u) cd $1 +docker build -t dev/rcamp --build-arg UWSGI_UID=$UWSGI_UID --build-arg UWSGI_GID=$UWSGI_GID rcamp/ docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml down --remove-orphans cd $1 docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build From 9f424f0f57e4597bf1aa7def2f10d4d497f1f2bb Mon Sep 17 00:00:00 2001 From: Zebula Sampedro Date: Fri, 14 Dec 2018 11:20:21 -0700 Subject: [PATCH 4/4] Added travis_retry prefix to tests. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9945a08..726aca9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - docker build -t dev/rcamp --build-arg UWSGI_UID=$UWSGI_UID --build-arg UWSGI_GID=$UWSGI_GID rcamp/ - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml build - docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'sleep 30s && python manage.py migrate' -- docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' +- travis_retry docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.test-backends.yml run --rm --service-ports rcamp-uwsgi bash -c 'python manage.py test' install: true script: true