Skip to content
Permalink
Browse files

Restructure apps and scripts for docker image builds (#705)

  • Loading branch information...
DaoWen authored and shamsimam committed Apr 25, 2019
1 parent bbde350 commit 87f5ad80ccad6610b917341cd90cfee9ffcd94a0
Showing with 132 additions and 87 deletions.
  1. +1 −1 .travis.yml
  2. +1 −1 README.md
  3. +1 −1 cli/bin/ci/run-integration-tests.sh
  4. +1 −1 cli/integration/tests/waiter/util.py
  5. +6 −0 containers/README.md
  6. +29 −0 containers/bin/build-docker-images.sh
  7. +2 −0 containers/custom-env/Dockerfile
  8. +2 −0 containers/custom-env/README.md
  9. +8 −0 containers/custom-env/bin/build-docker-image.sh
  10. 0 {kubernetes/fileserver-container → containers/fileserver}/Dockerfile
  11. 0 {kubernetes/fileserver-container → containers/fileserver}/README.md
  12. +8 −0 containers/fileserver/bin/build-docker-image.sh
  13. 0 {kubernetes/fileserver-container → containers/fileserver}/fileserver-start
  14. 0 {kubernetes/fileserver-container → containers/fileserver}/nginx.conf
  15. +2 −5 { → containers}/test-apps/Dockerfile
  16. 0 { → containers}/test-apps/README.md
  17. +13 −0 containers/test-apps/bin/build-docker-image.sh
  18. +10 −0 containers/test-apps/bin/ci/run.sh
  19. +6 −0 containers/test-apps/bin/ci/setup.sh
  20. +3 −0 containers/test-apps/kitchen/.gitignore
  21. 0 { → containers}/test-apps/kitchen/README.md
  22. +0 −6 { → containers}/test-apps/kitchen/bin/ci/run.sh
  23. 0 { → containers}/test-apps/kitchen/bin/ci/setup.sh
  24. 0 { → containers}/test-apps/kitchen/bin/ci/ssl-env.sh
  25. 0 { → containers}/test-apps/kitchen/bin/kitchen
  26. 0 { → containers}/test-apps/kitchen/requirements_test.txt
  27. 0 { → containers}/test-apps/kitchen/setup.cfg
  28. 0 { → containers}/test-apps/kitchen/tests/__init__.py
  29. 0 { → containers}/test-apps/kitchen/tests/kitchen/__init__.py
  30. 0 { → containers}/test-apps/kitchen/tests/kitchen/conftest.py
  31. 0 { → containers}/test-apps/kitchen/tests/kitchen/test_basic_http.py
  32. 0 { → containers}/test-apps/kitchen/tests/kitchen/util.py
  33. 0 { → containers}/test-apps/nginx/README.md
  34. 0 { → containers}/test-apps/nginx/bin/run-nginx-server.sh
  35. 0 { → containers}/test-apps/nginx/data/nginx-template.conf
  36. 0 { → containers}/test-apps/sediment/.gitignore
  37. 0 { → containers}/test-apps/sediment/README.md
  38. 0 { → containers}/test-apps/sediment/bin/run-sediment-server.sh
  39. 0 { → containers}/test-apps/sediment/pom.xml
  40. 0 { → containers}/test-apps/sediment/src/main/java/com/twosigma/waiter/sediment/HttpServer.java
  41. 0 { → containers}/test-apps/sediment/src/main/java/com/twosigma/waiter/sediment/RequestInfo.java
  42. 0 { → containers}/test-apps/sediment/src/main/java/com/twosigma/waiter/sediment/Servlet.java
  43. 0 { → containers}/test-apps/sediment/src/test/java/com/twosigma/waiter/sediment/ServletTest.java
  44. +0 −4 {test-apps/kitchen → containers/test-apps/waiter-init}/.gitignore
  45. +9 −0 containers/test-apps/waiter-init/bin/ci/run.sh
  46. 0 {test-apps/kitchen → containers/test-apps/waiter-init}/bin/test-waiter-init
  47. +1 −1 {test-apps/kitchen → containers/test-apps/waiter-init}/bin/waiter-init
  48. +0 −17 test-apps/bin/build-docker-image.sh
  49. +0 −26 test-apps/bin/push-docker-image.sh
  50. +1 −1 token-syncer/bin/ci/run-integration-tests.sh
  51. +1 −1 waiter/bin/ci/run-integration-tests-composite-scheduler.sh
  52. +3 −3 waiter/bin/ci/run-integration-tests-k8s-scheduler.sh
  53. +11 −9 waiter/bin/ci/run-integration-tests-marathon-scheduler.sh
  54. +1 −1 waiter/bin/ci/run-integration-tests-shell-scheduler.sh
  55. +8 −5 waiter/bin/ci/run-integration-tests.sh
  56. +3 −3 waiter/project.clj
  57. +1 −1 waiter/src/waiter/settings.clj
@@ -31,7 +31,7 @@ matrix:
- name: 'Kitchen tests'
language: python
python: "3.6"
before_script: cd test-apps/kitchen && ./bin/ci/setup.sh
before_script: cd containers/test-apps && ./bin/ci/setup.sh
script: ./bin/ci/run.sh

- name: 'Token syncer tests'
@@ -27,7 +27,7 @@ The quickest way to get Mesos, Marathon, and Waiter running locally is with [doc
1. Install `docker`
1. Install `minimesos`
1. Clone down this repo
1. Run `kitchen/bin/build-docker-image.sh` to build the minimesos agent image with kitchen baked in
1. Run `containers/bin/build-docker-images.sh` to build the minimesos agent image with kitchen and other test apps baked in
1. `cd waiter`
1. Run `minimesos up` to start Mesos, ZooKeeper, and Marathon
1. Run `bin/run-using-minimesos.sh` to start Waiter
@@ -64,7 +64,7 @@ curl -s ${WAITER_URI_2}/state

# Run the integration tests
export WAITER_URI=127.0.0.1:${WAITER_PORT}
export WAITER_CLI_TEST_DEFAULT_CMD="${ROOT_DIR}/test-apps/kitchen/bin/kitchen --port \${PORT0}"
export WAITER_CLI_TEST_DEFAULT_CMD="${ROOT_DIR}/containers/test-apps/kitchen/bin/kitchen --port \${PORT0}"
export WAITER_TEST_MULTI_CLUSTER=true
cd ${CLI_DIR}/integration
pytest
@@ -102,7 +102,7 @@ def post_token(waiter_url, token_name, token_definition, assert_response=True, e
def minimal_service_cmd():
this_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
kitchen = os.path.join(this_dir, os.pardir, os.pardir, os.pardir, os.pardir,
'test-apps', 'kitchen', 'bin', 'kitchen')
'containers', 'test-apps', 'kitchen', 'bin', 'kitchen')
kitchen = os.path.abspath(kitchen)
return f'{kitchen} -p $PORT0'

@@ -0,0 +1,6 @@
This directory contains code and configurations for building the docker containers
that are used to run and test Waiter in our development environments.

As part of the setup for both our minimesos (Marathon) and minikube (Kubernetes)
continuous integration pipelines, we run the ./bin/build-docker-images.sh
script found in this directory.
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Usage: build-docker-images.sh [OPTION]
#
# Builds docker images that are used with our default configurations to run and test Waiter
# in both the Marathon (minimesos) and Kubernetes (minikube) environments.
#
# Mutually exclusive options:
# --minikube Build images in the minikube docker context
# --all Build images in both the local and the minikube docker contexts

set -ex

if [ "$1" == --minikube ]; then
echo 'Using minikube docker environment'
eval $(minikube docker-env)
elif [ "$1" == --all ]; then
eval "$0 --minikube"
echo 'Continuing using local docker environment'
fi

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
cd "$DIR"

# the custom-env container uses test-apps as a base, so this order is important
./test-apps/bin/build-docker-image.sh
./custom-env/bin/build-docker-image.sh

# the fileserver container is independent
./fileserver/bin/build-docker-image.sh
@@ -0,0 +1,2 @@
FROM twosigma/waiter-test-apps
ENV INTEGRATION_TEST_SENTINEL_VALUE="Integration Test Sentinel Value"
@@ -0,0 +1,2 @@
This image is used in our integration tests to verify that the "image" field works as expected.
The test expects a specific value in the INTEGRATION_TEST_SENTINEL_VALUE environment variable.
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
cd "$DIR"

CUSTOM_IMAGE=twosigma/integration
echo "Building docker image for ${CUSTOM_IMAGE}"
docker build -t ${CUSTOM_IMAGE} .
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
cd "$DIR"

FILESERVER_IMAGE=twosigma/waiter-fileserver
echo "Building docker image for ${FILESERVER_IMAGE}"
docker build -t ${FILESERVER_IMAGE} .
File renamed without changes.
File renamed without changes.
@@ -3,14 +3,11 @@ FROM twosigma/mesos-agent:1.0.0-1.0.1604
# we need nginx for our http/2 tests
# gettext-base is needed for envsubst in the nginx server
# python is needed to launch kitchen
RUN apt-get update \
&& apt-get install -y gettext-base \
&& apt-get install -y nginx \
&& apt-get install -y python3
RUN apt-get update && apt-get install -y gettext-base nginx python3

COPY kitchen/bin/waiter-init /usr/bin/waiter-init
COPY kitchen/bin/kitchen /opt/kitchen/kitchen
COPY nginx/bin/run-nginx-server.sh /opt/nginx/bin/run-nginx-server.sh
COPY nginx/data/nginx-template.conf /opt/nginx/data/nginx-template.conf
COPY sediment/bin/run-sediment-server.sh /opt/sediment/bin/run-sediment-server.sh
COPY sediment/data/sediment-uberjar.jar /opt/sediment/data/sediment-uberjar.jar
COPY waiter-init/bin/waiter-init /usr/bin/waiter-init
File renamed without changes.
@@ -0,0 +1,13 @@
#!/usr/bin/env bash

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
cd "$DIR"

TEST_APPS_IMAGE=twosigma/waiter-test-apps

pushd ./sediment
mvn clean package
popd

echo "Building docker image for ${TEST_APPS_IMAGE}"
docker build -t ${TEST_APPS_IMAGE} .
@@ -0,0 +1,10 @@
#!/bin/bash

pushd kitchen
./bin/ci/run.sh
popd

pushd waiter-init
./bin/ci/run.sh
popd

@@ -0,0 +1,6 @@
#!/bin/bash

pushd kitchen
./bin/ci/setup.sh
popd

@@ -0,0 +1,3 @@
.cache
__pycache__
kitchen.log
File renamed without changes.
@@ -12,9 +12,3 @@ python --version
pytest --version

pytest

#
# Run waiter-init script tests
#

./bin/test-waiter-init
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,7 +1,3 @@
.cache
__pycache__
kitchen.log

/latest
/r[0-9]
/.r[0-9].index.json
@@ -0,0 +1,9 @@
#!/bin/bash

set -ex

#
# Run waiter-init script tests
#

./bin/test-waiter-init
File renamed without changes.
@@ -1,7 +1,7 @@
#!/bin/bash
#
# A wrapper script for the Waiter-specific setup for a user command in a Waiter-K8s pod.
# The script is usually invoked by prepending it to the user's Waiter command.
# The script is usually invoked by prepending it to the user's Waiter command array.
# If this script is invoked by dumb-init (github.com/Yelp/dumb-init) or a similar utility,
# please ensure that it is run in single-child mode.
#

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
@@ -16,7 +16,7 @@ TEST_SELECTOR=${2:-integration}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_DIR=${DIR}/../../../
WAITER_DIR=${PROJECT_DIR}/waiter
KITCHEN_DIR=${PROJECT_DIR}/test-apps/kitchen
KITCHEN_DIR=${PROJECT_DIR}/containers/test-apps/kitchen
SYNCER_DIR=${PROJECT_DIR}/token-syncer

pushd ${WAITER_DIR}
@@ -16,7 +16,7 @@ TEST_SELECTOR=${2:-integration}

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WAITER_DIR=${DIR}/../..
TEST_APPS_DIR=${WAITER_DIR}/../test-apps
TEST_APPS_DIR=${WAITER_DIR}/../containers/test-apps
KITCHEN_DIR=${TEST_APPS_DIR}/kitchen
NGINX_DIR=${TEST_APPS_DIR}/nginx
SEDIMENT_DIR=${TEST_APPS_DIR}/sediment
@@ -16,7 +16,7 @@ TEST_SELECTOR=${2:-integration}

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WAITER_DIR=${DIR}/../..
TEST_APPS_DIR=${WAITER_DIR}/../test-apps
CONTAINERS_DIR=${WAITER_DIR}/../containers

# Start minikube
${DIR}/minikube-setup.sh
@@ -29,10 +29,10 @@ if [[ $TEST_SELECTOR =~ heavy$ ]]; then
fi

# Ensure we have the docker image for the pods
${TEST_APPS_DIR}/bin/build-docker-image.sh
${CONTAINERS_DIR}/bin/build-docker-images.sh

# Start waiter
: ${WAITER_PORT:=9091}
export WAITER_PORT=9091
${WAITER_DIR}/bin/run-using-k8s.sh ${WAITER_PORT} &

# Start monitoring state of Kubernetes pods
@@ -11,25 +11,27 @@

set -e

export WAITER_PORT=9091
TEST_COMMAND=${1:-parallel-test}
TEST_SELECTOR=${2:-integration}

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WAITER_DIR=${DIR}/../..
TEST_APPS_DIR=${WAITER_DIR}/../test-apps
TEST_APPS_DIR=${WAITER_DIR}/../containers/test-apps

# Build mesos agent container with Kitchen packed in
${TEST_APPS_DIR}/bin/build-docker-image.sh

# Start minimesos
export MINIMESOS_CMD=${DIR}/minimesos
pushd ${WAITER_DIR}
${MINIMESOS_CMD} up
popd
if [ -n "$CONTINUOUS_INTEGRATION" ]; then
# Start minimesos
export MINIMESOS_CMD=${DIR}/minimesos
pushd ${WAITER_DIR}
${MINIMESOS_CMD} up
popd

# Start waiter
: ${WAITER_PORT:=9091}
${WAITER_DIR}/bin/run-using-minimesos.sh ${WAITER_PORT} &
# Start waiter
${WAITER_DIR}/bin/run-using-minimesos.sh ${WAITER_PORT} &
fi

# Run the integration tests
export WAITER_TEST_KITCHEN_CMD=/opt/kitchen/kitchen
@@ -16,7 +16,7 @@ TEST_SELECTOR=${2:-integration}

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WAITER_DIR=${DIR}/../..
TEST_APPS_DIR=${WAITER_DIR}/../test-apps
TEST_APPS_DIR=${WAITER_DIR}/../containers/test-apps
KITCHEN_DIR=${TEST_APPS_DIR}/kitchen
NGINX_DIR=${TEST_APPS_DIR}/nginx
SEDIMENT_DIR=${TEST_APPS_DIR}/sediment
@@ -15,12 +15,12 @@ WAITER_DIR=${DIR}/../..
SCHEDULER="${1:-shell}-scheduler"
SUBCMD="${DIR}/run-integration-tests-${SCHEDULER}.sh ${2:-parallel-test} ${3:-integration}"

if [ "${TRAVIS}" == true ]; then

export GRAPHITE_SERVER_PORT=5555
# Start netcat to listen to a port. The Codahale Graphite reporter will be able to report without failing and spamming logs.
nc -kl localhost $GRAPHITE_SERVER_PORT > /dev/null &
# Start netcat to listen to a port. The Codahale Graphite reporter will be able to report without failing and spamming logs.
export GRAPHITE_SERVER_PORT=5555
nc -kl localhost $GRAPHITE_SERVER_PORT > /dev/null &
ncat_pid=$!

if [ "${TRAVIS}" == true ]; then
# Capture integration test command output into a log file
mkdir -p ${WAITER_DIR}/log
bash -x -c "${SUBCMD}" &> >(tee ${WAITER_DIR}/log/travis.log)
@@ -34,3 +34,6 @@ if [ "${TRAVIS}" == true ]; then
else
eval ${SUBCMD}
fi

# Clean up ncat server
kill -9 $ncat_pid
@@ -131,13 +131,13 @@
:test {:jvm-opts
[~(str "-Dwaiter.test.kitchen.cmd="
(or (System/getenv "WAITER_TEST_KITCHEN_CMD")
(.getCanonicalPath (clojure.java.io/file "../test-apps/kitchen/bin/kitchen"))))
(.getCanonicalPath (clojure.java.io/file "../containers/test-apps/kitchen/bin/kitchen"))))
~(str "-Dwaiter.test.nginx.cmd="
(or (System/getenv "WAITER_TEST_NGINX_CMD")
(.getCanonicalPath (clojure.java.io/file "../test-apps/nginx/bin/run-nginx-server.sh"))))
(.getCanonicalPath (clojure.java.io/file "../containers/test-apps/nginx/bin/run-nginx-server.sh"))))
~(str "-Dwaiter.test.sediment.cmd="
(or (System/getenv "WAITER_TEST_SEDIMENT_CMD")
(.getCanonicalPath (clojure.java.io/file "../test-apps/sediment/bin/run-sediment-server.sh"))))]
(.getCanonicalPath (clojure.java.io/file "../containers/test-apps/sediment/bin/run-sediment-server.sh"))))]
:parallel-test {:pools {:serial (constantly 1)
:parallel (fn []
(or (some-> (System/getenv "LEIN_TEST_THREADS") Long/valueOf)
@@ -343,7 +343,7 @@
:default {:factory-fn 'waiter.authorization/noop-authorizer}}
:cluster-name "waiter"
:fileserver {:cmd ["/bin/fileserver-start"]
:image "twosigma/waiter-fileserver:20190410"
:image "twosigma/waiter-fileserver"
:resources {:cpu 0.1 :mem 128}
:scheme "http"}
:http-options {:conn-timeout 10000

0 comments on commit 87f5ad8

Please sign in to comment.
You can’t perform that action at this time.