Add missing integration tests, clean up dockerfiles, and remove Anaco…

…nda dependencies (#435)

* added missing mxnet, pytorch, and pytorch-onnx-caffe2 integration tests

* change data_shapes parameter in integration test to data_iter.provide_data, a DataDesc object

* updated documentation in deployers/mxnet.py to match changes

* typo in file name

* changed print statements to logging

* format code

* fixed some onnx issues but the container still is not working

* fixed container build issues related to conda segfaults and out of date packages

* reverted .gitignore

* fixed mxnet issues

* fixed onnx tests

* format and remove conda update conda

* major overhaul of docker images

* Removed conda module deps analyzer

* All docker images building

* removed local module tests

* format code

* install pyspark in dev image

* add xgboost

* fixed onnx caffe docker image

* changed way we handle docker login for k8s docker repo

* check out specific caffe2 commit and install awscli in clippertests docker image

* catch exception in kubernetes integration test

* install codedepends dependencies

* check R packages are properly installed

* removed conda module analysis code

* added maintainer label to all dockerfiles

* Add Rscript for installing R container test dependencies

* Added trycatch to r install scripts

* added data_iter to mxnet example

bench/build_bench_docker_images.sh

@@ -24,7 +24,7 @@ fi
 # Build the Clipper Docker images
 # Assume local clipper/py-rpc base image (if exists) or pulled image is correct
 
-docker build -t clipper/py-rpc:$tag -f ./RPCDockerfile ./
+docker build -t clipper/py-rpc:$tag -f ./Py2RPCDockerfile ./
 if [ $# -eq 0 ]; then
 	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f dockerfiles/SumBenchDockerfile ./
 	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f dockerfiles/NoopBenchDockerfile ./

bin/aws_docker_repo_login.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set +x
+
+RETVAL=$(aws ecr get-login --no-include-email --region us-west-1)
+eval "$RETVAL"
+
+set -x

bin/build_docker_images.sh

@@ -244,19 +244,19 @@ build_images () {
     create_image lib_base ClipperLibBaseDockerfile $private
     create_image query_frontend QueryFrontendDockerfile $public
     create_image management_frontend ManagementFrontendDockerfile $public
+    create_image dev ClipperDevDockerfile  $public
     create_image unittests ClipperTestsDockerfile  $private
 
     # Build containers
     create_image spark-scala-container SparkScalaContainerDockerfile $public
     create_image r-container-base RContainerDockerfile $public
 
     # First build Python base image
-    create_image py-rpc RPCDockerfile $public
+    create_image py-rpc Py2RPCDockerfile $public
     create_image sum-container SumDockerfile  $private
     create_image noop-container NoopDockerfile $public
     create_image python-closure-container PyClosureContainerDockerfile $public
     create_image pyspark-container PySparkContainerDockerfile $public
-    create_image tf_cifar_container TensorFlowCifarDockerfile $public
     create_image tf-container TensorFlowDockerfile $public
     create_image pytorch-container PyTorchContainerDockerfile $public
     create_image caffe2-onnx-container Caffe2OnnxDockerfile $public

bin/ci_checks.sh

@@ -22,6 +22,9 @@ cd $DIR/..
 tag=$(<VERSION.txt)
 cd -
 
+# Log in to Kubernetes Docker repo
+$DIR/aws_docker_repo_login.sh
+
 # Test docker login
 docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
 

bin/run_ci.sh

@@ -27,9 +27,10 @@ docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_fron
 
 # Run tests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
-    -v /home/jenkins/.docker:/root/.docker \
     -e CLIPPER_K8S_CERT_AUTH=$CLIPPER_K8S_CERT_AUTH \
     -e CLIPPER_K8S_CLIENT_CERT=$CLIPPER_K8S_CLIENT_CERT \
     -e CLIPPER_K8S_CLIENT_KEY=$CLIPPER_K8S_CLIENT_KEY \
     -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
+    -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+    -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
     clipper/unittests:$tag

bin/run_unittests.sh

@@ -142,6 +142,9 @@ function run_integration_tests {
   python ../integration-tests/deploy_pyspark_sparkml_models.py
   python ../integration-tests/kubernetes_integration_test.py
   python ../integration-tests/deploy_tensorflow_models.py
+  python ../integration-tests/deploy_mxnet_models.py 
+  python ../integration-tests/deploy_pytorch_models.py 
+  python ../integration-tests/deploy_pytorch_to_caffe2_with_onnx.py
   ../integration-tests/r_integration_test/rclipper_test.sh
   python ../integration-tests/clipper_metric_docker.py 
   python ../integration-tests/clipper_metric_kube.py 
@@ -198,8 +201,7 @@ case $args in
     -r | --rpc-container )  set_test_environment
                             run_rpc_container_tests
                             ;;
-    -i | --integration_tests )  set_test_environment
-                            run_integration_tests
+    -i | --integration_tests ) run_integration_tests
                             ;;
     -h | --help )           usage
                             ;;