Add Python 3 support

bench/build_bench_docker_images.sh

@@ -25,6 +25,7 @@ fi
 # Assume local clipper/py-rpc base image (if exists) or pulled image is correct
 
 docker build -t clipper/py-rpc:$tag -f ./Py2RPCDockerfile ./
+docker build -t clipper/py3-rpc:$tag -f ./Py3RPCDockerfile ./
 if [ $# -eq 0 ]; then
 	time docker build build --build-arg CODE_VERSION=$tag -t clipper/sum-bench:$tag -f dockerfiles/SumBenchDockerfile ./
 	time docker build build --build-arg CODE_VERSION=$tag -t clipper/noop-bench:$tag -f dockerfiles/NoopBenchDockerfile ./

bin/build_docker_images.sh

@@ -245,17 +245,24 @@ build_images () {
     create_image query_frontend QueryFrontendDockerfile $public
     create_image management_frontend ManagementFrontendDockerfile $public
     create_image dev ClipperDevDockerfile  $public
+    create_image py35-dev ClipperPy35DevDockerfile  $public
     create_image unittests ClipperTestsDockerfile  $private
+    create_image py35tests ClipperPy35TestsDockerfile  $private
 
-    # Build containers
+
+    # Build containers for other languages
     create_image spark-scala-container SparkScalaContainerDockerfile $public
     create_image r-container-base RContainerDockerfile $public
 
     # First build Python base image
     create_image py-rpc Py2RPCDockerfile $public
+    create_image py35-rpc Py35RPCDockerfile $public
+    create_image py36-rpc Py36RPCDockerfile $public
     create_image sum-container SumDockerfile  $private
     create_image noop-container NoopDockerfile $public
     create_image python-closure-container PyClosureContainerDockerfile $public
+    create_image python35-closure-container Py35ClosureContainerDockerfile $public
+    create_image python36-closure-container Py36ClosureContainerDockerfile $public
     create_image pyspark-container PySparkContainerDockerfile $public
     create_image tf-container TensorFlowDockerfile $public
     create_image pytorch-container PyTorchContainerDockerfile $public

bin/ci_checks.sh

@@ -4,6 +4,9 @@ set -e
 set -u
 set -o pipefail
 
+
+run_all=$1
+
 function clean_up {
     # Clean up credentials
     rm $KUBECONFIG
@@ -48,5 +51,9 @@ kubectl get nodes
 # Set kubectl proxy for k8s tests later
 kubectl proxy --port 8080 &
 
-$DIR/check_format.sh
-$DIR/run_unittests.sh
+if [[ $run_all = "true" ]]; then
+    $DIR/check_format.sh
+    $DIR/run_unittests.sh
+else
+    $DIR/run_unittests.sh -i
+fi

bin/construct_kube_config.py

@@ -29,33 +29,33 @@ def write_config(dest_path):
         'Config',
         'preferences': {},
         'current-context':
-        'cluster.clipper-k8s-testing.com',
+        'jenkins.clipper-k8s-testing.com',
         'contexts': [{
-            'name': 'cluster.clipper-k8s-testing.com',
+            'name': 'jenkins.clipper-k8s-testing.com',
             'context': {
-                'cluster': 'cluster.clipper-k8s-testing.com',
-                'user': 'cluster.clipper-k8s-testing.com'
+                'cluster': 'jenkins.clipper-k8s-testing.com',
+                'user': 'jenkins.clipper-k8s-testing.com'
             }
         }],
         'clusters': [{
             'cluster': {
                 'certificate-authority-data': cert_auth_data,
-                'server': 'https://api.cluster.clipper-k8s-testing.com'
+                'server': 'https://api.jenkins.clipper-k8s-testing.com'
             },
-            'name': 'cluster.clipper-k8s-testing.com'
+            'name': 'jenkins.clipper-k8s-testing.com'
         }],
         'apiVersion':
         'v1',
         'users': [{
-            'name': 'cluster.clipper-k8s-testing.com',
+            'name': 'jenkins.clipper-k8s-testing.com',
             'user': {
                 'username': 'admin',
                 'password': pw_data,
                 'client-key-data': client_key_data,
                 'client-certificate-data': client_cert_data
             }
         }, {
-            'name': 'cluster.clipper-k8s-testing.com-basic-auth',
+            'name': 'jenkins.clipper-k8s-testing.com-basic-auth',
             'user': {
                 'username': 'admin',
                 'password': pw_data

bin/run_ci.sh

@@ -34,3 +34,13 @@ docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /
     -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
     -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
     clipper/unittests:$tag
+
+# Python 3 unittests
+docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
+    -e CLIPPER_K8S_CERT_AUTH=$CLIPPER_K8S_CERT_AUTH \
+    -e CLIPPER_K8S_CLIENT_CERT=$CLIPPER_K8S_CLIENT_CERT \
+    -e CLIPPER_K8S_CLIENT_KEY=$CLIPPER_K8S_CLIENT_KEY \
+    -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
+    -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+    -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+    clipper/py35tests:$tag

bin/run_unittests.sh

@@ -35,7 +35,7 @@ function clean_up {
     # echo Killing jobs
     echo Exiting unit tests...
     kill $(jobs -p) &> /dev/null
-    echo
+    echo "Cleanup exit code: $?"
     sleep 2
     # echo Remaining background jobs: $(jobs -l)
     exit
@@ -149,6 +149,8 @@ function run_integration_tests {
   ../integration-tests/r_integration_test/rclipper_test.sh
   python ../integration-tests/clipper_metric_docker.py 
   python ../integration-tests/clipper_metric_kube.py 
+  echo "Exit code: $?"
+  echo "GREPTHIS Done running unit tests"
 }
 
 function run_all_tests {
@@ -178,33 +180,34 @@ else
 fi
 
 case $args in
-    -a | --all )            set_test_environment
-                            run_all_tests
-                            ;;
-    -l | --libclipper )     set_test_environment
-                            run_libclipper_tests
-                            ;;
-    -m | --management )     set_test_environment
-                            run_management_tests
-                            ;;
-    -f | --frontend )       set_test_environment
-                            run_frontend_tests
-                            ;;
-    -j | --jvm-container )  set_test_environment
-                            run_jvm_container_tests
-                            ;;
-    -c | --cpp-container )  set_test_environment
-                            run_cpp_container_tests
-                            ;;
-    -rc | --r-container )   set_test_environment
-                            run_r_container_tests
-                            ;;
-    -r | --rpc-container )  set_test_environment
-                            run_rpc_container_tests
-                            ;;
-    -i | --integration_tests ) run_integration_tests
-                            ;;
-    -h | --help )           usage
-                            ;;
-    * )                     usage
+    -a | --all )                set_test_environment
+                                run_all_tests
+                                ;;
+    -l | --libclipper )         set_test_environment
+                                run_libclipper_tests
+                                ;;
+    -m | --management )         set_test_environment
+                                run_management_tests
+                                ;;
+    -f | --frontend )           set_test_environment
+                                run_frontend_tests
+                                ;;
+    -j | --jvm-container )      set_test_environment
+                                run_jvm_container_tests
+                                ;;
+    -c | --cpp-container )      set_test_environment
+                                run_cpp_container_tests
+                                ;;
+    -rc | --r-container )       set_test_environment
+                                run_r_container_tests
+                                ;;
+    -r | --rpc-container )      set_test_environment
+                                run_rpc_container_tests
+                                ;;
+    -i | --integration_tests )  set_test_environment
+                                run_integration_tests
+                                ;;
+    -h | --help )               usage
+                                ;;
+    * )                         usage
 esac

clipper_admin/MANIFEST.in

@@ -1 +1,2 @@
-#include README.rst
+include README.rst LICENSE.txt clipper_admin/VERSION.txt
+recursive-include tests *.yaml