Merge pull request #368 from cliveseldon/sagemaker

WIP: Train on Sagemaker, Deploy on Seldon Core

.gitignore

@@ -150,5 +150,11 @@ eggs/
 *.egg-info/
 ./pytest_cache
 
+examples/models/sagemaker/sagemaker-scikit-learn-container
+examples/models/sagemaker/aws-config-secret.yaml
+examples/models/sagemaker/scikit_learn_iris_code.tar.gz
+examples/models/sagemaker/scikit_learn_iris_deployment.json
+
 #go example
 examples/wrappers/go/.idea
+

cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java

@@ -83,7 +83,18 @@ public void updateRaw(String json,String seldonDeploymentName,String namespace)
 			logger.info(json);
 			ApiClient client = Config.defaultClient();
 			CustomObjectsApi api = new CustomObjectsApi(client);
-			api.replaceNamespacedCustomObject(GROUP, VERSION, namespace, KIND_PLURAL, seldonDeploymentName,json.getBytes());
+			if (replaceStatusResource)
+			{
+				try 
+				{
+					api.replaceNamespacedCustomObjectStatus(GROUP, VERSION, namespace, KIND_PLURAL, seldonDeploymentName,json.getBytes());
+				} catch (ApiException e) {
+					replaceStatusResource = false; // Stop using the /status endpoint (maybe because the k8s version does not have this <1.10)
+					logger.warn("Failed to update deployment in kubernetes ",e);
+				}
+			}
+			if (!replaceStatusResource)
+				api.replaceNamespacedCustomObject(GROUP, VERSION, namespace, KIND_PLURAL, seldonDeploymentName,json.getBytes());
 		} catch (InvalidProtocolBufferException e) {
 			logger.error("Failed to update deployment in kubernetes ",e);
 		} catch (ApiException e) {

cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java

@@ -115,11 +115,11 @@ private void failDeployment(JsonNode mlDep,Exception e,String namespace)
 			//Update seldon deployment
 			crdHandler.updateRaw(json, name,namespace);
 		} catch (JsonParseException e1) {
-			logger.error("Fasile to create status for failed parse",e);
+			logger.error("Failed to create status for failed parse",e);
 		} catch (InvalidProtocolBufferException e1) {
-			logger.error("Fasile to create status for failed parse",e);
+			logger.error("Failed to create status for failed parse",e);
 		} catch (IOException e1) {
-			logger.error("Fasile to create status for failed parse",e);
+			logger.error("Failed to create status for failed parse",e);
 		}
 	}
 

examples/models/sagemaker/Makefile

@@ -0,0 +1,22 @@
+BUCKET=seldon-sagemaker-testing
+
+scikit_learn_iris_code.tar.gz:
+	cd scikit_learn_iris && tar -cvf ../scikit_learn_iris_code.tar .
+	gzip -f scikit_learn_iris_code.tar
+
+.PHONY: upload_scikit_code
+upload_scikit_code:
+	aws s3 cp scikit_learn_iris_code.tar.gz s3://${BUCKET}/scikit_learn_iris_code.tar.gz
+
+
+#Need to build sklearn image
+
+docker_serve:
+	docker run -it --rm -p 8080:8080 -e SAGEMAKER_MODEL_DIRECTORY="s3://seldon-sagemaker-testing/scikit_learn_iris/sagemaker-scikit-learn-2019-01-04-19-26-40-470/output/model.tar.gz" -e SAGEMAKER_SUBMIT_DIRECTORY="s3://seldon-sagemaker-testing/scikit_learn_iris_code.tar.gz"  -e SAGEMAKER_PROGRAM="scikit_learn_iris.py" -v ~/.aws:/root/.aws sklearn-final:0.20.0-cpu-py3 serve
+
+
+clean:
+	rm -f scikit_learn_iris_code.tar.gz
+	rm -rf sagemaker-scikit-learn-container
+
+

examples/models/sagemaker/README.md

@@ -0,0 +1,5 @@
+# AWS Sagemaker and Seldon Core
+
+In this tutorial we will show how to train a model using [AWS Sagemaker](https://aws.amazon.com/sagemaker/) and then deploy it locally on Seldon Core. An example showing this for a scikit-learn Iris model is shown in the [Jupyter notebook](sagemaker_seldon_scikit_iris_example.ipynb).
+
+

examples/models/sagemaker/aws-config-secret.yaml.tmpl

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-config
+type: Opaque
+data:
+  region: <region base64 encoded>
+  aws_access_key_id: <id base64 encoded>
+  aws_secret_access_key: <key base64 encoded>

examples/models/sagemaker/contract.json

@@ -0,0 +1,39 @@
+{
+    "features":[
+	{
+	    "name":"sepal_length",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[4,8]
+	},
+	{
+	    "name":"sepal_width",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[2,5]
+	},
+	{
+	    "name":"petal_length",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[1,10]
+	},
+	{
+	    "name":"petal_width",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[0,3]
+	}
+    ],
+    "targets":[
+	{
+	    "name":"class",
+	    "dtype":"FLOAT",
+	    "ftype":"continuous",
+	    "range":[0,1],
+	    "repeat":3
+	}
+    ]
+}
+
+