Added update to use pvc instead of param

examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+description: Seldon Batch Workflow
+keywords:
+- kubernetes
+- machine-learning
+name: seldon-batch-workflow
+sources:
+- https://github.com/SeldonIO/seldon-core
+version: 0.1

examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/README.md

@@ -0,0 +1,3 @@
+# Seldon Batch Workflow
+
+This chart creates a batch workflow which leverages the seldon batch processor functionality.

examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/templates/workflow.yaml

@@ -0,0 +1,88 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  {{- if eq .Values.workflow.useNameAsGenerateName false }}
+  name: {{ .Values.workflow.name }}
+  {{- else }}
+  generateName: {{ .Values.workflow.name }}
+  {{- end }}
+  namespace: {{ .Values.workflow.namespace }}
+spec:
+  entrypoint: seldon-batch-process
+  templates:
+  - name: seldon-batch-process
+    steps:
+    {{- if eq .Values.seldonDeployment.create true }}
+    - - name: create-seldon-resource            
+        template: create-seldon-resource-template
+    - - name: wait-seldon-resource
+        template: wait-seldon-resource-template
+    {{- end }}
+    - - name: process-batch-inputs
+        template: process-batch-inputs-template
+
+  {{- if eq .Values.seldonDeployment.create true }}
+  - name: create-seldon-resource-template
+    resource:
+      action: create
+      manifest: |
+        apiVersion: machinelearning.seldon.io/v1
+        kind: SeldonDeployment
+        metadata:
+          name: "{{ .Values.seldonDeployment.name }}"
+          namespace: {{ .Values.workflow.namespace }}
+          ownerReferences:
+          - apiVersion: argoproj.io/v1alpha1
+            blockOwnerDeletion: true
+            kind: Workflow
+            name: "{{`{{workflow.name}}`}}"
+            uid: "{{`{{workflow.uid}}`}}"
+        spec:
+          name: "{{ .Values.seldonDeployment.name }}"
+          predictors:
+            - graph:
+                children: []
+                implementation: {{ .Values.seldonDeployment.server }}
+                modelUri: {{ .Values.seldonDeployment.modelUri }}
+                name: classifier
+              name: default
+              replicas: {{ .Values.seldonDeployment.replicas }}
+                
+  - name: wait-seldon-resource-template
+    script:
+      image: bitnami/kubectl:1.17
+      command: [bash]
+      source: |
+        sleep 3
+        kubectl rollout status \
+            deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}')
+  {{- end }}
+
+  - name: process-batch-inputs-template
+    script:
+      image: {{ .Values.seldonDeployment.image }}
+      command: [bash]
+      source: |
+        seldon-batch-processor \
+            --deployment-name "{{ .Values.seldonDeployment.name }}" \
+            --namespace "{{ .Values.workflow.namespace }}" \
+            --host "{{ .Values.batchWorker.host }}" \
+            --workers "{{ .Values.batchWorker.workers }}" \
+            --retries "{{ .Values.batchWorker.retries }}" \
+            --input-data-path "{{ .Values.minio.inputDataPath }}" \
+            --output-data-path "{{ .Values.minio.outputDataPath }}"
+      env:
+      - name: AWS_ACCESS_KEY_ID
+        valueFrom: 
+          secretKeyRef:
+            name: {{ .Values.minio.secret.name }}
+            key: {{ .Values.minio.secret.keyName.accesskey }}
+      - name: AWS_SECRET_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.minio.secret.name }}
+            key: {{ .Values.minio.secret.keyName.secretkey }}
+      - name: AWS_ENDPOINT_URL
+        value: {{ .Values.minio.endpoint }}
+

examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/values.yaml

@@ -0,0 +1,45 @@
+workflow:
+  # Name of the overarching argo workflow
+  name: seldon-batch-process
+  # If true the randomly generated string will be apended as the name of the workflow
+  useNameAsGenerateName: false
+  # Namespace where to create the workflow and all resources in batch job
+  namespace: default
+# Seldon deployment to be created for batch processing
+seldonDeployment:
+  # Whether to create the deployment or to skip creation
+  create: true
+  # Name to use for the seldon deployment which by default appends generated workflow ID
+  name: seldon-{{workflow.uid}}
+  # Image to use for the batch client
+  image: seldonio/seldon-core-s2i-python37:1.1.1-SNAPSHOT
+  # Prepackaged model server to use [see https://docs.seldon.io/projects/seldon-core/en/latest/servers/overview.html]
+  server: SKLEARN_SERVER
+  # The URL for the model that is to be used
+  modelUri: gs://seldon-models/sklearn/iris
+  # The number of seldon deployment replicas to launch
+  replicas: 10
+# The batch worker is the component that will send the requests from the files
+batchWorker:
+  # Endpoint of for the batch client to contact the seldon deployment
+  host: istio-ingressgateway.istio-system.svc.cluster.local
+  # Number of parallel batch client workers to process the data
+  workers: 100
+  # Number of times client will try to send a request if it fails
+  retries: 3
+minio:
+  # The location of the minio endpoint 
+  endpoint: http://minio.default.svc.cluster.local:9000
+  # This is the secret that should contain the values to access minio
+  secret:
+    # The name of the secret which by default is "minio" but you can create a different one
+    name: minio
+    keyName:
+      # The key name inside that secret to find the access key to authenticate minio
+      accesskey: accesskey
+      # The key name inside that secret to find the secret key to authenticate minio
+      secretkey: secretkey
+  # The name of the file inside of minio that will contain the batch data to process
+  inputDataPath: data/input-data.txt
+  # The name of the file inside of minio that will contain the batch data to process
+  outputDataPath: data/output-data-{{workflow.uid}}.txt

examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/templates/workflow.yaml

@@ -10,6 +10,20 @@ metadata:
   namespace: {{ .Values.workflow.namespace }}
 spec:
   entrypoint: seldon-batch-process
+  volumeClaimTemplates:
+  - metadata:
+      name: "{{ .Values.pvc.name }}"
+      ownerReferences:
+      - apiVersion: argoproj.io/v1alpha1
+        blockOwnerDeletion: true
+        kind: Workflow
+        name: "{{`{{workflow.name}}`}}"
+        uid: "{{`{{workflow.uid}}`}}"
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: "{{ .Values.pvc.storage }}"
   templates:
   - name: seldon-batch-process
     steps:
@@ -21,16 +35,8 @@ spec:
         template: download-object-store-template
     - - name: process-batch-inputs
         template: process-batch-inputs-template
-        arguments:
-          artifacts:
-          - name: input-data
-            from: "{{`{{steps.download-object-store.outputs.artifacts.input-data}}`}}"
     - - name: upload-object-store
         template: upload-object-store-template
-        arguments:
-          artifacts:
-          - name: output-data
-            from: "{{`{{steps.process-batch-inputs.outputs.artifacts.output-data}}`}}"
 
   - name: create-seldon-resource-template
     resource:
@@ -63,7 +69,7 @@ spec:
       image: bitnami/kubectl:1.17
       command: [bash]
       source: |
-        sleep 3
+        sleep {{ .Values.seldonDeployment.waitTime }}
         kubectl rollout status \
             deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}')
                      
@@ -83,26 +89,20 @@ spec:
             key: {{ .Values.minio.secret.keyName.secretkey }}
       - name: MINIO_SERVER_HOST
         value: {{ .Values.minio.endpoint }}
+      volumeMounts:
+      - name: "{{ .Values.pvc.name }}"
+        mountPath: /assets
       command: [sh]
       source: |
         mc config host add minio-local $MINIO_SERVER_HOST $MINIO_SERVER_ACCESS_KEY $MINIO_SERVER_ACCESS_KEY
-        mc cp minio-local/data/input-data.txt /assets/input-data.txt
-    outputs:
-      artifacts:
-      - name: input-data
-        path: /assets/input-data.txt
+        mc cp minio-local/{{ .Values.minio.inputDataPath }} /assets/input-data.txt
             
   - name: process-batch-inputs-template
-    inputs:
-      artifacts:
-      - name: input-data
-        path: /assets/input-data.txt
-    outputs:
-      artifacts:
-      - name: output-data
-        path: /assets/output-data.txt
     script:
       image: {{ .Values.seldonDeployment.image }}
+      volumeMounts:
+      - name: "{{ .Values.pvc.name }}"
+        mountPath: /assets
       command: [bash]
       source: |
         seldon-batch-processor \
@@ -115,12 +115,11 @@ spec:
             --output-data-path "/assets/output-data.txt"
       
   - name: upload-object-store-template
-    inputs:
-      artifacts:
-      - name: output-data
-        path: /assets/output-data.txt
     script:
       image: minio/mc:RELEASE.2020-04-17T08-55-48Z
+      volumeMounts:
+      - name: "{{ .Values.pvc.name }}"
+        mountPath: /assets
       command: [sh]
       env:
       - name: MINIO_SERVER_ACCESS_KEY

examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/values.yaml

@@ -5,6 +5,11 @@ workflow:
   useNameAsGenerateName: false
   # Namespace where to create the workflow and all resources in batch job
   namespace: default
+pvc:
+  # Name of the persistent volume claim to be created
+  name: seldon-pvc
+  # Size of the storage volume to be created for the batch job 
+  storage: 2Mi
 # Seldon deployment to be created for batch processing
 seldonDeployment:
   # Name to use for the seldon deployment which by default appends generated workflow ID
@@ -25,6 +30,8 @@ batchWorker:
   workers: 100
   # Number of times client will try to send a request if it fails
   retries: 3
+  # Waiting time before checks for deployment to ensure kubernetes cluster registers create
+  waitTime: 5
 minio:
   # The location of the minio endpoint 
   endpoint: http://minio.default.svc.cluster.local:9000