From 030f3bdf19a8a0a92763089b03b35432d8125ee9 Mon Sep 17 00:00:00 2001 From: cliveseldon Date: Tue, 29 Nov 2022 20:29:13 +0000 Subject: [PATCH] Fix http calls mirror pipelines not working (#660) * Fix http calls mirror pipelines not working * retest --- samples/local-experiments.ipynb | 187 +++++++++++++++++---- samples/local-experiments.md | 77 ++++++++- scheduler/pkg/kafka/pipeline/httpserver.go | 7 +- 3 files changed, 227 insertions(+), 44 deletions(-) diff --git a/samples/local-experiments.ipynb b/samples/local-experiments.ipynb index 7235c27d89..3b09fd1446 100644 --- a/samples/local-experiments.ipynb +++ b/samples/local-experiments.ipynb @@ -1187,7 +1187,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 1, "id": "20f19234", "metadata": { "scrolled": false @@ -1214,7 +1214,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 2, "id": "40ba5a74", "metadata": {}, "outputs": [ @@ -1247,7 +1247,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 3, "id": "b4ec8d19", "metadata": {}, "outputs": [ @@ -1275,7 +1275,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 4, "id": "115699a4", "metadata": {}, "outputs": [ @@ -1303,7 +1303,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 5, "id": "632b4a58", "metadata": {}, "outputs": [ @@ -1341,7 +1341,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 6, "id": "2466a9b3", "metadata": {}, "outputs": [ @@ -1367,7 +1367,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 7, "id": "0ac0054e", "metadata": {}, "outputs": [ @@ -1400,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 8, "id": "81c6023d", "metadata": {}, "outputs": [ @@ -1427,7 +1427,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 9, "id": "35a0f1f8", "metadata": {}, "outputs": [ @@ -1453,7 +1453,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 10, "id": "e777b0c2", "metadata": {}, "outputs": [ @@ -1479,7 +1479,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 11, "id": "44085e38", "metadata": {}, "outputs": [ @@ -1507,7 +1507,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 12, "id": "01bf3aa8", "metadata": { "scrolled": false @@ -1535,7 +1535,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 13, "id": "be5c5426", "metadata": {}, "outputs": [ @@ -1561,7 +1561,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 14, "id": "54ac44c1", "metadata": {}, "outputs": [ @@ -1581,7 +1581,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 15, "id": "78debf8f", "metadata": {}, "outputs": [ @@ -1601,7 +1601,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 16, "id": "c04346fa", "metadata": {}, "outputs": [ @@ -1628,7 +1628,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 17, "id": "0aa43131", "metadata": {}, "outputs": [ @@ -1655,7 +1655,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 18, "id": "8bb044d7", "metadata": {}, "outputs": [ @@ -1675,7 +1675,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 19, "id": "cd5c869f", "metadata": {}, "outputs": [ @@ -1683,8 +1683,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"pipelineName\":\"pipeline-add10\", \"versions\":[{\"pipeline\":{\"name\":\"pipeline-add10\", \"uid\":\"cdnnbmdle28c73ckejmg\", \"version\":1, \"steps\":[{\"name\":\"add10\"}], \"output\":{\"steps\":[\"add10.outputs\"]}, \"kubernetesMeta\":{}}, \"state\":{\"pipelineVersion\":1, \"status\":\"PipelineReady\", \"reason\":\"created pipeline\", \"lastChangeTimestamp\":\"2022-11-12T10:30:49.807877487Z\"}}]}\n", - "{\"pipelineName\":\"pipeline-mul10\", \"versions\":[{\"pipeline\":{\"name\":\"pipeline-mul10\", \"uid\":\"cdnnbmdle28c73ckejn0\", \"version\":1, \"steps\":[{\"name\":\"mul10\"}], \"output\":{\"steps\":[\"mul10.outputs\"]}, \"kubernetesMeta\":{}}, \"state\":{\"pipelineVersion\":1, \"status\":\"PipelineReady\", \"reason\":\"created pipeline\", \"lastChangeTimestamp\":\"2022-11-12T10:30:50.009058654Z\"}}]}\n" + "{\"pipelineName\":\"pipeline-add10\",\"versions\":[{\"pipeline\":{\"name\":\"pipeline-add10\",\"uid\":\"ce35uho4mlvc73d8elh0\",\"version\":1,\"steps\":[{\"name\":\"add10\"}],\"output\":{\"steps\":[\"add10.outputs\"]},\"kubernetesMeta\":{}},\"state\":{\"pipelineVersion\":1,\"status\":\"PipelineReady\",\"reason\":\"created pipeline\",\"lastChangeTimestamp\":\"2022-11-29T19:36:39.645836943Z\",\"modelsReady\":true}}]}\n", + "{\"pipelineName\":\"pipeline-mul10\",\"versions\":[{\"pipeline\":{\"name\":\"pipeline-mul10\",\"uid\":\"ce35uho4mlvc73d8elhg\",\"version\":1,\"steps\":[{\"name\":\"mul10\"}],\"output\":{\"steps\":[\"mul10.outputs\"]},\"kubernetesMeta\":{}},\"state\":{\"pipelineVersion\":1,\"status\":\"PipelineReady\",\"reason\":\"created pipeline\",\"lastChangeTimestamp\":\"2022-11-29T19:36:39.824124774Z\",\"modelsReady\":true}}]}\n" ] } ], @@ -1695,7 +1695,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 20, "id": "9c2b666e", "metadata": {}, "outputs": [ @@ -1703,7 +1703,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"outputs\":[{\"name\":\"OUTPUT\", \"datatype\":\"FP32\", \"shape\":[\"4\"], \"contents\":{\"fp32Contents\":[11, 12, 13, 14]}}]}\r\n" + "{\"outputs\":[{\"name\":\"OUTPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[11,12,13,14]}}]}\r\n" ] } ], @@ -1714,7 +1714,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 21, "id": "ef844ae7", "metadata": {}, "outputs": [ @@ -1722,7 +1722,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"outputs\":[{\"name\":\"OUTPUT\", \"datatype\":\"FP32\", \"shape\":[\"4\"], \"contents\":{\"fp32Contents\":[10, 20, 30, 40]}}]}\r\n" + "{\"outputs\":[{\"name\":\"OUTPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[10,20,30,40]}}]}\r\n" ] } ], @@ -1733,7 +1733,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 22, "id": "3a9f756f", "metadata": {}, "outputs": [ @@ -1764,7 +1764,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 23, "id": "3777e310", "metadata": {}, "outputs": [ @@ -1782,7 +1782,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 24, "id": "63d087f3", "metadata": {}, "outputs": [ @@ -1807,7 +1807,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 25, "id": "319ef08f", "metadata": {}, "outputs": [ @@ -1815,12 +1815,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Success: map[:add10_1::50 :pipeline-add10.pipeline::50]\r\n" + "{\"outputs\":[{\"name\":\"OUTPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[11,12,13,14]}}]}\r\n" ] } ], "source": [ - "!seldon pipeline infer pipeline-add10 -i 50 --inference-mode grpc \\\n", + "!seldon pipeline infer pipeline-add10 -i 1 --inference-mode grpc \\\n", " '{\"model_name\":\"add10\",\"inputs\":[{\"name\":\"INPUT\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[4]}]}' " ] }, @@ -1834,7 +1834,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 26, "id": "086359f1", "metadata": {}, "outputs": [ @@ -1842,7 +1842,87 @@ "name": "stdout", "output_type": "stream", "text": [ - "seldon_model_infer_total{code=\"OK\",method_type=\"grpc\",model=\"mul10\",model_internal=\"mul10_1\",server=\"triton\",server_replica=\"0\"} 51\r\n" + "seldon_model_infer_total{code=\"OK\",method_type=\"grpc\",model=\"mul10\",model_internal=\"mul10_1\",server=\"triton\",server_replica=\"0\"} 2\r\n" + ] + } + ], + "source": [ + "!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep mul10_1" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "25d03bc0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon_model_infer_total{code=\"OK\",method_type=\"grpc\",model=\"add10\",model_internal=\"add10_1\",server=\"triton\",server_replica=\"0\"} 2\r\n" + ] + } + ], + "source": [ + "!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep add10_1" + ] + }, + { + "cell_type": "markdown", + "id": "4bacaccb", + "metadata": {}, + "source": [ + "Let's do an http call and check agaib the two models" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "94d784bb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\r\n", + "\t\"model_name\": \"\",\r\n", + "\t\"outputs\": [\r\n", + "\t\t{\r\n", + "\t\t\t\"data\": [\r\n", + "\t\t\t\t11,\r\n", + "\t\t\t\t12,\r\n", + "\t\t\t\t13,\r\n", + "\t\t\t\t14\r\n", + "\t\t\t],\r\n", + "\t\t\t\"name\": \"OUTPUT\",\r\n", + "\t\t\t\"shape\": [\r\n", + "\t\t\t\t4\r\n", + "\t\t\t],\r\n", + "\t\t\t\"datatype\": \"FP32\"\r\n", + "\t\t}\r\n", + "\t]\r\n", + "}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline infer pipeline-add10 -i 1 \\\n", + " '{\"model_name\":\"add10\",\"inputs\":[{\"name\":\"INPUT\",\"data\":[1,2,3,4],\"datatype\":\"FP32\",\"shape\":[4]}]}' " + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "509efbcb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon_model_infer_total{code=\"OK\",method_type=\"grpc\",model=\"mul10\",model_internal=\"mul10_1\",server=\"triton\",server_replica=\"0\"} 3\r\n" ] } ], @@ -1852,7 +1932,46 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 30, + "id": "d4d86a74", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon_model_infer_total{code=\"OK\",method_type=\"grpc\",model=\"add10\",model_internal=\"add10_1\",server=\"triton\",server_replica=\"0\"} 3\r\n" + ] + } + ], + "source": [ + "!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep add10_1" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "eb8be18f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon.default.model.mul10.inputs\tce35uksj0jvc73bqr0pg\t{\"inputs\":[{\"name\":\"INPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[1,2,3,4]}}]}\r\n", + "seldon.default.model.mul10.outputs\tce35uksj0jvc73bqr0pg\t{\"modelName\":\"mul10_1\",\"modelVersion\":\"1\",\"outputs\":[{\"name\":\"OUTPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[10,20,30,40]}}]}\r\n", + "seldon.default.pipeline.pipeline-mul10.inputs\tce35uksj0jvc73bqr0pg\t{\"inputs\":[{\"name\":\"INPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[1,2,3,4]}}]}\r\n", + "seldon.default.pipeline.pipeline-mul10.outputs\tce35uksj0jvc73bqr0pg\t{\"outputs\":[{\"name\":\"OUTPUT\",\"datatype\":\"FP32\",\"shape\":[\"4\"],\"contents\":{\"fp32Contents\":[10,20,30,40]}}]}\r\n" + ] + } + ], + "source": [ + "!seldon pipeline inspect pipeline-mul10" + ] + }, + { + "cell_type": "code", + "execution_count": 32, "id": "c84bad04", "metadata": {}, "outputs": [ diff --git a/samples/local-experiments.md b/samples/local-experiments.md index 39ba423b44..719459aae7 100644 --- a/samples/local-experiments.md +++ b/samples/local-experiments.md @@ -910,8 +910,8 @@ Unload both models. !seldon pipeline status pipeline-mul10 -w PipelineReady ``` - {"pipelineName":"pipeline-add10", "versions":[{"pipeline":{"name":"pipeline-add10", "uid":"cdnnbmdle28c73ckejmg", "version":1, "steps":[{"name":"add10"}], "output":{"steps":["add10.outputs"]}, "kubernetesMeta":{}}, "state":{"pipelineVersion":1, "status":"PipelineReady", "reason":"created pipeline", "lastChangeTimestamp":"2022-11-12T10:30:49.807877487Z"}}]} - {"pipelineName":"pipeline-mul10", "versions":[{"pipeline":{"name":"pipeline-mul10", "uid":"cdnnbmdle28c73ckejn0", "version":1, "steps":[{"name":"mul10"}], "output":{"steps":["mul10.outputs"]}, "kubernetesMeta":{}}, "state":{"pipelineVersion":1, "status":"PipelineReady", "reason":"created pipeline", "lastChangeTimestamp":"2022-11-12T10:30:50.009058654Z"}}]} + {"pipelineName":"pipeline-add10","versions":[{"pipeline":{"name":"pipeline-add10","uid":"ce35uho4mlvc73d8elh0","version":1,"steps":[{"name":"add10"}],"output":{"steps":["add10.outputs"]},"kubernetesMeta":{}},"state":{"pipelineVersion":1,"status":"PipelineReady","reason":"created pipeline","lastChangeTimestamp":"2022-11-29T19:36:39.645836943Z","modelsReady":true}}]} + {"pipelineName":"pipeline-mul10","versions":[{"pipeline":{"name":"pipeline-mul10","uid":"ce35uho4mlvc73d8elhg","version":1,"steps":[{"name":"mul10"}],"output":{"steps":["mul10.outputs"]},"kubernetesMeta":{}},"state":{"pipelineVersion":1,"status":"PipelineReady","reason":"created pipeline","lastChangeTimestamp":"2022-11-29T19:36:39.824124774Z","modelsReady":true}}]} @@ -920,7 +920,7 @@ Unload both models. '{"model_name":"add10","inputs":[{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' ``` - {"outputs":[{"name":"OUTPUT", "datatype":"FP32", "shape":["4"], "contents":{"fp32Contents":[11, 12, 13, 14]}}]} + {"outputs":[{"name":"OUTPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[11,12,13,14]}}]} @@ -929,7 +929,7 @@ Unload both models. '{"model_name":"add10","inputs":[{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' ``` - {"outputs":[{"name":"OUTPUT", "datatype":"FP32", "shape":["4"], "contents":{"fp32Contents":[10, 20, 30, 40]}}]} + {"outputs":[{"name":"OUTPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[10,20,30,40]}}]} @@ -978,11 +978,11 @@ Unload both models. ```python -!seldon pipeline infer pipeline-add10 -i 50 --inference-mode grpc \ +!seldon pipeline infer pipeline-add10 -i 1 --inference-mode grpc \ '{"model_name":"add10","inputs":[{"name":"INPUT","contents":{"fp32_contents":[1,2,3,4]},"datatype":"FP32","shape":[4]}]}' ``` - Success: map[:add10_1::50 :pipeline-add10.pipeline::50] + {"outputs":[{"name":"OUTPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[11,12,13,14]}}]} Let's check that the mul10 model was called. @@ -992,7 +992,70 @@ Let's check that the mul10 model was called. !curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep mul10_1 ``` - seldon_model_infer_total{code="OK",method_type="grpc",model="mul10",model_internal="mul10_1",server="triton",server_replica="0"} 51 + seldon_model_infer_total{code="OK",method_type="grpc",model="mul10",model_internal="mul10_1",server="triton",server_replica="0"} 2 + + + +```python +!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep add10_1 +``` + + seldon_model_infer_total{code="OK",method_type="grpc",model="add10",model_internal="add10_1",server="triton",server_replica="0"} 2 + + +Let's do an http call and check agaib the two models + + +```python +!seldon pipeline infer pipeline-add10 -i 1 \ + '{"model_name":"add10","inputs":[{"name":"INPUT","data":[1,2,3,4],"datatype":"FP32","shape":[4]}]}' +``` + + { + "model_name": "", + "outputs": [ + { + "data": [ + 11, + 12, + 13, + 14 + ], + "name": "OUTPUT", + "shape": [ + 4 + ], + "datatype": "FP32" + } + ] + } + + + +```python +!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep mul10_1 +``` + + seldon_model_infer_total{code="OK",method_type="grpc",model="mul10",model_internal="mul10_1",server="triton",server_replica="0"} 3 + + + +```python +!curl -s 0.0.0:9007/metrics | grep seldon_model_infer_total | grep add10_1 +``` + + seldon_model_infer_total{code="OK",method_type="grpc",model="add10",model_internal="add10_1",server="triton",server_replica="0"} 3 + + + +```python +!seldon pipeline inspect pipeline-mul10 +``` + + seldon.default.model.mul10.inputs ce35uksj0jvc73bqr0pg {"inputs":[{"name":"INPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[1,2,3,4]}}]} + seldon.default.model.mul10.outputs ce35uksj0jvc73bqr0pg {"modelName":"mul10_1","modelVersion":"1","outputs":[{"name":"OUTPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[10,20,30,40]}}]} + seldon.default.pipeline.pipeline-mul10.inputs ce35uksj0jvc73bqr0pg {"inputs":[{"name":"INPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[1,2,3,4]}}]} + seldon.default.pipeline.pipeline-mul10.outputs ce35uksj0jvc73bqr0pg {"outputs":[{"name":"OUTPUT","datatype":"FP32","shape":["4"],"contents":{"fp32Contents":[10,20,30,40]}}]} diff --git a/scheduler/pkg/kafka/pipeline/httpserver.go b/scheduler/pkg/kafka/pipeline/httpserver.go index d5ef461551..b9f81a57bb 100644 --- a/scheduler/pkg/kafka/pipeline/httpserver.go +++ b/scheduler/pkg/kafka/pipeline/httpserver.go @@ -196,10 +196,11 @@ func (g *GatewayHttpServer) infer(w http.ResponseWriter, req *http.Request, reso func getResourceFromHeaders(req *http.Request, logger log.FieldLogger) (string, bool, error) { modelHeader := req.Header.Get(resources.SeldonModelHeader) - modelInternalHeader := req.Header.Get(resources.SeldonInternalModelHeader) + // may have multiple header values due to shadow/mirror processing + modelInternalHeader := req.Header.Values(resources.SeldonInternalModelHeader) logger.Debugf("Seldon model header %s and seldon internal model header %s", modelHeader, modelInternalHeader) - if modelInternalHeader != "" { - return createResourceNameFromHeader(modelInternalHeader) + if len(modelInternalHeader) > 0 { + return createResourceNameFromHeader(modelInternalHeader[len(modelInternalHeader)-1]) // get last header if multiple } else { return createResourceNameFromHeader(modelHeader) }