diff --git a/cli/cmd/get.go b/cli/cmd/get.go index e2bf145257..5b33f09308 100644 --- a/cli/cmd/get.go +++ b/cli/cmd/get.go @@ -272,7 +272,7 @@ func describeAPI(name string, resourcesRes *schema.GetResourcesResponse, flagVer out += fmt.Sprintf("\n%s curl %s?debug=true -X POST -H \"Content-Type: application/json\" -d @sample.json", console.Bold("curl:"), apiEndpoint) - if api.TensorFlow != nil || api.ONNX != nil { + if api.Predictor.Type == userconfig.TensorFlowPredictorType || api.Predictor.Type == userconfig.ONNXPredictorType { out += "\n\n" + describeModelInput(groupStatus, apiEndpoint) } diff --git a/docs/deployments/onnx.md b/docs/deployments/onnx.md index f855dfa607..32a69d7e2f 100644 --- a/docs/deployments/onnx.md +++ b/docs/deployments/onnx.md @@ -10,9 +10,10 @@ You can deploy ONNX models as web services by defining a class that implements C - kind: api name: # API name (required) endpoint: # the endpoint for the API (default: //) - onnx: + predictor: + type: onnx + path: # path to a python file with an ONNXPredictor class definition, relative to the Cortex root (required) model: # S3 path to an exported model (e.g. s3://my-bucket/exported_model.onnx) (required) - predictor: # path to a python file with an ONNXPredictor class definition, relative to the Cortex root (required) config: # dictionary passed to the constructor of a Predictor (optional) python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) env: # dictionary of environment variables @@ -36,9 +37,10 @@ See [packaging ONNX models](../packaging-models/onnx.md) for information about e ```yaml - kind: api name: my-api - onnx: + predictor: + type: onnx + path: predictor.py model: s3://my-bucket/my-model.onnx - predictor: predictor.py compute: gpu: 1 ``` diff --git a/docs/deployments/prediction-monitoring.md b/docs/deployments/prediction-monitoring.md index 9ae8470570..d1a20c4380 100644 --- a/docs/deployments/prediction-monitoring.md +++ b/docs/deployments/prediction-monitoring.md @@ -21,8 +21,9 @@ For classification models, the tracker should be configured with `model_type: cl ```yaml - kind: api name: iris - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py tracker: model_type: classification ``` diff --git a/docs/deployments/python.md b/docs/deployments/python.md index d99841eec2..17509209bb 100644 --- a/docs/deployments/python.md +++ b/docs/deployments/python.md @@ -15,8 +15,9 @@ In addition to supporting Python models via the Python Predictor interface, Cort - kind: api name: # API name (required) endpoint: # the endpoint for the API (default: //) - python: - predictor: # path to a python file with a PythonPredictor class definition, relative to the Cortex root (required) + predictor: + type: python + path: # path to a python file with a PythonPredictor class definition, relative to the Cortex root (required) config: # dictionary passed to the constructor of a Predictor (optional) python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) env: # dictionary of environment variables diff --git a/docs/deployments/tensorflow.md b/docs/deployments/tensorflow.md index 4e0cea3b18..95af9cd7d0 100644 --- a/docs/deployments/tensorflow.md +++ b/docs/deployments/tensorflow.md @@ -10,9 +10,10 @@ You can deploy TensorFlow models as web services by defining a class that implem - kind: api name: # API name (required) endpoint: # the endpoint for the API (default: //) - tensorflow: + predictor: + type: tensorflow + path: # path to a python file with a TensorFlowPredictor class definition, relative to the Cortex root (required) model: # S3 path to an exported model (e.g. s3://my-bucket/exported_model) (required) - predictor: # path to a python file with a TensorFlowPredictor class definition, relative to the Cortex root (required) signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) config: # dictionary that can be used to configure custom values (optional) python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) @@ -37,9 +38,10 @@ See [packaging TensorFlow models](../packaging-models/tensorflow.md) for how to ```yaml - kind: api name: my-api - tensorflow: + predictor: + type: tensorflow + path: predictor.py model: s3://my-bucket/my-model - predictor: predictor.py compute: gpu: 1 ``` diff --git a/docs/packaging-models/onnx.md b/docs/packaging-models/onnx.md index 98d3736d02..705b9d7b10 100644 --- a/docs/packaging-models/onnx.md +++ b/docs/packaging-models/onnx.md @@ -31,7 +31,8 @@ Reference your model in an `api`: ```yaml - kind: api name: my-api - onnx: + predictor: + type: onnx model: s3://my-bucket/model.onnx predictor: predictor.py ``` diff --git a/docs/packaging-models/tensorflow.md b/docs/packaging-models/tensorflow.md index b66300ab7e..7109ef9012 100644 --- a/docs/packaging-models/tensorflow.md +++ b/docs/packaging-models/tensorflow.md @@ -38,7 +38,8 @@ Reference your model in an `api`: ```yaml - kind: api name: my-api - tensorflow: + predictor: + type: tensorflow model: s3://my-bucket/bert predictor: predictor.py ``` @@ -56,7 +57,8 @@ Reference the zipped model in an `api`: ```yaml - kind: api name: my-api - tensorflow: + predictor: + type: tensorflow model: s3://my-bucket/bert.zip predictor: predictor.py ``` diff --git a/examples/pytorch/answer-generator/cortex.yaml b/examples/pytorch/answer-generator/cortex.yaml index 40449d2ec4..c8a3153636 100644 --- a/examples/pytorch/answer-generator/cortex.yaml +++ b/examples/pytorch/answer-generator/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: generator - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: device: cuda # use "cpu" to run on CPUs compute: diff --git a/examples/pytorch/image-classifier/cortex.yaml b/examples/pytorch/image-classifier/cortex.yaml index 308f90f010..bb044dca0d 100644 --- a/examples/pytorch/image-classifier/cortex.yaml +++ b/examples/pytorch/image-classifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py compute: cpu: 1 gpu: 1 diff --git a/examples/pytorch/iris-classifier/cortex.yaml b/examples/pytorch/iris-classifier/cortex.yaml index 4bd8973150..e75ef31c4c 100644 --- a/examples/pytorch/iris-classifier/cortex.yaml +++ b/examples/pytorch/iris-classifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: model: s3://cortex-examples/pytorch/iris-classifier/weights.pth tracker: diff --git a/examples/pytorch/language-identifier/cortex.yaml b/examples/pytorch/language-identifier/cortex.yaml index c36d60705e..f3e238f77c 100644 --- a/examples/pytorch/language-identifier/cortex.yaml +++ b/examples/pytorch/language-identifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: identifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py tracker: model_type: classification compute: diff --git a/examples/pytorch/reading-comprehender/cortex.yaml b/examples/pytorch/reading-comprehender/cortex.yaml index ecdb7578cb..52a35ae19d 100644 --- a/examples/pytorch/reading-comprehender/cortex.yaml +++ b/examples/pytorch/reading-comprehender/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: comprehender - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py compute: cpu: 1 gpu: 1 diff --git a/examples/pytorch/search-completer/cortex.yaml b/examples/pytorch/search-completer/cortex.yaml index bf8ad11c69..4b3027b918 100644 --- a/examples/pytorch/search-completer/cortex.yaml +++ b/examples/pytorch/search-completer/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: completer - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py compute: cpu: 1 gpu: 1 diff --git a/examples/pytorch/text-generator/cortex.yaml b/examples/pytorch/text-generator/cortex.yaml index 6e6d576d1f..80ec350a9a 100644 --- a/examples/pytorch/text-generator/cortex.yaml +++ b/examples/pytorch/text-generator/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: generator - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: num_words: 50 device: cuda # use "cpu" to run on CPUs diff --git a/examples/pytorch/text-summarizer/cortex.yaml b/examples/pytorch/text-summarizer/cortex.yaml index ddfd7496f8..b6525efc48 100644 --- a/examples/pytorch/text-summarizer/cortex.yaml +++ b/examples/pytorch/text-summarizer/cortex.yaml @@ -5,7 +5,8 @@ - kind: api name: summarizer - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py compute: mem: 4G diff --git a/examples/sklearn/iris-classifier/README.md b/examples/sklearn/iris-classifier/README.md index 6d86311e18..5be80f2d5f 100644 --- a/examples/sklearn/iris-classifier/README.md +++ b/examples/sklearn/iris-classifier/README.md @@ -112,8 +112,9 @@ Create a `cortex.yaml` file and add the configuration below. A `deployment` spec - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -178,8 +179,9 @@ Add a `tracker` to your `cortex.yaml` and specify that this is a classification - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -221,8 +223,9 @@ This model is fairly small but larger models may require more compute resources. - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -267,8 +270,9 @@ If you trained another model and want to A/B test it with your previous model, s - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -280,8 +284,9 @@ If you trained another model and want to A/B test it with your previous model, s - kind: api name: another-classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/another-model.pkl @@ -353,8 +358,9 @@ Next, add the `api` to `cortex.yaml`: - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -366,8 +372,9 @@ Next, add the `api` to `cortex.yaml`: - kind: api name: another-classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/another-model.pkl @@ -380,8 +387,9 @@ Next, add the `api` to `cortex.yaml`: - kind: api name: batch-classifier - python: - predictor: batch-predictor.py + predictor: + type: python + path: batch-predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl diff --git a/examples/sklearn/iris-classifier/cortex.yaml b/examples/sklearn/iris-classifier/cortex.yaml index c543a4151a..fe74e2928c 100644 --- a/examples/sklearn/iris-classifier/cortex.yaml +++ b/examples/sklearn/iris-classifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl @@ -18,8 +19,9 @@ - kind: api name: another-classifier - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/another-model.pkl @@ -31,8 +33,9 @@ - kind: api name: batch-classifier - python: - predictor: batch-predictor.py + predictor: + type: python + path: batch-predictor.py config: bucket: cortex-examples key: sklearn/iris-classifier/model.pkl diff --git a/examples/sklearn/mpg-estimator/cortex.yaml b/examples/sklearn/mpg-estimator/cortex.yaml index afc76a427f..0a153ab90a 100644 --- a/examples/sklearn/mpg-estimator/cortex.yaml +++ b/examples/sklearn/mpg-estimator/cortex.yaml @@ -5,7 +5,8 @@ - kind: api name: estimator - python: - predictor: predictor.py + predictor: + type: python + path: predictor.py config: model: s3://cortex-examples/sklearn/mpg-estimator/linreg diff --git a/examples/tensorflow/image-classifier/cortex.yaml b/examples/tensorflow/image-classifier/cortex.yaml index 8f5f418952..5010f63c9a 100644 --- a/examples/tensorflow/image-classifier/cortex.yaml +++ b/examples/tensorflow/image-classifier/cortex.yaml @@ -5,9 +5,10 @@ - kind: api name: classifier - tensorflow: + predictor: + type: tensorflow + path: predictor.py model: s3://cortex-examples/tensorflow/image-classifier/inception - predictor: predictor.py tracker: model_type: classification compute: diff --git a/examples/tensorflow/iris-classifier/cortex.yaml b/examples/tensorflow/iris-classifier/cortex.yaml index 2d17651be5..5bc19a9938 100644 --- a/examples/tensorflow/iris-classifier/cortex.yaml +++ b/examples/tensorflow/iris-classifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: classifier - tensorflow: + predictor: + type: tensorflow + path: predictor.py model: s3://cortex-examples/tensorflow/iris-classifier/nn - predictor: predictor.py tracker: model_type: classification diff --git a/examples/tensorflow/sentiment-analyzer/cortex.yaml b/examples/tensorflow/sentiment-analyzer/cortex.yaml index 972c522b52..c48fd10880 100644 --- a/examples/tensorflow/sentiment-analyzer/cortex.yaml +++ b/examples/tensorflow/sentiment-analyzer/cortex.yaml @@ -5,9 +5,10 @@ - kind: api name: analyzer - tensorflow: + predictor: + type: tensorflow + path: predictor.py model: s3://cortex-examples/tensorflow/sentiment-analyzer/bert - predictor: predictor.py tracker: model_type: classification compute: diff --git a/examples/tensorflow/text-generator/cortex.yaml b/examples/tensorflow/text-generator/cortex.yaml index 12d1518e77..66dd063bc6 100644 --- a/examples/tensorflow/text-generator/cortex.yaml +++ b/examples/tensorflow/text-generator/cortex.yaml @@ -5,9 +5,10 @@ - kind: api name: generator - tensorflow: + predictor: + type: tensorflow + path: predictor.py model: s3://cortex-examples/tensorflow/text-generator/gpt-2/124M - predictor: predictor.py compute: cpu: 1 gpu: 1 diff --git a/examples/xgboost/iris-classifier/cortex.yaml b/examples/xgboost/iris-classifier/cortex.yaml index 29e7b8329d..736c86147b 100644 --- a/examples/xgboost/iris-classifier/cortex.yaml +++ b/examples/xgboost/iris-classifier/cortex.yaml @@ -5,8 +5,9 @@ - kind: api name: classifier - onnx: + predictor: + type: onnx + path: predictor.py model: s3://cortex-examples/xgboost/iris-classifier/gbtree.onnx - predictor: predictor.py tracker: model_type: classification diff --git a/pkg/operator/api/userconfig/apis.go b/pkg/operator/api/userconfig/apis.go index 50b29b0e6c..d791388eed 100644 --- a/pkg/operator/api/userconfig/apis.go +++ b/pkg/operator/api/userconfig/apis.go @@ -37,12 +37,10 @@ type APIs []*API type API struct { ResourceFields - Endpoint *string `json:"endpoint" yaml:"endpoint"` - TensorFlow *TensorFlow `json:"tensorflow" yaml:"tensorflow"` - ONNX *ONNX `json:"onnx" yaml:"onnx"` - Python *Python `json:"python" yaml:"python"` - Tracker *Tracker `json:"tracker" yaml:"tracker"` - Compute *APICompute `json:"compute" yaml:"compute"` + Endpoint *string `json:"endpoint" yaml:"endpoint"` + Predictor *Predictor `json:"predictor" yaml:"predictor"` + Tracker *Tracker `json:"tracker" yaml:"tracker"` + Compute *APICompute `json:"compute" yaml:"compute"` } type Tracker struct { @@ -50,57 +48,75 @@ type Tracker struct { ModelType ModelType `json:"model_type" yaml:"model_type"` } -var configValidation = &cr.StructFieldValidation{ - StructField: "Config", - InterfaceMapValidation: &cr.InterfaceMapValidation{ - StringKeysOnly: true, - AllowEmpty: true, - Default: map[string]interface{}{}, - }, -} - -var envValidation = &cr.StructFieldValidation{ - StructField: "Env", - StringMapValidation: &cr.StringMapValidation{ - Default: map[string]string{}, - AllowEmpty: true, - }, -} - -func ensurePythonPathSuffix(path string) (string, error) { - return s.EnsureSuffix(path, "/"), nil -} - -var pythonPathValidation = &cr.StructFieldValidation{ - StructField: "PythonPath", - StringPtrValidation: &cr.StringPtrValidation{ - AllowEmpty: true, - Validator: ensurePythonPathSuffix, - }, -} - -type TensorFlow struct { - Model string `json:"model" yaml:"model"` - Predictor string `json:"predictor" yaml:"predictor"` - SignatureKey *string `json:"signature_key" yaml:"signature_key"` +type Predictor struct { + Type PredictorType `json:"type" yaml:"type"` + Path string `json:"path" yaml:"path"` + Model *string `json:"model" yaml:"model"` PythonPath *string `json:"python_path" yaml:"python_path"` Config map[string]interface{} `json:"config" yaml:"config"` Env map[string]string `json:"env" yaml:"env"` + SignatureKey *string `json:"signature_key" yaml:"signature_key"` } -type ONNX struct { - Model string `json:"model" yaml:"model"` - Predictor string `json:"predictor" yaml:"predictor"` - PythonPath *string `json:"python_path" yaml:"python_path"` - Config map[string]interface{} `json:"config" yaml:"config"` - Env map[string]string `json:"env" yaml:"env"` +var predictorValidation = &cr.StructFieldValidation{ + StructField: "Predictor", + StructValidation: &cr.StructValidation{ + Required: true, + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Type", + StringValidation: &cr.StringValidation{ + Required: true, + AllowedValues: PredictorTypeStrings(), + }, + Parser: func(str string) (interface{}, error) { + return PredictorTypeFromString(str), nil + }, + }, + { + StructField: "Path", + StringValidation: &cr.StringValidation{ + Required: true, + }, + }, + { + StructField: "Model", + StringPtrValidation: &cr.StringPtrValidation{ + Validator: cr.S3PathValidator(), + }, + }, + { + StructField: "PythonPath", + StringPtrValidation: &cr.StringPtrValidation{ + AllowEmpty: true, + Validator: ensurePythonPathSuffix, + }, + }, + { + StructField: "Config", + InterfaceMapValidation: &cr.InterfaceMapValidation{ + StringKeysOnly: true, + AllowEmpty: true, + Default: map[string]interface{}{}, + }, + }, + { + StructField: "Env", + StringMapValidation: &cr.StringMapValidation{ + Default: map[string]string{}, + AllowEmpty: true, + }, + }, + { + StructField: "SignatureKey", + StringPtrValidation: &cr.StringPtrValidation{}, + }, + }, + }, } -type Python struct { - Predictor string `json:"predictor" yaml:"predictor"` - PythonPath *string `json:"python_path" yaml:"python_path"` - Config map[string]interface{} `json:"config" yaml:"config"` - Env map[string]string `json:"env" yaml:"env"` +func ensurePythonPathSuffix(path string) (string, error) { + return s.EnsureSuffix(path, "/"), nil } var apiValidation = &cr.StructValidation{ @@ -141,75 +157,7 @@ var apiValidation = &cr.StructValidation{ }, }, }, - { - StructField: "TensorFlow", - StructValidation: &cr.StructValidation{ - DefaultNil: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Model", - StringValidation: &cr.StringValidation{ - Required: true, - Validator: cr.S3PathValidator(), - }, - }, - { - StructField: "Predictor", - StringValidation: &cr.StringValidation{ - Required: true, - }, - }, - { - StructField: "SignatureKey", - StringPtrValidation: &cr.StringPtrValidation{}, - }, - pythonPathValidation, - configValidation, - envValidation, - }, - }, - }, - { - StructField: "ONNX", - StructValidation: &cr.StructValidation{ - DefaultNil: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Model", - StringValidation: &cr.StringValidation{ - Required: true, - Validator: cr.S3PathValidator(), - }, - }, - { - StructField: "Predictor", - StringValidation: &cr.StringValidation{ - Required: true, - }, - }, - pythonPathValidation, - configValidation, - envValidation, - }, - }, - }, - { - StructField: "Python", - StructValidation: &cr.StructValidation{ - DefaultNil: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Predictor", - StringValidation: &cr.StringValidation{ - Required: true, - }, - }, - pythonPathValidation, - configValidation, - envValidation, - }, - }, - }, + predictorValidation, apiComputeFieldValidation, typeFieldValidation, }, @@ -297,18 +245,9 @@ func (api *API) UserConfigStr() string { sb.WriteString(api.ResourceFields.UserConfigStr()) sb.WriteString(fmt.Sprintf("%s: %s\n", EndpointKey, *api.Endpoint)) - if api.TensorFlow != nil { - sb.WriteString(fmt.Sprintf("%s:\n", TensorFlowKey)) - sb.WriteString(s.Indent(api.TensorFlow.UserConfigStr(), " ")) - } - if api.ONNX != nil { - sb.WriteString(fmt.Sprintf("%s:\n", ONNXKey)) - sb.WriteString(s.Indent(api.ONNX.UserConfigStr(), " ")) - } - if api.Python != nil { - sb.WriteString(fmt.Sprintf("%s:\n", PythonKey)) - sb.WriteString(s.Indent(api.Python.UserConfigStr(), " ")) - } + sb.WriteString(fmt.Sprintf("%s:\n", PredictorKey)) + sb.WriteString(s.Indent(api.Predictor.UserConfigStr(), " ")) + if api.Compute != nil { sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) sb.WriteString(s.Indent(api.Compute.UserConfigStr(), " ")) @@ -357,125 +296,119 @@ func (apis APIs) Validate(deploymentName string, projectFileMap map[string][]byt return nil } -func (tf *TensorFlow) Validate(projectFileMap map[string][]byte) error { - awsClient, err := aws.NewFromS3Path(tf.Model, false) - if err != nil { - return err - } - if strings.HasSuffix(tf.Model, ".zip") { - if ok, err := awsClient.IsS3PathFile(tf.Model); err != nil || !ok { - return errors.Wrap(ErrorExternalNotFound(tf.Model), TensorFlowKey, ModelKey) - } - } else { - path, err := GetTFServingExportFromS3Path(tf.Model, awsClient) - if path == "" || err != nil { - return errors.Wrap(ErrorInvalidTensorFlowDir(tf.Model), TensorFlowKey, ModelKey) - } - tf.Model = path - } - if _, ok := projectFileMap[tf.Predictor]; !ok { - return errors.Wrap(ErrorImplDoesNotExist(tf.Predictor), TensorFlowKey, PredictorKey) - } - if tf.PythonPath != nil { - if err := ValidatePythonPath(*tf.PythonPath, projectFileMap); err != nil { - return errors.Wrap(err, TensorFlowKey) - } - } - return nil -} - -func (tf *TensorFlow) UserConfigStr() string { +func (predictor *Predictor) UserConfigStr() string { var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelKey, tf.Model)) - sb.WriteString(fmt.Sprintf("%s: %s\n", PredictorKey, tf.Predictor)) - if tf.PythonPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *tf.PythonPath)) + sb.WriteString(fmt.Sprintf("%s: %s\n", TypeKey, predictor.Type)) + sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, predictor.Path)) + if predictor.Model != nil { + sb.WriteString(fmt.Sprintf("%s: %s\n", ModelKey, *predictor.Model)) } - if tf.SignatureKey != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", SignatureKeyKey, *tf.SignatureKey)) + if predictor.PythonPath != nil { + sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *predictor.PythonPath)) } - if len(tf.Config) > 0 { + if len(predictor.Config) > 0 { sb.WriteString(fmt.Sprintf("%s:\n", ConfigKey)) - d, _ := yaml.Marshal(&tf.Config) + d, _ := yaml.Marshal(&predictor.Config) sb.WriteString(s.Indent(string(d), " ")) } - if len(tf.Env) > 0 { + if len(predictor.Env) > 0 { sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) - d, _ := yaml.Marshal(&tf.Env) + d, _ := yaml.Marshal(&predictor.Env) sb.WriteString(s.Indent(string(d), " ")) } + if predictor.SignatureKey != nil { + sb.WriteString(fmt.Sprintf("%s: %s\n", SignatureKeyKey, *predictor.SignatureKey)) + } return sb.String() } -func (onnx *ONNX) Validate(projectFileMap map[string][]byte) error { - awsClient, err := aws.NewFromS3Path(onnx.Model, false) - if err != nil { - return err - } - if ok, err := awsClient.IsS3PathFile(onnx.Model); err != nil || !ok { - return errors.Wrap(ErrorExternalNotFound(onnx.Model), ONNXKey, ModelKey) +func (predictor *Predictor) Validate(projectFileMap map[string][]byte) error { + switch predictor.Type { + case PythonPredictorType: + if err := predictor.PythonValidate(); err != nil { + return err + } + case TensorFlowPredictorType: + if err := predictor.TensorFlowValidate(); err != nil { + return err + } + case ONNXPredictorType: + if err := predictor.ONNXValidate(); err != nil { + return err + } } - if _, ok := projectFileMap[onnx.Predictor]; !ok { - return errors.Wrap(ErrorImplDoesNotExist(onnx.Predictor), ONNXKey, PredictorKey) + + if _, ok := projectFileMap[predictor.Path]; !ok { + return errors.Wrap(ErrorImplDoesNotExist(predictor.Path), PathKey) } - if onnx.PythonPath != nil { - if err := ValidatePythonPath(*onnx.PythonPath, projectFileMap); err != nil { - return errors.Wrap(err, ONNXKey) + + if predictor.PythonPath != nil { + if err := ValidatePythonPath(*predictor.PythonPath, projectFileMap); err != nil { + return err } } + return nil } -func (onnx *ONNX) UserConfigStr() string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelKey, onnx.Model)) - sb.WriteString(fmt.Sprintf("%s: %s\n", PredictorKey, onnx.Predictor)) - - if onnx.PythonPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *onnx.PythonPath)) +func (predictor *Predictor) TensorFlowValidate() error { + if predictor.Model == nil { + return ErrorFieldMustBeDefinedForPredictorType(ModelKey, TensorFlowPredictorType) } - if len(onnx.Config) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", ConfigKey)) - d, _ := yaml.Marshal(&onnx.Config) - sb.WriteString(s.Indent(string(d), " ")) + + model := *predictor.Model + + awsClient, err := aws.NewFromS3Path(model, false) + if err != nil { + return err } - if len(onnx.Env) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) - d, _ := yaml.Marshal(&onnx.Env) - sb.WriteString(s.Indent(string(d), " ")) + if strings.HasSuffix(model, ".zip") { + if ok, err := awsClient.IsS3PathFile(model); err != nil || !ok { + return errors.Wrap(ErrorExternalNotFound(model), TensorFlowKey, ModelKey) + } + } else { + path, err := GetTFServingExportFromS3Path(model, awsClient) + if path == "" || err != nil { + return errors.Wrap(ErrorInvalidTensorFlowDir(model), TensorFlowKey, ModelKey) + } + predictor.Model = pointer.String(path) } - return sb.String() + + return nil } -func (python *Python) Validate(projectFileMap map[string][]byte) error { - if _, ok := projectFileMap[python.Predictor]; !ok { - return errors.Wrap(ErrorImplDoesNotExist(python.Predictor), PythonKey, PredictorKey) +func (predictor *Predictor) ONNXValidate() error { + if predictor.Model == nil { + return ErrorFieldMustBeDefinedForPredictorType(ModelKey, ONNXPredictorType) } - if python.PythonPath != nil { - if err := ValidatePythonPath(*python.PythonPath, projectFileMap); err != nil { - return errors.Wrap(err, PythonKey) - } + + model := *predictor.Model + + awsClient, err := aws.NewFromS3Path(model, false) + if err != nil { + return err } + if ok, err := awsClient.IsS3PathFile(model); err != nil || !ok { + return errors.Wrap(ErrorExternalNotFound(model), ONNXKey, ModelKey) + } + + if predictor.SignatureKey != nil { + return ErrorFieldNotSupportedByPredictorType(SignatureKeyKey, ONNXPredictorType) + } + return nil } -func (python *Python) UserConfigStr() string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", PredictorKey, python.Predictor)) - if python.PythonPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *python.PythonPath)) +func (predictor *Predictor) PythonValidate() error { + if predictor.SignatureKey != nil { + return ErrorFieldNotSupportedByPredictorType(SignatureKeyKey, PythonPredictorType) } - if len(python.Config) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", ConfigKey)) - d, _ := yaml.Marshal(&python.Config) - sb.WriteString(s.Indent(string(d), " ")) - } - if len(python.Env) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) - d, _ := yaml.Marshal(&python.Env) - sb.WriteString(s.Indent(string(d), " ")) + + if predictor.Model != nil { + return ErrorFieldNotSupportedByPredictorType(ModelKey, PythonPredictorType) } - return sb.String() + + return nil } func (api *API) Validate(deploymentName string, projectFileMap map[string][]byte) error { @@ -483,36 +416,8 @@ func (api *API) Validate(deploymentName string, projectFileMap map[string][]byte api.Endpoint = pointer.String("/" + deploymentName + "/" + api.Name) } - specifiedModelFormats := []string{} - if api.TensorFlow != nil { - specifiedModelFormats = append(specifiedModelFormats, TensorFlowKey) - } - if api.ONNX != nil { - specifiedModelFormats = append(specifiedModelFormats, ONNXKey) - } - if api.Python != nil { - specifiedModelFormats = append(specifiedModelFormats, PythonKey) - } - - if len(specifiedModelFormats) == 0 { - return ErrorSpecifyOneModelFormatFoundNone(TensorFlowKey, ONNXKey, PythonKey) - } else if len(specifiedModelFormats) > 1 { - return ErrorSpecifyOneModelFormatFoundMultiple(specifiedModelFormats, TensorFlowKey, ONNXKey, PythonKey) - } else { - switch specifiedModelFormats[0] { - case TensorFlowKey: - if err := api.TensorFlow.Validate(projectFileMap); err != nil { - return errors.Wrap(err, Identify(api)) - } - case ONNXKey: - if err := api.ONNX.Validate(projectFileMap); err != nil { - return errors.Wrap(err, Identify(api)) - } - case PythonKey: - if err := api.Python.Validate(projectFileMap); err != nil { - return errors.Wrap(err, Identify(api)) - } - } + if err := api.Predictor.Validate(projectFileMap); err != nil { + return errors.Wrap(err, Identify(api), PredictorKey) } if err := api.Compute.Validate(); err != nil { diff --git a/pkg/operator/api/userconfig/config_key.go b/pkg/operator/api/userconfig/config_key.go index cf9ceff154..cccf9215c3 100644 --- a/pkg/operator/api/userconfig/config_key.go +++ b/pkg/operator/api/userconfig/config_key.go @@ -24,6 +24,8 @@ const ( // API ModelKey = "model" + TypeKey = "type" + PathKey = "path" PredictorKey = "predictor" EndpointKey = "endpoint" SignatureKeyKey = "signature_key" diff --git a/pkg/operator/api/userconfig/errors.go b/pkg/operator/api/userconfig/errors.go index a3de62df04..31d18b526e 100644 --- a/pkg/operator/api/userconfig/errors.go +++ b/pkg/operator/api/userconfig/errors.go @@ -46,6 +46,8 @@ const ( ErrExternalNotFound ErrONNXDoesntSupportZip ErrInvalidTensorFlowDir + ErrFieldMustBeDefinedForPredictorType + ErrFieldNotSupportedByPredictorType ErrDuplicateEndpoints ) @@ -69,6 +71,8 @@ var errorKinds = []string{ "err_external_not_found", "err_onnx_doesnt_support_zip", "err_invalid_tensorflow_dir", + "err_field_must_be_defined_for_predictor_type", + "err_field_not_supported_by_predictor_type", "err_duplicate_endpoints", } @@ -274,9 +278,23 @@ func ErrorInvalidTensorFlowDir(path string) error { } } +func ErrorFieldMustBeDefinedForPredictorType(fieldKey string, predictorType PredictorType) error { + return Error{ + Kind: ErrFieldMustBeDefinedForPredictorType, + message: fmt.Sprintf("%s field must be defined for predictor type %s", fieldKey, predictorType.String()), + } +} + +func ErrorFieldNotSupportedByPredictorType(fieldKey string, predictorType PredictorType) error { + return Error{ + Kind: ErrFieldNotSupportedByPredictorType, + message: fmt.Sprintf("%s is not supported field for predictor type %s", fieldKey, predictorType.String()), + } +} + func ErrorDuplicateEndpoints(endpoint string, apiNames ...string) error { return Error{ Kind: ErrDuplicateEndpoints, - message: fmt.Sprintf("multiple APIs specifiy the same endpoint (endpoint %s is used by the %s APIs)", s.UserStr(endpoint), s.UserStrsAnd(apiNames)), + message: fmt.Sprintf("multiple APIs specify the same endpoint (endpoint %s is used by the %s APIs)", s.UserStr(endpoint), s.UserStrsAnd(apiNames)), } } diff --git a/pkg/operator/api/userconfig/predictor_type.go b/pkg/operator/api/userconfig/predictor_type.go new file mode 100644 index 0000000000..b6547705d1 --- /dev/null +++ b/pkg/operator/api/userconfig/predictor_type.go @@ -0,0 +1,80 @@ +/* +Copyright 2019 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package userconfig + +type PredictorType int + +const ( + UnknownPredictorType PredictorType = iota + PythonPredictorType + TensorFlowPredictorType + ONNXPredictorType +) + +var predictorTypes = []string{ + "unknown", + "python", + "tensorflow", + "onnx", +} + +func PredictorTypeFromString(s string) PredictorType { + for i := 0; i < len(predictorTypes); i++ { + if s == predictorTypes[i] { + return PredictorType(i) + } + } + return UnknownPredictorType +} + +func PredictorTypeStrings() []string { + return predictorTypes[1:] +} + +func (t PredictorType) String() string { + return predictorTypes[t] +} + +// MarshalText satisfies TextMarshaler +func (t PredictorType) MarshalText() ([]byte, error) { + return []byte(t.String()), nil +} + +// UnmarshalText satisfies TextUnmarshaler +func (t *PredictorType) UnmarshalText(text []byte) error { + enum := string(text) + for i := 0; i < len(predictorTypes); i++ { + if enum == predictorTypes[i] { + *t = PredictorType(i) + return nil + } + } + + *t = UnknownPredictorType + return nil +} + +// UnmarshalBinary satisfies BinaryUnmarshaler +// Needed for msgpack +func (t *PredictorType) UnmarshalBinary(data []byte) error { + return t.UnmarshalText(data) +} + +// MarshalBinary satisfies BinaryMarshaler +func (t PredictorType) MarshalBinary() ([]byte, error) { + return []byte(t.String()), nil +} diff --git a/pkg/operator/context/apis.go b/pkg/operator/context/apis.go index d7f4339705..1e20de61c6 100644 --- a/pkg/operator/context/apis.go +++ b/pkg/operator/context/apis.go @@ -35,9 +35,7 @@ func getAPIs(config *userconfig.Config, deploymentVersion string, projectID stri buf.WriteString(*apiConfig.Endpoint) buf.WriteString(s.Obj(apiConfig.Tracker)) buf.WriteString(deploymentVersion) - buf.WriteString(s.Obj(apiConfig.TensorFlow)) - buf.WriteString(s.Obj(apiConfig.ONNX)) - buf.WriteString(s.Obj(apiConfig.Python)) + buf.WriteString(s.Obj(apiConfig.Predictor)) buf.WriteString(projectID) id := hash.Bytes(buf.Bytes()) diff --git a/pkg/operator/workloads/api_workload.go b/pkg/operator/workloads/api_workload.go index 234c68d632..9980dff713 100644 --- a/pkg/operator/workloads/api_workload.go +++ b/pkg/operator/workloads/api_workload.go @@ -93,12 +93,12 @@ func (aw *APIWorkload) Start(ctx *context.Context) error { desiredReplicas := getRequestedReplicasFromDeployment(api, k8sDeloyment, hpa) var deploymentSpec *kapps.Deployment - switch { - case api.TensorFlow != nil: + switch api.Predictor.Type { + case userconfig.TensorFlowPredictorType: deploymentSpec = tfAPISpec(ctx, api, aw.WorkloadID, desiredReplicas) - case api.ONNX != nil: + case userconfig.ONNXPredictorType: deploymentSpec = onnxAPISpec(ctx, api, aw.WorkloadID, desiredReplicas) - case api.Python != nil: + case userconfig.PythonPredictorType: deploymentSpec = pythonAPISpec(ctx, api, aw.WorkloadID, desiredReplicas) default: return errors.New(api.Name, "unknown model format encountered") // unexpected @@ -278,6 +278,8 @@ func tfAPISpec( tfServingLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) } + tensorflowModel := *ctx.APIs[api.Name].Predictor.Model + downloadConfig := downloadContainerConfig{ LastLog: fmt.Sprintf(downloaderLastLog, "tensorflow"), DownloadArgs: []downloadContainerArg{ @@ -290,9 +292,9 @@ func tfAPISpec( HideUnzippingLog: true, }, { - From: ctx.APIs[api.Name].TensorFlow.Model, + From: tensorflowModel, To: path.Join(consts.EmptyDirMountPath, "model"), - Unzip: strings.HasSuffix(ctx.APIs[api.Name].TensorFlow.Model, ".zip"), + Unzip: strings.HasSuffix(tensorflowModel, ".zip"), ItemName: "the model", TFModelVersionRename: path.Join(consts.EmptyDirMountPath, "model", "1"), }, @@ -301,7 +303,7 @@ func tfAPISpec( envVars := []kcore.EnvVar{} - for name, val := range api.TensorFlow.Env { + for name, val := range api.Predictor.Env { envVars = append(envVars, kcore.EnvVar{ Name: name, Value: val, @@ -319,10 +321,10 @@ func tfAPISpec( }, ) - if api.TensorFlow.PythonPath != nil { + if api.Predictor.PythonPath != nil { envVars = append(envVars, kcore.EnvVar{ Name: "PYTHON_PATH", - Value: path.Join(consts.EmptyDirMountPath, "project", *api.TensorFlow.PythonPath), + Value: path.Join(consts.EmptyDirMountPath, "project", *api.Predictor.PythonPath), }) } @@ -497,7 +499,7 @@ func pythonAPISpec( envVars := []kcore.EnvVar{} - for name, val := range api.Python.Env { + for name, val := range api.Predictor.Env { envVars = append(envVars, kcore.EnvVar{ Name: name, Value: val, @@ -515,10 +517,10 @@ func pythonAPISpec( }, ) - if api.Python.PythonPath != nil { + if api.Predictor.PythonPath != nil { envVars = append(envVars, kcore.EnvVar{ Name: "PYTHON_PATH", - Value: path.Join(consts.EmptyDirMountPath, "project", *api.Python.PythonPath), + Value: path.Join(consts.EmptyDirMountPath, "project", *api.Predictor.PythonPath), }) } @@ -648,7 +650,7 @@ func onnxAPISpec( HideUnzippingLog: true, }, { - From: ctx.APIs[api.Name].ONNX.Model, + From: *ctx.APIs[api.Name].Predictor.Model, To: path.Join(consts.EmptyDirMountPath, "model"), ItemName: "the model", }, @@ -657,7 +659,7 @@ func onnxAPISpec( envVars := []kcore.EnvVar{} - for name, val := range api.ONNX.Env { + for name, val := range api.Predictor.Env { envVars = append(envVars, kcore.EnvVar{ Name: name, Value: val, @@ -675,10 +677,10 @@ func onnxAPISpec( }, ) - if api.ONNX.PythonPath != nil { + if api.Predictor.PythonPath != nil { envVars = append(envVars, kcore.EnvVar{ Name: "PYTHON_PATH", - Value: path.Join(consts.EmptyDirMountPath, "project", *api.ONNX.PythonPath), + Value: path.Join(consts.EmptyDirMountPath, "project", *api.Predictor.PythonPath), }) } diff --git a/pkg/workloads/cortex/lib/context.py b/pkg/workloads/cortex/lib/context.py index 9ead2a6fa3..e7b88e7f08 100644 --- a/pkg/workloads/cortex/lib/context.py +++ b/pkg/workloads/cortex/lib/context.py @@ -129,22 +129,23 @@ def load_module(self, module_prefix, module_name, impl_path): def get_predictor_class(self, api_name, project_dir): api = self.apis[api_name] - if api.get("tensorflow") is not None: - api_type = "tensorflow" + + if api["predictor"]["type"] == "tensorflow": target_class_name = "TensorFlowPredictor" - elif api.get("onnx") is not None: - api_type = "onnx" + validations = TENSORFLOW_CLASS_VALIDATION + elif api["predictor"]["type"] == "onnx": target_class_name = "ONNXPredictor" - elif api.get("python") is not None: - api_type = "python" + validations = ONNX_CLASS_VALIDATION + elif api["predictor"]["type"] == "python": target_class_name = "PythonPredictor" + validations = PYTHON_CLASS_VALIDATION try: impl = self.load_module( - "predictor", api["name"], os.path.join(project_dir, api[api_type]["predictor"]) + "predictor", api["name"], os.path.join(project_dir, api["predictor"]["path"]) ) except CortexException as e: - e.wrap("api " + api_name, "error in " + api[api_type]["predictor"]) + e.wrap("api " + api_name, "error in " + api["predictor"]["path"]) raise finally: refresh_logger() @@ -164,15 +165,9 @@ def get_predictor_class(self, api_name, project_dir): if predictor_class is None: raise UserException("{} class is not defined".format(target_class_name)) - if api_type == "tensorflow": - _validate_impl(predictor_class, TENSORFLOW_CLASS_VALIDATION) - elif api_type == "onnx": - _validate_impl(predictor_class, ONNX_CLASS_VALIDATION) - elif api_type == "python": - _validate_impl(predictor_class, PYTHON_CLASS_VALIDATION) - + _validate_impl(predictor_class, validations) except CortexException as e: - e.wrap("api " + api_name, "error in " + api[api_type]["predictor"]) + e.wrap("api " + api_name, "error in " + api["predictor"]["path"]) raise return predictor_class diff --git a/pkg/workloads/cortex/onnx_serve/api.py b/pkg/workloads/cortex/onnx_serve/api.py index b3e031b1e8..a2bfb7d65c 100644 --- a/pkg/workloads/cortex/onnx_serve/api.py +++ b/pkg/workloads/cortex/onnx_serve/api.py @@ -22,7 +22,7 @@ from waitress import serve from cortex.lib import util, Context, api_utils -from cortex.lib.log import cx_logger, debug_obj +from cortex.lib.log import cx_logger, debug_obj, refresh_logger from cortex.lib.exceptions import CortexException, UserRuntimeException, UserException from cortex.onnx_serve.client import ONNXClient @@ -81,7 +81,7 @@ def predict(): try: output = predictor.predict(payload) except Exception as e: - raise UserRuntimeException(api["onnx"]["predictor"], "predict", str(e)) from e + raise UserRuntimeException(api["predictor"]["path"], "predict", str(e)) from e debug_obj("prediction", output, debug) except Exception as e: @@ -121,18 +121,25 @@ def start(args): local_cache["api"] = api local_cache["ctx"] = ctx - if api.get("onnx") is None: - raise CortexException(api["name"], "onnx key not configured") + if api["predictor"]["type"] != "onnx": + raise CortexException(api["name"], "predictor type is not onnx") - cx_logger().info("loading the predictor from {}".format(api["onnx"]["predictor"])) + cx_logger().info("loading the predictor from {}".format(api["predictor"]["path"])) - _, prefix = ctx.storage.deconstruct_s3_path(api["onnx"]["model"]) + _, prefix = ctx.storage.deconstruct_s3_path(api["predictor"]["model"]) model_path = os.path.join(args.model_dir, os.path.basename(prefix)) local_cache["client"] = ONNXClient(model_path) predictor_class = ctx.get_predictor_class(api["name"], args.project_dir) - local_cache["predictor"] = predictor_class(local_cache["client"], api["onnx"]["config"]) + try: + local_cache["predictor"] = predictor_class( + local_cache["client"], api["predictor"]["config"] + ) + except Exception as e: + raise UserRuntimeException(api["predictor"]["path"], "__init__", str(e)) from e + finally: + refresh_logger() except Exception as e: cx_logger().exception("failed to start api") sys.exit(1) @@ -146,8 +153,8 @@ def start(args): cx_logger().info("ONNX model signature: {}".format(local_cache["client"].input_signature)) waitress_kwargs = {} - if api["onnx"].get("config") is not None: - for key, value in api["onnx"]["config"].items(): + if api["predictor"].get("config") is not None: + for key, value in api["predictor"]["config"].items(): if key.startswith("waitress_"): waitress_kwargs[key[len("waitress_") :]] = value diff --git a/pkg/workloads/cortex/python_serve/api.py b/pkg/workloads/cortex/python_serve/api.py index 144e25a09c..64f8f8f597 100644 --- a/pkg/workloads/cortex/python_serve/api.py +++ b/pkg/workloads/cortex/python_serve/api.py @@ -92,7 +92,7 @@ def predict(): output = predictor.predict(payload) debug_obj("prediction", output, debug) except Exception as e: - raise UserRuntimeException(api["python"]["path"], "predict", str(e)) from e + raise UserRuntimeException(api["predictor"]["path"], "predict", str(e)) from e except Exception as e: cx_logger().exception("prediction failed") return prediction_failed(str(e)) @@ -120,16 +120,16 @@ def start(args): local_cache["api"] = api local_cache["ctx"] = ctx - if api.get("python") is None: - raise CortexException(api["name"], "python key not configured") + if api["predictor"]["type"] != "python": + raise CortexException(api["name"], "predictor type is not python") - cx_logger().info("loading the predictor from {}".format(api["python"]["predictor"])) + cx_logger().info("loading the predictor from {}".format(api["predictor"]["path"])) predictor_class = ctx.get_predictor_class(api["name"], args.project_dir) try: - local_cache["predictor"] = predictor_class(api["python"]["config"]) + local_cache["predictor"] = predictor_class(api["predictor"]["config"]) except Exception as e: - raise UserRuntimeException(api["python"]["path"], "__init__", str(e)) from e + raise UserRuntimeException(api["predictor"]["path"], "__init__", str(e)) from e finally: refresh_logger() except: @@ -143,8 +143,8 @@ def start(args): cx_logger().warn("an error occurred while attempting to load classes", exc_info=True) waitress_kwargs = {} - if api["python"].get("config") is not None: - for key, value in api["python"]["config"].items(): + if api["predictor"].get("config") is not None: + for key, value in api["predictor"]["config"].items(): if key.startswith("waitress_"): waitress_kwargs[key[len("waitress_") :]] = value diff --git a/pkg/workloads/cortex/tf_api/api.py b/pkg/workloads/cortex/tf_api/api.py index b18302d06c..2302c5176e 100644 --- a/pkg/workloads/cortex/tf_api/api.py +++ b/pkg/workloads/cortex/tf_api/api.py @@ -22,7 +22,7 @@ from waitress import serve from cortex.lib import util, Context, api_utils -from cortex.lib.log import cx_logger, debug_obj +from cortex.lib.log import cx_logger, debug_obj, refresh_logger from cortex.lib.exceptions import UserRuntimeException, UserException, CortexException from cortex.tf_api.client import TensorFlowClient @@ -81,7 +81,7 @@ def predict(): try: output = predictor.predict(payload) except Exception as e: - raise UserRuntimeException(api["tensorflow"]["predictor"], "predict", str(e)) from e + raise UserRuntimeException(api["predictor"]["path"], "predict", str(e)) from e debug_obj("prediction", output, debug) except Exception as e: cx_logger().exception("prediction failed") @@ -167,20 +167,26 @@ def start(args): local_cache["api"] = api local_cache["ctx"] = ctx + if api["predictor"]["type"] != "tensorflow": + raise CortexException(api["name"], "predictor type is not tensorflow") + local_cache["client"] = TensorFlowClient( - "localhost:" + str(args.tf_serve_port), api["tensorflow"]["signature_key"] + "localhost:" + str(args.tf_serve_port), api["predictor"]["signature_key"] ) - if api.get("tensorflow") is None: - raise CortexException(api["name"], "tensorflow key not configured") - - cx_logger().info("loading the predictor from {}".format(api["tensorflow"]["predictor"])) + cx_logger().info("loading the predictor from {}".format(api["predictor"]["path"])) predictor_class = ctx.get_predictor_class(api["name"], args.project_dir) - local_cache["predictor"] = predictor_class( - local_cache["client"], api["tensorflow"]["config"] - ) + try: + local_cache["predictor"] = predictor_class( + local_cache["client"], api["predictor"]["config"] + ) + except Exception as e: + raise UserRuntimeException(api["predictor"]["path"], "__init__", str(e)) from e + finally: + refresh_logger() + except Exception as e: cx_logger().exception("failed to start api") sys.exit(1) @@ -200,8 +206,8 @@ def start(args): cx_logger().info("TensorFlow model signature: {}".format(local_cache["client"].input_signature)) waitress_kwargs = {} - if api["tensorflow"].get("config") is not None: - for key, value in api["tensorflow"]["config"].items(): + if api["predictor"].get("config") is not None: + for key, value in api["predictor"]["config"].items(): if key.startswith("waitress_"): waitress_kwargs[key[len("waitress_") :]] = value