From d850471c17e35893a9735e5f6ead195662370956 Mon Sep 17 00:00:00 2001 From: vishal Date: Wed, 23 Sep 2020 00:39:10 -0400 Subject: [PATCH 1/4] Include deployment id as a dimension in api metrics --- pkg/operator/resources/realtimeapi/api.go | 5 +++++ pkg/operator/resources/realtimeapi/metrics.go | 4 ++++ pkg/workloads/cortex/lib/type/api.py | 3 +++ 3 files changed, 12 insertions(+) diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 42823ed863..5ce7e47a30 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -150,6 +150,11 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", errors.Wrap(err, "upload api spec") } + // Use api spec indexed by PredictorID for replicas to prevent rolling updates when SpecID changes without PredictorID changing + if err := config.AWS.UploadJSONToS3(api, config.Cluster.Bucket, api.PredictorKey); err != nil { + return "", errors.Wrap(err, "upload predictor spec") + } + if err := applyK8sDeployment(api, prevDeployment); err != nil { return "", err } diff --git a/pkg/operator/resources/realtimeapi/metrics.go b/pkg/operator/resources/realtimeapi/metrics.go index acf03df83e..4675cffe66 100644 --- a/pkg/operator/resources/realtimeapi/metrics.go +++ b/pkg/operator/resources/realtimeapi/metrics.go @@ -236,6 +236,10 @@ func getAPIDimensions(api *spec.API) []*cloudwatch.Dimension { Name: aws.String("PredictorID"), Value: aws.String(api.PredictorID), }, + { + Name: aws.String("DeploymentID"), + Value: aws.String(api.DeploymentID), + }, } } diff --git a/pkg/workloads/cortex/lib/type/api.py b/pkg/workloads/cortex/lib/type/api.py index 71aa77bcf1..2c6d6bce10 100644 --- a/pkg/workloads/cortex/lib/type/api.py +++ b/pkg/workloads/cortex/lib/type/api.py @@ -32,6 +32,8 @@ def __init__(self, provider, storage, model_dir, cache_dir=".", **kwargs): self.provider = provider self.id = kwargs["id"] self.predictor_id = kwargs["predictor_id"] + print(kwargs) + self.deployment_id = kwargs["deployment_id"] self.key = kwargs["key"] self.metadata_root = kwargs["metadata_root"] self.name = kwargs["name"] @@ -70,6 +72,7 @@ def metric_dimensions_with_id(self): return [ {"Name": "APIName", "Value": self.name}, {"Name": "PredictorID", "Value": self.predictor_id}, + {"Name": "DeploymentID", "Value": self.deployment_id}, ] def metric_dimensions(self): From 8cdaba0506b0a5d0dd736c19ab1ec1f11c0cc5ca Mon Sep 17 00:00:00 2001 From: vishal Date: Wed, 23 Sep 2020 10:45:29 -0400 Subject: [PATCH 2/4] Update api.py --- pkg/workloads/cortex/lib/type/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/workloads/cortex/lib/type/api.py b/pkg/workloads/cortex/lib/type/api.py index 2c6d6bce10..016cbbfabd 100644 --- a/pkg/workloads/cortex/lib/type/api.py +++ b/pkg/workloads/cortex/lib/type/api.py @@ -32,7 +32,6 @@ def __init__(self, provider, storage, model_dir, cache_dir=".", **kwargs): self.provider = provider self.id = kwargs["id"] self.predictor_id = kwargs["predictor_id"] - print(kwargs) self.deployment_id = kwargs["deployment_id"] self.key = kwargs["key"] self.metadata_root = kwargs["metadata_root"] From a380f953f406a4f0cabb6d40630a467e5476afc6 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 23 Sep 2020 14:34:56 -0400 Subject: [PATCH 3/4] Update api.go --- pkg/operator/resources/realtimeapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 5ce7e47a30..902b8261f9 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -150,7 +150,7 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", errors.Wrap(err, "upload api spec") } - // Use api spec indexed by PredictorID for replicas to prevent rolling updates when SpecID changes without PredictorID changing + // Reuploading api spec to the same PredictorID but this time with a different DeploymentID due to refresh if err := config.AWS.UploadJSONToS3(api, config.Cluster.Bucket, api.PredictorKey); err != nil { return "", errors.Wrap(err, "upload predictor spec") } From eeabceea24f2f313ce72d48ccfad2eb1a436c836 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 23 Sep 2020 14:42:42 -0400 Subject: [PATCH 4/4] Update api.go --- pkg/operator/resources/realtimeapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 902b8261f9..a322870e8e 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -150,7 +150,7 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", errors.Wrap(err, "upload api spec") } - // Reuploading api spec to the same PredictorID but this time with a different DeploymentID due to refresh + // Reupload api spec to the same PredictorID but with the new DeploymentID if err := config.AWS.UploadJSONToS3(api, config.Cluster.Bucket, api.PredictorKey); err != nil { return "", errors.Wrap(err, "upload predictor spec") }