From abfeae7b78ba84e6113a452d7e6ab27dcf37ccd8 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 29 Jun 2020 19:44:45 +0300 Subject: [PATCH 1/4] Remove unnecessary env & reposition env source --- pkg/operator/resources/syncapi/k8s_specs.go | 4 ---- pkg/workloads/cortex/serve/run.sh | 14 +++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/operator/resources/syncapi/k8s_specs.go b/pkg/operator/resources/syncapi/k8s_specs.go index 46b910ab66..e3a49ee0ff 100644 --- a/pkg/operator/resources/syncapi/k8s_specs.go +++ b/pkg/operator/resources/syncapi/k8s_specs.go @@ -637,10 +637,6 @@ func getEnvVars(api *spec.API, container string) []kcore.EnvVar { Name: "CORTEX_THREADS_PER_PROCESS", Value: s.Int32(api.Predictor.ThreadsPerProcess), }, - kcore.EnvVar{ - Name: "CORTEX_MAX_REPLICA_CONCURRENCY", - Value: s.Int64(api.Autoscaling.MaxReplicaConcurrency), - }, kcore.EnvVar{ Name: "CORTEX_MAX_PROCESS_CONCURRENCY", // add 1 because it was required to achieve the target concurrency for 1 process, 1 thread diff --git a/pkg/workloads/cortex/serve/run.sh b/pkg/workloads/cortex/serve/run.sh index d252a0f807..2c28fb21a1 100755 --- a/pkg/workloads/cortex/serve/run.sh +++ b/pkg/workloads/cortex/serve/run.sh @@ -39,6 +39,13 @@ rm -rf /mnt/workspace/api_readiness.txt # allow for the liveness check to pass until the API is running echo "9999999999" > /mnt/workspace/api_liveness.txt +# export environment variables +if [ -f "/mnt/project/.env" ]; then + set -a + source /mnt/project/.env + set +a +fi + export PYTHONPATH=$PYTHONPATH:$PYTHON_PATH # ensure predictor print() statements are always flushed export PYTHONUNBUFFERED=TRUE @@ -49,13 +56,6 @@ if [ "$CORTEX_PROVIDER" != "local" ]; then sysctl -w net.ipv4.tcp_fin_timeout=30 >/dev/null fi -# export environment variables -if [ -f "/mnt/project/.env" ]; then - set -a - source /mnt/project/.env - set +a -fi - # execute script if present in project's directory if [ -f "/mnt/project/dependencies.sh" ]; then bash -e /mnt/project/dependencies.sh From 9cdc5cd79c1c25eba5c638e32d881ad30f307961 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 29 Jun 2020 21:43:30 +0300 Subject: [PATCH 2/4] Set default max replica concurrency for local dep --- cli/local/docker_spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/local/docker_spec.go b/cli/local/docker_spec.go index 0691781db9..b7a9ebba66 100644 --- a/cli/local/docker_spec.go +++ b/cli/local/docker_spec.go @@ -93,7 +93,7 @@ func getAPIEnv(api *spec.API, awsClient *aws.Client) []string { "CORTEX_THREADS_PER_PROCESS="+s.Int32(api.Predictor.ThreadsPerProcess), // add 1 because it was required to achieve the target concurrency for 1 process, 1 thread "CORTEX_MAX_PROCESS_CONCURRENCY="+s.Int64(1+int64(math.Round(float64(consts.DefaultMaxReplicaConcurrency)/float64(api.Predictor.ProcessesPerReplica)))), - "CORTEX_SO_MAX_CONN=1000", + "CORTEX_SO_MAX_CONN="+s.Int64(consts.DefaultMaxReplicaConcurrency+100), "AWS_REGION="+awsClient.Region, ) From ad14a556046a050a6c6e7fe04ba4cefd2adc1bb5 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 30 Jun 2020 04:01:37 +0300 Subject: [PATCH 3/4] Add comment to indicate what the param stands for --- pkg/workloads/cortex/serve/start_uvicorn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/workloads/cortex/serve/start_uvicorn.py b/pkg/workloads/cortex/serve/start_uvicorn.py index 200b9a2b39..342e2ec62f 100644 --- a/pkg/workloads/cortex/serve/start_uvicorn.py +++ b/pkg/workloads/cortex/serve/start_uvicorn.py @@ -87,7 +87,9 @@ def main(): host="0.0.0.0", port=int(os.environ["CORTEX_SERVING_PORT"]), workers=int(os.environ["CORTEX_PROCESSES_PER_REPLICA"]), - limit_concurrency=int(os.environ["CORTEX_MAX_PROCESS_CONCURRENCY"]), + limit_concurrency=int( + os.environ["CORTEX_MAX_PROCESS_CONCURRENCY"] + ), # this is a per process limit backlog=int(os.environ["CORTEX_SO_MAX_CONN"]), log_config=log_config, log_level="info", From fb1ba650d9ec6254d436d2a927006fee603d7d3b Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Mon, 29 Jun 2020 22:34:38 -0700 Subject: [PATCH 4/4] Add comment --- cli/local/docker_spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/local/docker_spec.go b/cli/local/docker_spec.go index b7a9ebba66..a3fa9bacef 100644 --- a/cli/local/docker_spec.go +++ b/cli/local/docker_spec.go @@ -93,7 +93,7 @@ func getAPIEnv(api *spec.API, awsClient *aws.Client) []string { "CORTEX_THREADS_PER_PROCESS="+s.Int32(api.Predictor.ThreadsPerProcess), // add 1 because it was required to achieve the target concurrency for 1 process, 1 thread "CORTEX_MAX_PROCESS_CONCURRENCY="+s.Int64(1+int64(math.Round(float64(consts.DefaultMaxReplicaConcurrency)/float64(api.Predictor.ProcessesPerReplica)))), - "CORTEX_SO_MAX_CONN="+s.Int64(consts.DefaultMaxReplicaConcurrency+100), + "CORTEX_SO_MAX_CONN="+s.Int64(consts.DefaultMaxReplicaConcurrency+100), // add a buffer to be safe "AWS_REGION="+awsClient.Region, )