/
test_eks_mxnet_inference.py
100 lines (73 loc) · 3.44 KB
/
test_eks_mxnet_inference.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import os
import random
import pytest
from invoke import run
import test.test_utils.eks as eks_utils
import test.test_utils as test_utils
@pytest.mark.model("squeezenet")
def test_eks_mxnet_squeezenet_inference(mxnet_inference):
if "eia" in mxnet_inference:
pytest.skip("Skipping EKS Test for EIA")
num_replicas = "1"
rand_int = random.randint(4001, 6000)
processor = "gpu" if "gpu" in mxnet_inference else "cpu"
model = "squeezenet=https://s3.amazonaws.com/model-server/models/squeezenet_v1.1/squeezenet_v1.1.model"
yaml_path = os.path.join(os.sep, "tmp", f"mxnet_single_node_{processor}_inference_{rand_int}.yaml")
inference_service_name = selector_name = f"squeezenet-service-{rand_int}"
search_replace_dict = {
"<MODELS>": model,
"<NUM_REPLICAS>": num_replicas,
"<SELECTOR_NAME>": selector_name,
"<INFERENCE_SERVICE_NAME>": inference_service_name,
"<DOCKER_IMAGE_BUILD_ID>": mxnet_inference
}
if processor == "gpu":
search_replace_dict["<NUM_GPUS>"] = "1"
eks_utils.write_eks_yaml_file_from_template(
eks_utils.get_single_node_inference_template_path("mxnet", processor), yaml_path, search_replace_dict
)
try:
run("kubectl apply -f {}".format(yaml_path))
port_to_forward = random.randint(49152, 65535)
if eks_utils.is_service_running(selector_name):
eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8080")
assert test_utils.request_mxnet_inference(port=port_to_forward)
except ValueError as excp:
eks_utils.LOGGER.error("Service is not running: %s", excp)
finally:
run(f"kubectl delete deployment {selector_name}")
run(f"kubectl delete service {selector_name}")
@pytest.mark.integration("gluonnlp")
@pytest.mark.model("bert_sst")
def test_eks_mxnet_gluonnlp_inference(mxnet_inference, py3_only):
if "eia" in mxnet_inference:
pytest.skip("Skipping EKS Test for EIA")
num_replicas = "1"
rand_int = random.randint(4001, 6000)
processor = "gpu" if "gpu" in mxnet_inference else "cpu"
model = "https://aws-dlc-sample-models.s3.amazonaws.com/bert_sst/bert_sst.mar"
yaml_path = os.path.join(os.sep, "tmp", f"mxnet_single_node_gluonnlp_{processor}_inference_{rand_int}.yaml")
inference_service_name = selector_name = f"gluonnlp-service-{processor}-{rand_int}"
search_replace_dict = {
"<MODELS>": model,
"<NUM_REPLICAS>": num_replicas,
"<SELECTOR_NAME>": selector_name,
"<INFERENCE_SERVICE_NAME>": inference_service_name,
"<DOCKER_IMAGE_BUILD_ID>": mxnet_inference
}
if processor == "gpu":
search_replace_dict["<NUM_GPUS>"] = "1"
eks_utils.write_eks_yaml_file_from_template(
eks_utils.get_single_node_inference_template_path("mxnet", processor), yaml_path, search_replace_dict
)
try:
run("kubectl apply -f {}".format(yaml_path))
port_to_forward = random.randint(49152, 65535)
if eks_utils.is_service_running(selector_name):
eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8080")
assert test_utils.request_mxnet_inference_gluonnlp(port=port_to_forward)
except ValueError as excp:
eks_utils.LOGGER.error("Service is not running: %s", excp)
finally:
run(f"kubectl delete deployment {selector_name}")
run(f"kubectl delete service {selector_name}")