Skip to content

Commit

Permalink
[System tests] Model monitoring test fix (#1351)
Browse files Browse the repository at this point in the history
  • Loading branch information
Eyal Salomon committed Sep 24, 2021
1 parent 6fe42fd commit c4f6b74
Showing 1 changed file with 22 additions and 23 deletions.
45 changes: 22 additions & 23 deletions tests/system/model_monitoring/test_model_monitoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
@TestMLRunSystem.skip_test_if_env_not_configured
@pytest.mark.enterprise
class TestModelMonitoringAPI(TestMLRunSystem):
project_name = "mm-system-test-project"

def test_clear_endpoint(self):
endpoint = self._mock_random_endpoint()
db = mlrun.get_run_db()
Expand Down Expand Up @@ -85,18 +87,12 @@ def test_store_endpoint_update_existing(self):

assert endpoint_after_update.status.state == updated_state

def test_list_endpoints(self):
db = mlrun.get_run_db()

# test before creating the endpoints on fresh new project
project_name = "test-list-endpoints"
project = mlrun.get_or_create_project(project_name, context="./")
project.set_model_monitoring_credentials(os.environ.get("V3IO_ACCESS_KEY"))

endpoints_out = db.list_model_endpoints(project_name)
def test_list_endpoints_on_empty_project(self):
endpoints_out = mlrun.get_run_db().list_model_endpoints(self.project_name)
assert len(endpoints_out.endpoints) == 0

db.delete_project(project_name)
def test_list_endpoints(self):
db = mlrun.get_run_db()

number_of_endpoints = 5
endpoints_in = [
Expand Down Expand Up @@ -220,9 +216,9 @@ def test_get_endpoint_metrics(self):

assert total == response_total

@pytest.mark.timeout(480)
@pytest.mark.timeout(200)
def test_basic_model_monitoring(self):
simulation_time = 60 * 5 # 5 minutes
simulation_time = 20 # 20 seconds
# Deploy Model Servers
project = mlrun.get_run_db().get_project(self.project_name)
project.set_model_monitoring_credentials(os.environ.get("V3IO_ACCESS_KEY"))
Expand All @@ -246,13 +242,14 @@ def test_basic_model_monitoring(self):
serving_fn.set_tracking()

model_name = "sklearn_RandomForestClassifier"
mlrun.get_dataitem(
os.path.relpath(str(self.assets_path / "model.pkl"))
).download("model.pkl")

# Log the model through the projects API so that it is available through the feature store API
# Upload the model through the projects API so that it is available to the serving function
project.log_model(
model_name, model_file="model.pkl", training_set=train_set,
model_name,
model_dir=os.path.relpath(self.assets_path),
model_file="model.pkl",
training_set=train_set,
artifact_path=f"v3io:///projects/{project.metadata.name}",
)
# Add the model to the serving function's routing spec
serving_fn.add_model(
Expand All @@ -274,9 +271,9 @@ def test_basic_model_monitoring(self):
)
sleep(uniform(0.2, 1.7))

@pytest.mark.timeout(480)
@pytest.mark.timeout(200)
def test_model_monitoring_voting_ensemble(self):
simulation_time = 60 * 5 # 5 minutes
simulation_time = 20 # 20 seconds
project = mlrun.get_run_db().get_project(self.project_name)
project.set_model_monitoring_credentials(os.environ.get("V3IO_ACCESS_KEY"))

Expand Down Expand Up @@ -309,14 +306,15 @@ def test_model_monitoring_voting_ensemble(self):
"sklearn_LogisticRegression",
"sklearn_AdaBoostClassifier",
]
mlrun.get_dataitem(
os.path.relpath(str(self.assets_path / "model.pkl"))
).download("model.pkl")

for name in model_names:
# Log the model through the projects API so that it is available through the feature store API
project.log_model(
name, model_file="model.pkl", training_set=train_set,
name,
model_dir=os.path.relpath(self.assets_path),
model_file="model.pkl",
training_set=train_set,
artifact_path=f"v3io:///projects/{project.metadata.name}",
)
# Add the model to the serving function's routing spec
serving_fn.add_model(
Expand Down Expand Up @@ -345,6 +343,7 @@ def _get_auth_info() -> mlrun.api.schemas.AuthInfo:
@pytest.fixture(autouse=True)
def cleanup_endpoints(self):
db = mlrun.get_run_db()

endpoints = db.list_model_endpoints(self.project_name)
for endpoint in endpoints.endpoints:
db.delete_model_endpoint_record(
Expand Down

0 comments on commit c4f6b74

Please sign in to comment.