diff --git a/simvue/sender.py b/simvue/sender.py index ee8f2913..30e869d9 100644 --- a/simvue/sender.py +++ b/simvue/sender.py @@ -56,6 +56,7 @@ def upload_cached_file( obj_type: str, file_path: pydantic.FilePath, id_mapping: dict[str, str], + throw_exceptions: bool, retry_failed_uploads: bool, lock: threading.Lock, ) -> None: @@ -71,6 +72,10 @@ def upload_cached_file( The path to the cached file to upload id_mapping : dict[str, str] A mapping of offline to online object IDs + throw_exceptions : bool + Whether to throw exceptions, or just log them + retry_failed_uploads : bool + Whether to retry failed uploads or ignore them lock : threading.Lock A lock to prevent multiple threads accessing the id mapping directory at once """ @@ -83,7 +88,10 @@ def upload_cached_file( try: _instance_class = getattr(simvue.api.objects, _exact_type) - except AttributeError: + except AttributeError as error: + if throw_exceptions: + raise error + _logger.error(f"Attempt to initialise unknown type '{_exact_type}'") _log_upload_failed(file_path) return @@ -109,11 +117,13 @@ def upload_cached_file( _new_id = obj_for_upload.id except Exception as error: - if "status 409" in error.args[0]: + if "status 409" in str(error): return + if throw_exceptions: + raise error _logger.error( - f"Error while committing '{_instance_class.__name__}': {error.args[0]}" + f"Error while committing '{_instance_class.__name__}': {str(error)}" ) _log_upload_failed(file_path) return @@ -182,6 +192,7 @@ def sender( max_workers: int = 5, threading_threshold: int = 10, objects_to_upload: list[str] = UPLOAD_ORDER, + throw_exceptions: bool = False, retry_failed_uploads: bool = False, ) -> dict[str, str]: """Send data from a local cache directory to the Simvue server. @@ -196,6 +207,8 @@ def sender( The number of cached files above which threading will be used objects_to_upload : list[str] Types of objects to upload, by default uploads all types of objects present in cache + throw_exceptions : bool, optional + Whether to throw exceptions as they are encountered in the sender, default is False (exceptions will be logged) retry_failed_uploads : bool, optional Whether to retry sending objects which previously failed, by default False @@ -238,6 +251,7 @@ def sender( obj_type=_obj_type, file_path=file_path, id_mapping=_id_mapping, + throw_exceptions=throw_exceptions, retry_failed_uploads=retry_failed_uploads, lock=_lock, ) @@ -251,11 +265,15 @@ def sender( obj_type=_obj_type, file_path=file_path, id_mapping=_id_mapping, + throw_exceptions=throw_exceptions, retry_failed_uploads=retry_failed_uploads, lock=_lock, ), _offline_files, ) + # This will raise any exceptions encountered during sending + for result in _results: + pass # Send heartbeats _headers: dict[str, str] = { diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 4a0d54dc..2654ff0a 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -116,7 +116,7 @@ def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_pl run_created.config(enable_emission_metrics=True) time.sleep(5) # Run should continue, but fail to log metrics until sender runs and creates file - id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"]) + id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], throw_exceptions=True) _run = RunObject(identifier=id_mapping[run_created.id]) _metric_names = [item[0] for item in _run.metrics] for _metric in ["emissions", "energy_consumed"]: @@ -126,7 +126,7 @@ def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_pl assert _delta_metric_name not in _metric_names # Sender should now have made a local file, and the run should be able to use it to create emissions metrics time.sleep(5) - id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"]) + id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], throw_exceptions=True) _run.refresh() _metric_names = [item[0] for item in _run.metrics] client = sv_cl.Client() @@ -318,7 +318,7 @@ def test_log_metrics_offline( run.log_metrics(METRICS) time.sleep(1) - id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) time.sleep(1) if metric_type == "tensor": @@ -441,7 +441,7 @@ def test_visibility_offline( retention_period=os.environ.get("SIMVUE_TESTING_RETENTION_PERIOD", "2 mins"), ) _id = run.id - _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) run.close() _retrieved_run = RunObject(identifier=_id_mapping.get(_id)) @@ -478,7 +478,7 @@ def test_log_events_offline(create_plain_run_offline: tuple[sv_run.Run, dict]) - run, _ = create_plain_run_offline run_name = run.name run.log_event(EVENT_MSG) - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) client = sv_cl.Client() attempts: int = 0 @@ -488,7 +488,7 @@ def test_log_events_offline(create_plain_run_offline: tuple[sv_run.Run, dict]) - not (event_data := client.get_events(client.get_run_id_from_name(run_name), count_limit=1)) ) and attempts < 5: time.sleep(1) - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) attempts += 1 assert event_data[0].get("message", EVENT_MSG) @@ -497,7 +497,7 @@ def test_log_events_offline(create_plain_run_offline: tuple[sv_run.Run, dict]) - @pytest.mark.offline def test_offline_tags(create_plain_run_offline: tuple[sv_run.Run, dict]) -> None: run, run_data = create_plain_run_offline - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) client = sv_cl.Client() tags = client.get_tags() @@ -557,7 +557,7 @@ def test_update_metadata_offline( # Try updating an already defined piece of metadata run.update_metadata({"a": 1}) - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) client = sv_cl.Client() run_info = client.get_run(client.get_run_id_from_name(run_name)) @@ -945,7 +945,7 @@ def test_save_file_offline( "w", ) as out_f: out_f.write("updated file!") - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) os.remove(out_name) client = sv_cl.Client() base_name = name or out_name.name @@ -1031,7 +1031,7 @@ def test_update_tags_offline( simvue_run.update_tags(["additional"]) - sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) client = sv_cl.Client() run_data = client.get_run(client.get_run_id_from_name(run_name)) @@ -1358,7 +1358,7 @@ def test_reconnect_functionality(mode, monkeypatch: pytest.MonkeyPatch) -> None: ) run_id = run.id if mode == "offline": - _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) run_id = _id_mapping.get(run_id) client = simvue.Client() @@ -1372,7 +1372,7 @@ def test_reconnect_functionality(mode, monkeypatch: pytest.MonkeyPatch) -> None: run.log_event("Testing!") if mode == "offline": - _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10, throw_exceptions=True) _reconnected_run = client.get_run(run_id) assert dict(_reconnected_run.metrics)["test_metric"]["last"] == 1 diff --git a/tests/functional/test_run_execute_process.py b/tests/functional/test_run_execute_process.py index a3dfb3bc..1337e301 100644 --- a/tests/functional/test_run_execute_process.py +++ b/tests/functional/test_run_execute_process.py @@ -24,7 +24,7 @@ def test_monitor_processes(create_plain_run_offline: tuple[Run, dict]): _run.add_process(f"process_1_{os.environ.get('PYTEST_XDIST_WORKER', 0)}", Command="Write-Output 'Hello World!'", executable="powershell") _run.add_process(f"process_2_{os.environ.get('PYTEST_XDIST_WORKER', 0)}", Command="Get-ChildItem", executable="powershell") _run.add_process(f"process_3_{os.environ.get('PYTEST_XDIST_WORKER', 0)}", Command="exit 0", executable="powershell") - sender(_run._sv_obj._local_staging_file.parents[1], 1, 10, ["folders", "runs", "alerts"]) + sender(_run._sv_obj._local_staging_file.parents[1], 1, 10, ["folders", "runs", "alerts"], throw_exceptions=True) @pytest.mark.executor diff --git a/tests/unit/test_event_alert.py b/tests/unit/test_event_alert.py index 254a44d7..2e4d7722 100644 --- a/tests/unit/test_event_alert.py +++ b/tests/unit/test_event_alert.py @@ -56,7 +56,7 @@ def test_event_alert_creation_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"events_alert_{_uuid}" assert _local_data.get("notification") == "none" - _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -106,7 +106,7 @@ def test_event_alert_modification_offline(offline_cache_setup) -> None: description=None ) _alert.commit() - _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -130,7 +130,7 @@ def test_event_alert_modification_offline(offline_cache_setup) -> None: _local_data = json.load(in_f) assert _local_data.get("description") == "updated!" - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) _online_alert.refresh() diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 41c2d2c1..1839f067 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -56,7 +56,7 @@ def test_events_creation_offline(offline_cache_setup) -> None: assert _local_data.get("events")[0].get("message") == "This is a test!" assert _local_data.get("events")[0].get("timestamp") == _timestamp - _id_mapping = sender(_events._local_staging_file.parents[1], 1, 10, ["folders", "runs", "events"]) + _id_mapping = sender(_events._local_staging_file.parents[1], 1, 10, ["folders", "runs", "events"], throw_exceptions=True) time.sleep(1) # Get online version of events diff --git a/tests/unit/test_file_artifact.py b/tests/unit/test_file_artifact.py index e59867b9..ad736c3a 100644 --- a/tests/unit/test_file_artifact.py +++ b/tests/unit/test_file_artifact.py @@ -103,7 +103,7 @@ def test_file_artifact_creation_offline(offline_cache_setup, snapshot) -> None: # If snapshot, check artifact definition file and a copy of the actual file exist in staging area assert len(list(_artifact._local_staging_file.parent.iterdir())) == 2 if snapshot else 1 - _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10, throw_exceptions=True) time.sleep(1) # Check file(s) deleted after upload @@ -158,12 +158,11 @@ def test_file_artifact_creation_offline_updated(offline_cache_setup, caplog, sna out_f.write("File changed!") if not snapshot: - with caplog.at_level(logging.ERROR): - _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) - assert "The SHA256 you specified did not match the calculated checksum." in caplog.text + with pytest.raises(RuntimeError, match="The SHA256 you specified did not match the calculated checksum."): + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10, throw_exceptions=True) return else: - _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10, throw_exceptions=True) time.sleep(1) _online_artifact = Artifact(_id_mapping[_artifact.id]) diff --git a/tests/unit/test_file_storage.py b/tests/unit/test_file_storage.py index 57eb0d24..1c5c9ce6 100644 --- a/tests/unit/test_file_storage.py +++ b/tests/unit/test_file_storage.py @@ -38,7 +38,7 @@ def test_create_file_storage_offline(offline_cache_setup) -> None: assert _local_data.get("is_enabled") == False assert _local_data.get("is_default") == False - _id_mapping = sender(_storage._local_staging_file.parents[1], 1, 10, ["storage"]) + _id_mapping = sender(_storage._local_staging_file.parents[1], 1, 10, ["storage"], throw_exceptions=True) time.sleep(1) _online_storage = FileStorage(_id_mapping.get(_storage.id)) assert _online_storage.name == _uuid diff --git a/tests/unit/test_folder.py b/tests/unit/test_folder.py index e234dfe6..7004551f 100644 --- a/tests/unit/test_folder.py +++ b/tests/unit/test_folder.py @@ -42,7 +42,7 @@ def test_folder_creation_offline(offline_cache_setup) -> None: assert _folder._local_staging_file.name.split(".")[0] == _folder.id assert _local_data.get("path", None) == _path - sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"], throw_exceptions=True) time.sleep(1) client = Client() @@ -96,7 +96,7 @@ def test_folder_modification_offline(offline_cache_setup) -> None: _folder = Folder.new(path=_path, offline=True) _folder.commit() - sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"], throw_exceptions=True) time.sleep(1) client = Client() @@ -115,7 +115,7 @@ def test_folder_modification_offline(offline_cache_setup) -> None: assert _local_data.get("description", None) == _description assert _local_data.get("tags", None) == _tags - sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"], throw_exceptions=True) time.sleep(1) _folder_online.refresh() diff --git a/tests/unit/test_grids.py b/tests/unit/test_grids.py index a3764d96..91c5a05e 100644 --- a/tests/unit/test_grids.py +++ b/tests/unit/test_grids.py @@ -72,7 +72,7 @@ def test_grid_creation_offline() -> None: assert _local_data.get("runs", [None])[0] == [_run.id, "A"] npt.assert_array_equal(numpy.array(_local_data.get("grid")), _grid_def) - _id_mapping = sender(_grid._local_staging_file.parents[1], 1, 10, ["folders", "runs", "grids"]) + _id_mapping = sender(_grid._local_staging_file.parents[1], 1, 10, ["folders", "runs", "grids"], throw_exceptions=True) time.sleep(1) # Get online version of grid _online_grid = Grid(_id_mapping.get(_grid.id)) @@ -184,7 +184,7 @@ def test_grid_metrics_creation_offline() -> None: _metrics.commit() _run.status = "completed" _run.commit() - _id_mapping = sender(_grid._local_staging_file.parents[1], 1, 10, ["folders", "runs", "grids", "grid_metrics"]) + _id_mapping = sender(_grid._local_staging_file.parents[1], 1, 10, ["folders", "runs", "grids", "grid_metrics"], throw_exceptions=True) time.sleep(1) # Online metrics assert list(GridMetrics.get(runs=[_id_mapping[_run.id]], metrics=["A"], step=_step)) diff --git a/tests/unit/test_metric_range_alert.py b/tests/unit/test_metric_range_alert.py index 3707e0da..56347eff 100644 --- a/tests/unit/test_metric_range_alert.py +++ b/tests/unit/test_metric_range_alert.py @@ -62,7 +62,7 @@ def test_metric_range_alert_creation_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"metrics_range_alert_{_uuid}" assert _local_data.get("notification") == "none" assert _local_data.get("alert").get("range_low") == 10 - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -124,7 +124,7 @@ def test_metric_range_alert_modification_offline(offline_cache_setup) -> None: offline=True ) _alert.commit() - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -149,7 +149,7 @@ def test_metric_range_alert_modification_offline(offline_cache_setup) -> None: _local_data = json.load(in_f) assert _local_data.get("description") == "updated!" - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) _online_alert.refresh() diff --git a/tests/unit/test_metric_threshold_alert.py b/tests/unit/test_metric_threshold_alert.py index 014cd8d9..8f04c698 100644 --- a/tests/unit/test_metric_threshold_alert.py +++ b/tests/unit/test_metric_threshold_alert.py @@ -61,7 +61,7 @@ def test_metric_threshold_alert_creation_offline(offline_cache_setup) -> None: assert _local_data.get("notification") == "none" assert _local_data.get("alert").get("threshold") == 10 - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -123,7 +123,7 @@ def test_metric_threshold_alert_modification_offline(offline_cache_setup) -> Non ) _alert.commit() - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -149,7 +149,7 @@ def test_metric_threshold_alert_modification_offline(offline_cache_setup) -> Non _local_data = json.load(in_f) assert _local_data.get("description") == "updated!" - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) _online_alert.refresh() diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index 4d2823a7..fea65482 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -88,7 +88,7 @@ def test_metrics_creation_offline(offline_cache_setup) -> None: assert _local_data.get("metrics")[0].get("step") == _step assert _local_data.get("metrics")[0].get("time") == _time - _id_mapping = sender(_metrics._local_staging_file.parents[1], 1, 10, ["folders", "runs", "metrics"]) + _id_mapping = sender(_metrics._local_staging_file.parents[1], 1, 10, ["folders", "runs", "metrics"], throw_exceptions=True) time.sleep(1) # Get online version of metrics diff --git a/tests/unit/test_object_artifact.py b/tests/unit/test_object_artifact.py index 0dfb5af1..b45eef1a 100644 --- a/tests/unit/test_object_artifact.py +++ b/tests/unit/test_object_artifact.py @@ -63,7 +63,7 @@ def test_object_artifact_creation_offline(offline_cache_setup) -> None: assert _local_data.get("mime_type") == "application/vnd.simvue.numpy.v1" assert _local_data.get("runs") == {_run.id: "input"} - _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10, throw_exceptions=True) time.sleep(1) _online_artifact = Artifact(_id_mapping.get(_artifact.id)) diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index 366dc7c9..395b658a 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -41,7 +41,7 @@ def test_run_creation_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"simvue_offline_run_{_uuid}" assert _local_data.get("folder") == _folder_name - sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve run @@ -119,7 +119,7 @@ def test_run_modification_offline(offline_cache_setup) -> None: assert _new_run.description == "Simvue test run" assert _new_run.name == "simvue_test_run" - sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve run @@ -139,7 +139,7 @@ def test_run_modification_offline(offline_cache_setup) -> None: _online_run.refresh() assert _online_run.tags == [] - sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"], throw_exceptions=True) time.sleep(1) _online_run.refresh() diff --git a/tests/unit/test_s3_storage.py b/tests/unit/test_s3_storage.py index 8fc96b45..2246e98a 100644 --- a/tests/unit/test_s3_storage.py +++ b/tests/unit/test_s3_storage.py @@ -71,7 +71,7 @@ def test_create_s3_offline(offline_cache_setup) -> None: assert not _local_data.get("user", None) assert not _local_data.get("usage", None) - _id_mapping = sender(_storage._local_staging_file.parents[1], 1, 10, ["storage"]) + _id_mapping = sender(_storage._local_staging_file.parents[1], 1, 10, ["storage"], throw_exceptions=True) _online_id = _id_mapping[_storage.id] time.sleep(1) diff --git a/tests/unit/test_sender.py b/tests/unit/test_sender.py index 0c8555a0..5d0933e6 100644 --- a/tests/unit/test_sender.py +++ b/tests/unit/test_sender.py @@ -13,11 +13,12 @@ import pathlib import requests +@pytest.mark.parametrize("throw_exceptions", (True, False)) @pytest.mark.parametrize("retry_failed_uploads", (True, False)) @pytest.mark.parametrize("parallel", (True, False)) @pytest.mark.offline -def test_sender_exception_handling(offline_cache_setup, caplog, retry_failed_uploads, parallel): +def test_sender_exception_handling(offline_cache_setup, caplog, throw_exceptions, retry_failed_uploads, parallel): # Create something which will produce an error when sent, eg a metric with invalid run ID for i in range(5): _metrics = Metrics.new( @@ -33,6 +34,11 @@ def test_sender_exception_handling(offline_cache_setup, caplog, retry_failed_upl offline=True ) _metrics.commit() + + if throw_exceptions: + with pytest.raises(ValueError): + sender(throw_exceptions=True, threading_threshold=1 if parallel else 10) + return with caplog.at_level(logging.ERROR): sender(threading_threshold=1 if parallel else 10) diff --git a/tests/unit/test_tag.py b/tests/unit/test_tag.py index 4959f3f7..381234d2 100644 --- a/tests/unit/test_tag.py +++ b/tests/unit/test_tag.py @@ -35,7 +35,7 @@ def test_tag_creation_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"test_tag_{_uuid}" - _id_mapping = sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"]) + _id_mapping = sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"], throw_exceptions=True) time.sleep(1) _online_id = _id_mapping.get(_tag.id) @@ -78,7 +78,7 @@ def test_tag_modification_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"test_tag_{_uuid}" - _id_mapping = sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"]) + _id_mapping = sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"], throw_exceptions=True) _online_id = _id_mapping.get(_tag.id) _online_tag = Tag(_online_id) @@ -101,7 +101,7 @@ def test_tag_modification_offline(offline_cache_setup) -> None: assert pydantic.color.parse_str(_local_data.get("colour")).r == 250 / 255 assert _local_data.get("description") == "modified test tag" - sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"]) + sender(_tag._local_staging_file.parents[1], 1, 10, ["tags"], throw_exceptions=True) time.sleep(1) # Check online version is updated diff --git a/tests/unit/test_tenant.py b/tests/unit/test_tenant.py index 38e3f9e3..04684467 100644 --- a/tests/unit/test_tenant.py +++ b/tests/unit/test_tenant.py @@ -40,7 +40,7 @@ def test_create_tenant_offline(offline_cache_setup) -> None: assert _local_data.get("name") == _uuid assert _local_data.get("is_enabled") == True - _id_mapping = sender(_new_tenant._local_staging_file.parents[1], 1, 10, ["tenants"]) + _id_mapping = sender(_new_tenant._local_staging_file.parents[1], 1, 10, ["tenants"], throw_exceptions=True) time.sleep(1) _online_user = Tenant(_id_mapping.get(_new_tenant.id)) assert _online_user.name == _uuid diff --git a/tests/unit/test_user.py b/tests/unit/test_user.py index 5a23349a..a53f3cfd 100644 --- a/tests/unit/test_user.py +++ b/tests/unit/test_user.py @@ -62,7 +62,7 @@ def test_create_user_offline(offline_cache_setup) -> None: assert _local_data.get("fullname") == "Joe Bloggs" assert _local_data.get("email") == "jbloggs@simvue.io" - _id_mapping = sender(_user._local_staging_file.parents[1], 1, 10, ["users"]) + _id_mapping = sender(_user._local_staging_file.parents[1], 1, 10, ["users"], throw_exceptions=True) time.sleep(1) _online_user = User(_id_mapping.get(_user.id)) assert _online_user.username == "jbloggs" diff --git a/tests/unit/test_user_alert.py b/tests/unit/test_user_alert.py index 7984ecce..f1f1acea 100644 --- a/tests/unit/test_user_alert.py +++ b/tests/unit/test_user_alert.py @@ -46,7 +46,7 @@ def test_user_alert_creation_offline(offline_cache_setup) -> None: assert _local_data.get("name") == f"users_alert_{_uuid}" assert _local_data.get("notification") == "none" - _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) _online_id = _alert._local_staging_file.parents[1].joinpath("server_ids", f"{_alert._local_staging_file.name.split('.')[0]}.txt").read_text() @@ -94,7 +94,7 @@ def test_user_alert_modification_offline(offline_cache_setup) -> None: ) _alert.commit() - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Get online ID and retrieve alert @@ -117,7 +117,7 @@ def test_user_alert_modification_offline(offline_cache_setup) -> None: with _alert._local_staging_file.open() as in_f: _local_data = json.load(in_f) assert _local_data.get("description") == "updated!" - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) _online_alert.refresh() @@ -191,7 +191,7 @@ def test_user_alert_status_offline(offline_cache_setup) -> None: _run.alerts = [_alert.id] _run.commit() - _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["folders", "runs", "alerts"]) + _id_mapping = sender(_alert._local_staging_file.parents[1], 1, 10, ["folders", "runs", "alerts"], throw_exceptions=True) time.sleep(1) # Get online aler, check status is not set @@ -206,7 +206,7 @@ def test_user_alert_status_offline(offline_cache_setup) -> None: _online_alert.refresh() assert not _online_alert.get_status(run_id=_id_mapping.get(_run.id)) - sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"]) + sender(_alert._local_staging_file.parents[1], 1, 10, ["alerts"], throw_exceptions=True) time.sleep(1) # Check online status has been updated