diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validator.py index ce34842cb9..6f9b07158c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_validator.py @@ -5,7 +5,6 @@ from monitoring.monitorlib.fetch.rid import ISA, FetchedISA, FetchedISAs from monitoring.monitorlib.mutate.rid import ChangedISA from monitoring.monitorlib.rid import RIDVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.scenarios.scenario import ( GenericTestScenario, PendingCheck, @@ -43,7 +42,6 @@ def __init__( def _fail_sub_check( self, _sub_check: PendingCheck, _summary: str, _details: str, t_dss: datetime ) -> None: - """Fails with Medium severity the sub_check and with High severity the main check.""" _sub_check.record_failed( summary=_summary, @@ -53,7 +51,6 @@ def _fail_sub_check( self._main_check.record_failed( summary=f"ISA request succeeded, but the DSS response is not valid: {_summary}", - severity=Severity.High, details=_details, query_timestamps=[t_dss], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py index ea4721cb30..d0b2de898b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/subscription_validation.py @@ -3,7 +3,6 @@ from monitoring.monitorlib.mutate.rid import ChangedSubscription from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource @@ -203,8 +202,7 @@ def _check_properly_truncated( else: check.record_failed( "Created record subscription has not been properly truncated to 24 hours", - Severity.Medium, - f"{self._dss.participant_id} DSS instance has returned a non-properly truncated subscription " + details=f"{self._dss.participant_id} DSS instance has returned a non-properly truncated subscription " f"(duration: {duration}) " f"when the expectation was either to fail or to truncate at 24 hours.", query_timestamps=[changed.query.request.timestamp], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/token_validation.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/token_validation.py index 51efc850a2..652d7498d3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/token_validation.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/token_validation.py @@ -18,7 +18,6 @@ ) from monitoring.monitorlib.rid import RIDVersion from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource @@ -105,8 +104,7 @@ def _wrong_auth_put(self): if put_wrong_scope.dss_query.success: check.record_failed( "Read scope can create ISA", - Severity.High, - f"Attempting to create ISA {self._isa_id} with read scope returned {put_wrong_scope.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with read scope returned {put_wrong_scope.dss_query.status_code}", query_timestamps=[ put_wrong_scope.dss_query.query.request.timestamp ], @@ -125,8 +123,7 @@ def _wrong_auth_put(self): if put_no_token.dss_query.success: check.record_failed( "Could create an ISA without a token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with no token returned {put_no_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with no token returned {put_no_token.dss_query.status_code}", query_timestamps=[put_no_token.dss_query.query.request.timestamp], ) @@ -143,8 +140,7 @@ def _wrong_auth_put(self): if put_fake_token.dss_query.success: check.record_failed( "Could create an ISA with a fake token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with a fake token returned {put_fake_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with a fake token returned {put_fake_token.dss_query.status_code}", query_timestamps=[put_fake_token.dss_query.query.request.timestamp], ) @@ -175,8 +171,7 @@ def _wrong_auth_get(self): if get_no_token.success: check.record_failed( "Could read an ISA without a token", - Severity.High, - f"Attempting to read ISA {self._isa_id} with no token returned {get_no_token.status_code}", + details=f"Attempting to read ISA {self._isa_id} with no token returned {get_no_token.status_code}", query_timestamps=[get_no_token.query.request.timestamp], ) @@ -188,8 +183,7 @@ def _wrong_auth_get(self): if get_fake_token.success: check.record_failed( "Could read an ISA with a fake token", - Severity.High, - f"Attempting to read ISA {self._isa_id} with a fake token returned {get_fake_token.status_code}", + details=f"Attempting to read ISA {self._isa_id} with a fake token returned {get_fake_token.status_code}", query_timestamps=[get_fake_token.query.request.timestamp], ) @@ -206,8 +200,7 @@ def _wrong_auth_mutate(self): if mutate_wrong_scope.dss_query.success: check.record_failed( "Read scope can mutate an ISA", - Severity.High, - f"Attempting to create ISA {self._isa_id} with read scope returned {mutate_wrong_scope.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with read scope returned {mutate_wrong_scope.dss_query.status_code}", query_timestamps=[ mutate_wrong_scope.dss_query.query.request.timestamp ], @@ -225,8 +218,7 @@ def _wrong_auth_mutate(self): if mutate_no_token.dss_query.success: check.record_failed( "Could mutate an ISA without a token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with no token returned {mutate_no_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with no token returned {mutate_no_token.dss_query.status_code}", query_timestamps=[ mutate_no_token.dss_query.query.request.timestamp ], @@ -244,8 +236,7 @@ def _wrong_auth_mutate(self): if mutate_fake_token.dss_query.success: check.record_failed( "Could mutate an ISA with a fake token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with a fake token returned {mutate_fake_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with a fake token returned {mutate_fake_token.dss_query.status_code}", query_timestamps=[ mutate_fake_token.dss_query.query.request.timestamp ], @@ -262,8 +253,7 @@ def _wrong_auth_delete(self): if del_wrong_scope.dss_query.success: check.record_failed( "Read scope can delete an ISA", - Severity.High, - f"Attempting to delete ISA {self._isa_id} with read scope returned {del_wrong_scope.dss_query.status_code}", + details=f"Attempting to delete ISA {self._isa_id} with read scope returned {del_wrong_scope.dss_query.status_code}", query_timestamps=[ del_wrong_scope.dss_query.query.request.timestamp ], @@ -284,8 +274,7 @@ def _wrong_auth_delete(self): if del_no_token.dss_query.success: check.record_failed( "Could mutate an ISA without a token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with no token returned {del_no_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with no token returned {del_no_token.dss_query.status_code}", query_timestamps=[del_no_token.dss_query.query.request.timestamp], ) @@ -304,8 +293,7 @@ def _wrong_auth_delete(self): if del_fake_token.dss_query.success: check.record_failed( "Could delete an ISA with a fake token", - Severity.High, - f"Attempting to create ISA {self._isa_id} with a fake token returned {del_fake_token.dss_query.status_code}", + details=f"Attempting to create ISA {self._isa_id} with a fake token returned {del_fake_token.dss_query.status_code}", query_timestamps=[del_fake_token.dss_query.query.request.timestamp], ) @@ -325,8 +313,7 @@ def _wrong_auth_search(self): if not search_ok.success: check.record_failed( "Search request failed although a valid token was used", - Severity.High, - f"Attempting to search ISAs with a valid token returned failure code: {search_ok.query.status_code}", + details=f"Attempting to search ISAs with a valid token returned failure code: {search_ok.query.status_code}", query_timestamps=[search_ok.query.request.timestamp], ) @@ -345,8 +332,7 @@ def _wrong_auth_search(self): if search_wrong_token.success: check.record_failed( "Search endpoint returned successfully without a token", - Severity.High, - f"Attempting to search ISAs with invalid token returned successful query: {search_wrong_token.query.status_code}", + details=f"Attempting to search ISAs with invalid token returned successful query: {search_wrong_token.query.status_code}", query_timestamps=[search_wrong_token.query.request.timestamp], ) @@ -365,8 +351,7 @@ def _wrong_auth_search(self): if search_no_token.success: check.record_failed( "Search endpoint returned successfully without a token", - Severity.High, - f"Attempting to search ISAs with no token returned successful query: {search_no_token.query.status_code}", + details=f"Attempting to search ISAs with no token returned successful query: {search_no_token.query.status_code}", query_timestamps=[search_no_token.query.request.timestamp], ) @@ -379,8 +364,7 @@ def _delete_isa(self): if not del_isa_ok.dss_query.success: check.record_failed( "Could not delete ISA with valid credentials", - Severity.High, - f"Attempting to delete ISA {self._isa_id} returned {del_isa_ok.dss_query.status_code}", + details=f"Attempting to delete ISA {self._isa_id} returned {del_isa_ok.dss_query.status_code}", query_timestamps=[del_isa_ok.dss_query.query.request.timestamp], ) @@ -398,8 +382,7 @@ def _delete_isa_if_exists(self): if not fetched.success and fetched.status_code != 404: check.record_failed( "ISA information could not be retrieved", - Severity.High, - f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}", + details=f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}", query_timestamps=[fetched.query.request.timestamp], ) @@ -420,8 +403,7 @@ def _delete_isa_if_exists(self): if not deleted.dss_query.success: check.record_failed( "Could not delete pre-existing ISA", - Severity.High, - f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}", + details=f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}", query_timestamps=[deleted.dss_query.query.request.timestamp], ) self._verify_notifications(deleted.notifications) @@ -437,8 +419,7 @@ def _verify_notifications(self, notifications: Dict[str, ISAChangeNotification]) if not notification.success: check.record_failed( "Could not notify ISA subscriber", - Severity.Medium, - f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}", + details=f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}", query_timestamps=[notification.query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py index 036876ed93..4975829685 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/utils.py @@ -4,7 +4,6 @@ from monitoring.monitorlib.infrastructure import UTMClientSession from monitoring.monitorlib.mutate import rid as mutate from monitoring.monitorlib.rid import RIDVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.scenarios.scenario import GenericTestScenario @@ -41,8 +40,7 @@ def delete_isa_if_exists( if not fetched.success and fetched.status_code != 404: check.record_failed( "ISA information could not be retrieved", - Severity.High, - f"{participant_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", + details=f"{participant_id} DSS instance returned {fetched.status_code} when queried for ISA {isa_id}", query_timestamps=[fetched.query.request.timestamp], ) @@ -61,8 +59,7 @@ def delete_isa_if_exists( if not deleted.dss_query.success: check.record_failed( "Could not delete pre-existing ISA", - Severity.High, - f"Attempting to delete ISA {isa_id} from the {participant_id} DSS returned error {deleted.dss_query.status_code}", + details=f"Attempting to delete ISA {isa_id} from the {participant_id} DSS returned error {deleted.dss_query.status_code}", query_timestamps=[deleted.dss_query.query.request.timestamp], ) @@ -82,7 +79,6 @@ def delete_isa_if_exists( if not notification.success: check.record_failed( "Could not notify ISA subscriber", - Severity.Medium, - f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", + details=f"Attempting to notify subscriber for ISA {isa_id} at {subscriber_url} resulted in {notification.status_code}", query_timestamps=[notification.query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 86b2ac6337..973236da8f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -23,7 +23,6 @@ ) from monitoring.monitorlib.rid import RIDVersion from monitoring.monitorlib.temporal import Time -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.configurations.configuration import ParticipantID from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance from monitoring.uss_qualifier.resources.netrid import NetRIDServiceProviders @@ -351,7 +350,6 @@ def _evaluate_observation( check.record_failed( summary="Observation failed", details=f"When queried for an observation in {geo.rect_str(rect)}, {observer.participant_id} returned code {query.status_code}", - severity=Severity.Medium, query_timestamps=[query.request.timestamp], ) return @@ -589,7 +587,6 @@ def _evaluate_clusters_observation( ): check.record_failed( summary=f"Cluster display area is smaller than {self._rid_version.min_cluster_size_percent} % of the view area required", - severity=Severity.Medium, details=f"Cluster covers {cluster.area_sqm} sqm and the view area is {view_area_sqm} sqm. Cluster area covers {cluster_area_sqm_percent} % of the view area and is less than the required {self._rid_version.min_cluster_size_percent} %", ) @@ -615,14 +612,12 @@ def _evaluate_clusters_observation( # Missing flight check.record_failed( summary="Error while evaluating clustered area view. Missing flight", - severity=Severity.Medium, details=f"{expected_count - clustered_flight_count} (~{uncertain_count}) missing flight(s)", ) elif clustered_flight_count > expected_count + uncertain_count: # Unexpected flight check.record_failed( summary="Error while evaluating clustered area view. Unexpected flight", - severity=Severity.Medium, details=f"{clustered_flight_count - expected_count} (~{uncertain_count}) unexpected flight(s)", ) elif clustered_flight_count == expected_count: @@ -680,7 +675,6 @@ def _evaluate_clusters_obfuscation_distance( # Cluster has a too small distance to the edge check.record_failed( summary="Error while evaluating cluster obfuscation. Cluster does not comply with the minimum obfuscation distance.", - severity=Severity.Medium, details=f"Cluster {clusters[c_idx].corners} ({clusters[c_idx].number_of_flights} flights): too small dimensions. Height: {cluster_height}m, width: {cluster_width}m, minimum: {min_dim}m.", ) @@ -720,7 +714,6 @@ def _evaluate_obfuscated_clusters_observation( # Flight was not obfuscated check.record_failed( summary="Error while evaluating obfuscation of individual flights. Flight was not obfuscated: it is at the center of the cluster.", - severity=Severity.Medium, details=f"Flight {injected_flight.flight.injection_id}: distance between cluster center ({center}) and flight telemetry position ({flight_pos} at {tel.timestamp}) is {distance} meters.", ) @@ -743,7 +736,6 @@ def _evaluate_obfuscated_clusters_observation( # Flight has no telemetry point belonging to a cluster check.record_failed( summary="Error while evaluating obfuscation of individual flights. Flight was outside of all clusters.", - severity=Severity.Medium, details=f"Flight {injected_flight.flight.injection_id}: has no telemetry position that belong to a cluster.", ) @@ -1111,7 +1103,6 @@ def _evaluate_sp_observation( if not sp_observation.dss_isa_query.success: check.record_failed( summary="Could not query ISAs from DSS", - severity=Severity.Medium, details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}", query_timestamps=[ sp_observation.dss_isa_query.query.request.initiated_at.datetime @@ -1322,7 +1313,6 @@ def _evaluate_flight_presence( check.record_failed( summary="Flight observed before it started", details=f"Flight {expected_flight.flight.injection_id} injected into {expected_flight.uss_participant_id} was observed by {observer_participant_id} at {t_response.isoformat()} before that flight should have started at {t_min.isoformat()}", - severity=Severity.Medium, query_timestamps=query_timestamps + [expected_flight.query_timestamp], ) @@ -1346,7 +1336,6 @@ def _evaluate_flight_presence( check.record_failed( summary="Flight still observed long after it ended", details=f"Flight {expected_flight.flight.injection_id} injected into {expected_flight.uss_participant_id} was observed by {observer_participant_id} at {t_response.isoformat()} after it ended at {t_max.isoformat()}", - severity=Severity.Medium, query_timestamps=query_timestamps + [expected_flight.query_timestamp], ) @@ -1370,7 +1359,6 @@ def _evaluate_flight_presence( check.record_failed( summary="Expected flight not observed", details=f"Flight {expected_flight.flight.injection_id} injected into {expected_flight.uss_participant_id} was not found in the observation by {observer_participant_id} at {t_response.isoformat()} even though it should have been active from {t_min.isoformat()} to {t_max.isoformat()}", - severity=Severity.Medium, query_timestamps=query_timestamps + [expected_flight.query_timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 50cf12c5db..b352e40195 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -22,7 +22,6 @@ from monitoring.monitorlib.mutate import rid as mutate from monitoring.monitorlib.mutate.rid import ChangedSubscription, ISAChange from monitoring.monitorlib.rid import RIDVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstance from monitoring.uss_qualifier.scenarios.astm.netrid.common.dss.isa_validator import ( ISAValidator, @@ -66,7 +65,6 @@ def _handle_query_error( self._scenario.record_query(q) check.record_failed( summary=f"Error when querying DSS", - severity=Severity.High, details=f"{str(e)}\n\nStack trace:\n{e.stacktrace}", query_timestamps=[q.request.timestamp for q in e.queries], ) @@ -77,7 +75,6 @@ def handle_query_result( q: RIDQuery, fail_msg: str, required_status_code: Optional[Set[int]] = None, - severity: Severity = Severity.High, fail_details: Optional[str] = None, ): """ @@ -96,7 +93,6 @@ def handle_query_result( :param q: the query to check :param fail_msg: the message to use when failing the check :param required_status_code: the set of status codes that are considered successful. If this is None then success is defined by `q.success` - :param severity: the severity of the check failure :param fail_details: the details passed to check.record_fail """ self._scenario.record_query(q.query) @@ -106,7 +102,6 @@ def handle_query_result( ): check.record_failed( summary=fail_msg, - severity=severity, details=( f"{fail_details}\n{q.status_code} response: " + "\n".join(q.errors) if fail_details is not None @@ -157,8 +152,7 @@ def search_isas( details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) sub_check.record_failed( "Search ISA response format was invalid", - Severity.Medium, - "Found the following schema validation errors in the DSS response:\n" + details="Found the following schema validation errors in the DSS response:\n" + details, query_timestamps=[t_dss], ) @@ -224,7 +218,6 @@ def get_isa( if isa_id != isa.isa.id: check.record_failed( summary=f"DSS did not return correct ISA", - severity=Severity.High, details=f"Expected ISA ID {isa_id} but got {isa.id}", query_timestamps=[isa.query.request.timestamp], ) @@ -427,9 +420,7 @@ def del_isa( details = "\n".join(f"[{e.json_path}] {e.message}" for e in errors) sub_check.record_failed( "Delete ISA response format was invalid", - Severity.Medium, - "Found the following schema validation errors in the DSS response:\n" - + details, + details=f"Found the following schema validation errors in the DSS response:\n{details}", query_timestamps=[t_dss], ) @@ -446,7 +437,6 @@ def _fail_sub_check( ) main_check.record_failed( summary=f"Delete ISA request succeeded, but the DSS response is not valid: {_summary}", - severity=Severity.High, details=_details, query_timestamps=[t_dss], ) @@ -537,7 +527,7 @@ def cleanup_isa( ) self.handle_query_result( - check, isa, f"Failed to get ISA {isa_id}", {404, 200}, Severity.Medium + check, isa, f"Failed to get ISA {isa_id}", {404, 200} ) if isa.status_code == 404: @@ -552,11 +542,7 @@ def cleanup_isa( ) self.handle_query_result( - check, - del_isa.dss_query, - f"Failed to delete ISA {isa_id}", - {404, 200}, - Severity.Medium, + check, del_isa.dss_query, f"Failed to delete ISA {isa_id}", {404, 200} ) return del_isa @@ -687,7 +673,6 @@ def get_sub( if sub_id != sub.subscription.id: check.record_failed( summary=f"DSS did not return correct subscription", - severity=Severity.High, details=f"Expected Subscription ID {sub_id} but got {sub.subscription.id}", query_timestamps=[sub.query.request.timestamp], ) @@ -927,7 +912,6 @@ def cleanup_subs_in_area( del_sub, f"Failed to delete subscription {sub}", {404, 200}, - Severity.Medium, ) except QueryError as e: self._handle_query_error(check, e) @@ -956,11 +940,7 @@ def cleanup_sub( ) self.handle_query_result( - check, - sub, - f"Failed to get subscription {sub_id}", - {404, 200}, - Severity.Medium, + check, sub, f"Failed to get subscription {sub_id}", {404, 200} ) if sub.status_code == 404: @@ -982,7 +962,6 @@ def cleanup_sub( del_sub, f"Failed to delete subscription {sub_id}", {404, 200}, - Severity.Medium, ) return del_sub @@ -1034,12 +1013,6 @@ def raw_request_with_expected_code( else: raise ValueError(f"Unknown RID version: {self._dss.rid_version}") - self.handle_query_result( - check, - rid_query, - fail_msg, - expected_error_codes, - Severity.Medium, - ) + self.handle_query_result(check, rid_query, fail_msg, expected_error_codes) return rid_query diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_clustering.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_clustering.md index 3090421fb4..11e51ac7b1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_clustering.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_clustering.md @@ -2,28 +2,28 @@ This fragment is implemented in `display_data_evaluator.py:RIDObservationEvaluator._evaluate_clusters_observation`. -#### Minimal obfuscation distance of individual flights check +#### ⚠️ Minimal obfuscation distance of individual flights check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0490](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall obfuscate individual UAs within a cluster. If a cluster with a single flight has a width or height smaller than 2 * *NetMinObfuscationDistance* (meaning that the USS did not buffer the single point by *NetMinObfuscationDistance* in all directions when forming the cluster bounds), this test will fail. -#### Individual flights obfuscation check +#### ⚠️ Individual flights obfuscation check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0490](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall obfuscate individual UAs within a cluster. If a cluster with a single flight has its center equal to the position of the flight, this test will fail. If a cluster with a single flight does not actually encompass the flight, this test will fail. -#### Minimal obfuscation distance of multiple flights clusters check +#### ⚠️ Minimal obfuscation distance of multiple flights clusters check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0480](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area. If a cluster with multiples flights has a width or height smaller than 2 * *NetMinObfuscationDistance*, this test will fail. A larger width or height might still be incorrect, but the test cannot at the moment determine this. -#### Clustering count check +#### ⚠️ Clustering count check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0480](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area. Taking into account the propagation time of the injected flights, if the total number of clustered UAs when this value is expected to be stable is not correct, this test will fail. -#### Minimal display area of clusters check +#### ⚠️ Minimal display area of clusters check For a display area with a diagonal greather than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v19.NET0480](../../../../requirements/astm/f3411/v19.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area covering no less than *NetMinClusterSize* percent of the display area size. This check validates that the display area of a cluster, measured and provided in square meters by the test harness, is no less than *NetMinClusterSize* percent of the display area. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_flight_presence.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_flight_presence.md index 445a9dc2a5..f09bbf1f29 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_flight_presence.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/display_data_evaluator_flight_presence.md @@ -2,11 +2,11 @@ This fragment is implemented in `display_data_evaluator.py:RIDObservationEvaluator._evaluate_flight_presence`. -#### Premature flight check +#### ⚠️ Premature flight check The timestamps of the injected telemetry usually start in the future. If a flight with injected telemetry only in the future is observed prior to the timestamp of the first telemetry point, this check will fail because the SP does not satisfy **[interuss.automated_testing.rid.injection.ExpectedBehavior](../../../../requirements/interuss/automated_testing/rid/injection.md)**. -#### Missing flight check +#### ⚠️ Missing flight check **[astm.f3411.v19.NET0610](../../../../requirements/astm/f3411/v19.md)** requires that SPs make all UAS operations discoverable over the duration of the flight plus *NetMaxNearRealTimeDataPeriod*, so each injected flight should be observable during this time. If a flight is not observed during its appropriate time period, this check will fail. @@ -14,6 +14,6 @@ The timestamps of the injected telemetry usually start in the future. If a flig The identity of flights is determined by precisely matching the known injected positions. If the flight can be found, the USS may not have met **[astm.f3411.v19.NET0260,Table1,9](../../../../requirements/astm/f3411/v19.md)** or **[astm.f3411.v19.NET0260,Table1,10](../../../../requirements/astm/f3411/v19.md)** prescribing provision of position data consistent with the common data dictionary. -#### Lingering flight check +#### ⚠️ Lingering flight check **[astm.f3411.v19.NET0260,NearRealTime](../../../../requirements/astm/f3411/v19.md)** requires a SP to provide flights up to *NetMaxNearRealTimeDataPeriod* in the past, but an SP should preserve privacy and ensure relevancy by not sharing flights that are further in the past than this window. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md index 0810afcea1..319208e6c0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -95,14 +95,14 @@ The ISA search area parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md index dbd11670c3..27d660ee7e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_expiry.md @@ -48,14 +48,14 @@ that ISAs that are in the searched area but have expired should not be returned. The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md index e5ca28c906..ba90ffb4d9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_subscription_interactions.md @@ -141,14 +141,14 @@ If the ISA cannot be mutated, **[astm.f3411.v19.DSS0030,a](../../../../../requir When an ISA is mutated, the DSS must return the identifiers for any subscription that was made to the ISA, or be in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. -#### ⚠️ Subscription to an ISA has its notification index incremented after mutation check +#### 🛑 Subscription to an ISA has its notification index incremented after mutation check When an ISA is mutated, the DSS must increment the notification index of any subscription to that ISA, and return the up-to-date subscription in the response to the query mutating the ISA. Failure to do so means that the DSS is not properly implementing **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. -#### Subscription that only barely overlaps the ISA contains the ISA check +#### 🛑 Subscription that only barely overlaps the ISA contains the ISA check A subscription that is created for a volume that only barely overlaps with the previously created ISA should still contain the ISA in the reply from the server, otherwise the DSS does not comply with **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)** @@ -182,22 +182,22 @@ Notifications to any subscriber to the created ISA need to be successful. If a The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely while also deleting any subscription it might have created for the ISA's area. -#### Successful ISA query check +#### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -#### Removed pre-existing ISA check +#### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -#### Notified subscriber check +#### ⚠️ Notified subscriber check When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. -#### Successful subscription search query check +#### ⚠️ Successful subscription search query check **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. -#### Subscription can be deleted check +#### ⚠️ Subscription can be deleted check **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md index de52e6b8a9..27a16c68ea 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_validation.md @@ -73,14 +73,14 @@ the DSS is expected to reject the request, otherwise it fails to meet **[astm.f3 The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_simple.md index 00557d0d9f..e90dbed616 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_simple.md @@ -110,11 +110,11 @@ The subscription version generated by the DSS must be a lower-case alphanumeric Query and search for the created subscription in various ways -#### Get Subscription by ID check +#### 🛑 Get Subscription by ID check If the freshly created subscription cannot be queried using its ID, the DSS is failing to meet **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)**. -#### Search for all subscriptions in ISA area check +#### 🛑 Search for all subscriptions in ISA area check If the DSS fails to let us search in the area for which the subscription was just created, it is failing to meet **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. @@ -122,7 +122,7 @@ If the DSS fails to let us search in the area for which the subscription was jus If the created subscription is not returned in a search that covers the area it was created for, the DSS is not properly implementing **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. -#### No huge search area allowed check +#### 🛑 No huge search area allowed check In accordance with **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**, the DSS should not allow searches for areas that are too big. @@ -203,7 +203,7 @@ The subscription version generated by the DSS must be a lower-case alphanumeric This test step mutates the previously created subscription to verify that the DSS reacts properly: notably, it checks that the subscription version is updated, including for changes that are not directly visible, such as changing the subscription's footprint. -#### Subscription can be mutated check +#### 🛑 Subscription can be mutated check If a subscription cannot be modified with a valid set of parameters, the DSS is failing to meet **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. @@ -366,15 +366,15 @@ The subscription version generated by the DSS must be a lower-case alphanumeric Attempt to query and search for the deleted subscription in various ways -#### Query by subscription ID should fail check +#### 🛑 Query by subscription ID should fail check If the DSS provides a successful reply to a direct query for the deleted subscription, it is in violation of **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)**. -#### Search for all subscriptions in ISA area check +#### 🛑 Search for all subscriptions in ISA area check If the DSS fails to let us search in the area for which the subscription was just created, it is failing to meet **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. -#### Search area that represents a loop is not allowed check +#### 🛑 Search area that represents a loop is not allowed check The DSS should not allow us to search for subscriptions using a list of vertices describing a loop (first and last points in the list of vertices are the same), otherwise it is failing to meet **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)**. @@ -387,11 +387,11 @@ If the DSS returns the deleted subscription in a search that covers the area it The cleanup phase of this test scenario removes the subscription with the known test ID if it has not been removed before. -#### Subscription can be queried by ID check +#### 🛑 Subscription can be queried by ID check If the DSS cannot be queried for the existing test ID, the DSS is likely not implementing **[astm.f3411.v19.DSS0030,e](../../../../../requirements/astm/f3411/v19.md)** correctly. -#### Subscription can be deleted check +#### 🛑 Subscription can be deleted check An attempt to delete a subscription when the correct version is provided should succeed, otherwise the DSS is in violation of **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md index de38b52b7c..ff261de178 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/subscription_validation.md @@ -30,7 +30,7 @@ This step ensures that we remove any subscription that may already exist for the The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed. -#### Create up to the maximum allowed number of subscriptions in an area check +#### 🛑 Create up to the maximum allowed number of subscriptions in an area check As per **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**, the DSS API is expected to allow us to create multiple subscriptions. @@ -39,7 +39,7 @@ to create multiple subscriptions. Now, create an 11th one and expect it to fail. -#### Enforce maximum number of subscriptions for an area check +#### 🛑 Enforce maximum number of subscriptions for an area check If the DSS successfully creates an 11th Subscription in the same area instead of rejecting it, it will not have performed the Subscription count validation as defined in **[astm.f3411.v19.DSS0050](../../../../../requirements/astm/f3411/v19.md)** @@ -48,11 +48,11 @@ it will not have performed the Subscription count validation as defined in **[as Clean up any subscriptions created. -#### Successful subscription search query check +#### 🛑 Successful subscription search query check If the query for subscriptions fails, **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** was not met. -#### Subscription can be deleted check +#### 🛑 Subscription can be deleted check If the deletion attempt fails, **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** was not met. @@ -62,7 +62,7 @@ If the deletion attempt fails, **[astm.f3411.v19.DSS0030,d](../../../../../requi The test will attempt to create a subscription for 24 hours and 10 minutes, and expect this to fail with an HTTP 400 error. -#### Too-long subscription creation rejected check +#### 🛑 Too-long subscription creation rejected check **[astm.f3411.v19.DSS0060](../../../../../requirements/astm/f3411/v19.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). @@ -73,11 +73,11 @@ If creation of the too-long-duration subscription succeeds, the test expects tha The test will attempt to create a valid subscription of 23 hours and 59 minutes, and then increase its duration to 24 hours and 10 minutes, expecting this update to fail. -#### Valid subscription created check +#### 🛑 Valid subscription created check The ability to create a valid subscription is required in **[astm.f3411.v19.DSS0030,c](../../../../../requirements/astm/f3411/v19.md)**. -#### Subscription duration limited during update check +#### 🛑 Subscription duration limited during update check If the DSS allows a user to extend an existing, valid subscription to a duration exceeding that specified in **[astm.f3411.v19.DSS0060](../../../../../requirements/astm/f3411/v19.md)**, this check will fail. @@ -93,10 +93,10 @@ The ability to delete an existing subscription is required in **[astm.f3411.v19. The cleanup phase of this test scenario will remove any subscription that may have been created during the test and that intersects with the test ISA. -### Successful subscription search query check +### 🛑 Successful subscription search query check If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v19.DSS0030,f](../../../../../requirements/astm/f3411/v19.md)** was not met. -### Subscription can be deleted check +### 🛑 Subscription can be deleted check If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v19.DSS0030,d](../../../../../requirements/astm/f3411/v19.md)** was not met. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/clean_workspace.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/clean_workspace.md index 6f4f92fa82..204cecc145 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/clean_workspace.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/clean_workspace.md @@ -2,26 +2,26 @@ This page describes the content of a common test step that ensures a clean workspace for testing interactions with a DSS -## Successful ISA query check +## 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -## Removed pre-existing ISA check +## ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,d](../../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -## Notified subscriber check +## 🛑 Notified subscriber check When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v19.NET0730](../../../../../../requirements/astm/f3411/v19.md)** requirement to implement the POST ISAs endpoint isn't met. -## Successful subscription search query check +## 🛑 Successful subscription search query check **[astm.f3411.v19.DSS0030,f](../../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. -## Subscription can be queried by ID check +## 🛑 Subscription can be queried by ID check If the DSS cannot be queried for the existing test ID, the DSS is likely not implementing **[astm.f3411.v19.DSS0030,e](../../../../../../requirements/astm/f3411/v19.md)** correctly. -## Subscription can be deleted check +## 🛑 Subscription can be deleted check **[astm.f3411.v19.DSS0030,d](../../../../../../requirements/astm/f3411/v19.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md index e90b202c83..36b3144f3a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas.md @@ -3,30 +3,30 @@ This page describes the content of a common test step where a search for ISAs should be successful. See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISAs search response format check +## ⚠️ ISAs search response format check While F3411-19 does not explicitly require the implementation of the ISA search endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -## ISA ID matches check +## ⚠️ ISA ID matches check The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -## ISA version format check +## ⚠️ ISA version format check Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. -## ISA version matches check +## ⚠️ ISA version matches check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -## ISA start time matches check +## ⚠️ ISA start time matches check The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. -## ISA end time matches check +## ⚠️ ISA end time matches check The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v19.DSS0030,a](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. -## ISA URL matches check +## ⚠️ ISA URL matches check When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v19.DSS0030,a](../../../../../../requirements/astm/f3411/v19.md)** is not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas_miss.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas_miss.md index dc3bf6d02b..5208ec67bd 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas_miss.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/search_isas_miss.md @@ -5,10 +5,10 @@ relevant to the current scenario are not expected to be present. See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISAs search response format check +## ⚠️ ISAs search response format check While F3411-19 does not explicitly require the implementation of the ISA search endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -## ISA version format check +## ⚠️ ISA version format check Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/token_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/token_validation.md index 0e6cd73f7b..5f5960d5e0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/token_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/token_validation.md @@ -32,74 +32,74 @@ part of the test. This step attempts to create and read ISAs by providing both the correct and incorrect scopes, omitting the token or providing an invalid one, and expects the DSS to properly behave in each case. -#### Read scope cannot create an ISA check +#### 🛑 Read scope cannot create an ISA check If an ISA can be created with a scope that does not provide write permission, the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. -#### Missing token prevents creating an ISA check +#### 🛑 Missing token prevents creating an ISA check If an ISA can be created without a token being present in the request, the DSS is in violation of **[astm.f3411.v19.DSS0010](../../../../../requirements/astm/f3411/v19.md)**. -#### Fake token prevents creating an ISA check +#### 🛑 Fake token prevents creating an ISA check If an ISA can be created with an incorrect token in the request, the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)**. -#### Correct token and scope can create ISA check +#### 🛑 Correct token and scope can create ISA check If the ISA cannot be created when the proper credentials are presented, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. -#### Missing token prevents reading an ISA check +#### 🛑 Missing token prevents reading an ISA check If the ISA that was created can be accessed without a token being present in the request, the DSS is in violation of **[astm.f3411.v19.DSS0010](../../../../../requirements/astm/f3411/v19.md)** -#### Fake token prevents reading an ISA check +#### 🛑 Fake token prevents reading an ISA check If the ISA that was created can be accessed using a token with an invalid signature, the DSS is in violation of **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** -#### Read scope cannot mutate an ISA check +#### 🛑 Read scope cannot mutate an ISA check If the existing ISA can be mutated by using a read-only scope, the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** -#### Missing token prevents mutating an ISA check +#### 🛑 Missing token prevents mutating an ISA check If the existing ISA can be mutated without a token being provided, the DSS is in violation of **[astm.f3411.v19.DSS0010](../../../../../requirements/astm/f3411/v19.md)** -#### Proper token is allowed to search for ISAs check +#### 🛑 Proper token is allowed to search for ISAs check If a valid token is presented as part of the search request, and the search parameters are valid, the DSS must return a 200 response, or be in violation of **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Fake token cannot mutate an ISA check +#### 🛑 Fake token cannot mutate an ISA check If the existing ISA can be mutated by using an invalid token, the DSS is in violation of **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** -#### Fake token cannot search for ISAs check +#### 🛑 Fake token cannot search for ISAs check If the DSS accepts search queries with an invalid token, it is in violation of **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Missing token cannot search for ISAs check +#### 🛑 Missing token cannot search for ISAs check If the DSS accepts search queries without a token, it is in violation of **[astm.f3411.v19.DSS0010](../../../../../requirements/astm/f3411/v19.md)**. -#### Read scope cannot delete an ISA check +#### 🛑 Read scope cannot delete an ISA check If the existing ISA can be deleted by using a read-only scope, the DSS is in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** -#### Missing token prevents ISA deletion check +#### 🛑 Missing token prevents ISA deletion check If the existing ISA can be deleted without a token being provided, the DSS is in violation of **[astm.f3411.v19.DSS0010](../../../../../requirements/astm/f3411/v19.md)** -#### Fake token cannot delete an ISA check +#### 🛑 Fake token cannot delete an ISA check If the existing ISA can be deleted by using an invalid token, the DSS is in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** -#### Correct token and scope can delete ISA check +#### 🛑 Correct token and scope can delete ISA check If the existing ISA cannot be deleted when the proper credentials are presented, the DSS is in violation of **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** -#### Notified subscriber check +#### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. @@ -107,14 +107,14 @@ When an ISA is deleted, subscribers must be notified. If a subscriber cannot be The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### 🛑 Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md index 8ecad2c617..fe36d11598 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md @@ -54,7 +54,7 @@ As per **[astm.f3411.v19.DSS0210](../../../../requirements/astm/f3411/v19.md)** As such, this check will fail if the resolved IP of the DSS host is a private IP address. This check is skipped if the test exclusion `allow_private_addresses` is set to `True`. -#### DSS instance is reachable check +#### 🛑 DSS instance is reachable check As per **[astm.f3411.v19.DSS0210](../../../../requirements/astm/f3411/v19.md)** the DSS instance should be publicly addressable. As such, this check will fail if the DSS is not reachable with a dummy query, @@ -108,7 +108,7 @@ Action: USS2@DSS*n*: GET Subscription[*P*] by ID Qualitatively proves: Can retrieve specific Subscription emplaced in primary DSS from all DSSs -#### Subscription[*P*] returned with proper response check +#### 🛑 Subscription[*P*] returned with proper response check **[astm.f3411.v19.DSS0130,A2-6-1,5](../../../../requirements/astm/f3411/v19.md)** @@ -162,11 +162,11 @@ Action: USS1@DSS*P*: PUT ISA[*P*] setting end time to now + D seconds Qualitatively proves: ISA[*P*] modified with proper response, all Subscription[i] 1≤i≤n are returned in subscribers with proper response -#### Can get ISA from primary DSS check +#### 🛑 Can get ISA from primary DSS check **[astm.f3411.v19.DSS0130,3,a](../../../../requirements/astm/f3411/v19.md)** -#### Can modify ISA in primary DSS check +#### 🛑 Can modify ISA in primary DSS check **[astm.f3411.v19.DSS0130,A2-6-1,1b](../../../../requirements/astm/f3411/v19.md)** @@ -198,7 +198,7 @@ Action: USS2@DSS*n*: GET Subscription[*n*] by ID Qualitatively proves: Subscription deletion from ID index was effective from primary DSS -#### 404 with proper response check +#### 🛑 404 with proper response check **[astm.f3411.v19.DSS0130,A2-6-1,4b](../../../../requirements/astm/f3411/v19.md)** @@ -210,7 +210,7 @@ Action: USS2@DSS*n*: GET Subscriptions using ISA[*P*]’s area Qualitatively proves: Subscription deletion from geographic index was effective from primary DSS -#### Subscriptions queried successfully check +#### 🛑 Subscriptions queried successfully check **[astm.f3411.v19.DSS0130,A2-6-1,6](../../../../requirements/astm/f3411/v19.md)** @@ -256,7 +256,7 @@ Action: USS1@DSS*P*: DELETE ISA[*P*] Qualitatively proves: ISA deletion triggers subscription notification requests -#### ISA[*P*] deleted with proper response check +#### 🛑 ISA[*P*] deleted with proper response check **[astm.f3411.v19.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v19.md)** @@ -284,7 +284,7 @@ Action: USS2@DSS*n*: GET Subscriptions using ISA[*P*]’s area Qualitatively proves: Expired Subscription removed from geographic index on primary DSS -#### Subscriptions queried successfully check +#### 🛑 Subscriptions queried successfully check **[astm.f3411.v19.DSS0130,A2-6-1,6](../../../../requirements/astm/f3411/v19.md)** @@ -300,7 +300,7 @@ Action: USS2@DSS*n*: GET Subscription[*n*] by ID Qualitatively proves: Expired Subscription removed from ID index on primary DSS -#### 404 with proper response check +#### 🛑 404 with proper response check TODO: Investigate expected behavior and "404 with proper response" check @@ -312,7 +312,7 @@ Action: USS1@DSS*P*: DELETE ISA[*P*] Qualitatively proves: ISA deletion does not trigger subscription notification requests for expired Subscriptions -#### ISA[*P*] deleted with proper response check +#### 🛑 ISA[*P*] deleted with proper response check **[astm.f3411.v19.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v19.md)** @@ -348,14 +348,14 @@ Action: USS2@DSS*P*: DELETE Subscription[*n*] Any entities (ISAs or Subscriptions) not deleted normally will be deleted here. -### ISA deleted with proper response check +### ⚠️ ISA deleted with proper response check **[astm.f3411.v19.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v19.md)** -### Subscription can be queried by ID check +### ⚠️ Subscription can be queried by ID check **[astm.f3411.v19.DSS0130,A2-6-1,4a](../../../../requirements/astm/f3411/v19.md)** -### Subscription can be deleted check +### ⚠️ Subscription can be deleted check **[astm.f3411.v19.DSS0130,A2-6-1,4a](../../../../requirements/astm/f3411/v19.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/flight_injection.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/flight_injection.md index 3999d15c94..82f2c90c2f 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/flight_injection.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/fragments/flight_injection.md @@ -15,7 +15,7 @@ This check will fail if the flight was not successfully injected. This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. -## Valid flight check +## 🛑 Valid flight check TODO: Validate injected flights, especially to make sure they contain the specified injection IDs diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/networked_uas_disconnect.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/networked_uas_disconnect.md index fcae5adac4..3efb5598ae 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/networked_uas_disconnect.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/networked_uas_disconnect.md @@ -60,16 +60,16 @@ uss_qualifier acts as a Display Provider to query Service Providers under test i #### [Flight presence checks](./display_data_evaluator_flight_presence.md) -#### Flights data format check +#### ⚠️ Flights data format check **[astm.f3411.v19.NET0710,1](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the /flights endpoint does not validate against the OpenAPI-specified schema. #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_sp_flight.md) -#### Recent positions timestamps check +#### 🛑 Recent positions timestamps check **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires all recent positions to be within the NetMaxNearRealTimeDataPeriod. This check will fail if any of the reported positions are older than the maximally allowed period plus NetSpDataResponseTime99thPercentile. -#### Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check +#### 🛑 Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires that when an aircraft enters or leaves the queried area, the last or first reported position outside the area is provided in the recent positions, as long as it is not older than NetMaxNearRealTimeDataPeriod. This implies that any recent position outside the area must be either preceded or followed by a position inside the area. @@ -84,13 +84,13 @@ Service providers are expected to convey to clients that a flight is not sending Service providers that return anything else than the last received telemetry during `NetMaxNearRealTimeDataPeriodSeconds` (60) after a flight stops disconnects will fail this check. -#### Successful observation check +#### ⚠️ Successful observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call to each observer is expected to succeed since a valid view was provided by uss_qualifier. #### [Clustering checks](./display_data_evaluator_clustering.md) -#### Duplicate flights check +#### 🛑 Duplicate flights check Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the same flight ID may not be reported by a Display Provider for two flights in the same observation. @@ -98,7 +98,7 @@ Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../req #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_dp_flight.md) -#### Telemetry being used when present check +#### 🛑 Telemetry being used when present check **[astm.f3411.v19.NET0290](../../../../requirements/astm/f3411/v19.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md index 13b9ba0a17..ecb57ff4d6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/operator_interactions.md @@ -54,7 +54,7 @@ uss_qualifier retrieves the current (pre-existing) set of operator notifications uss_qualifier attempts to inject a flight into the Service Provider under test, knowing that the Service Provider will not be able to create an ISA. -#### Flight failed check +#### 🛑 Flight failed check Since the DSS is known to fail when attempting to create an ISA, if the Service Provider successfully creates the flight, they will have not met **[astm.f3411.v19.NET0610](../../../../requirements/astm/f3411/v19.md)**. @@ -64,7 +64,7 @@ TODO: Implement uss_qualifier retrieves the current (after failed flight) set of operator notifications. -#### Operator notified of discoverability failure check +#### 🛑 Operator notified of discoverability failure check The "after" set of operator notifications should contain at least one more entry than the "before" set of operator notifications. If there was no new operator notification, the Service Provider will not have met **[astm.f3411.v19.NET0620](../../../../requirements/astm/f3411/v19.md)**. @@ -84,13 +84,13 @@ uss_qualifier retrieves the current (pre-existing) set of operator notifications Throughout the duration of the flight, uss_qualifier periodically polls for operator notifications. -#### Insufficient telemetry operator notification check +#### 🛑 Insufficient telemetry operator notification check If the Service Provider under test does not provide an operator notification in Phase 2 (regarding the telemetry feed stopping), it will not have complied with **[astm.f3411.v19.NET0040](../../../../requirements/astm/f3411/v19.md)**. TODO: Implement -#### Missing data operator notification check +#### 🛑 Missing data operator notification check If the Service Provider under test does not provide an operator notification in Phase 4 (regarding missing data fields in reported telemetry), it will not have complied with **[astm.f3411.v19.NET0030](../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/sp_notification_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/sp_notification_behavior.md index a6b24f1909..759f04629e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/sp_notification_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/sp_notification_behavior.md @@ -39,7 +39,7 @@ A [`DSSInstancesResource`](../../../../resources/astm/f3411/dss.py) from which a Before injecting the test flights, a subscription is created on the DSS for the configured mock USS to allow it to validate that the Service Providers under test correctly send out notifications. -#### Subscription creation succeeds check +#### 🛑 Subscription creation succeeds check As per **[astm.f3411.v19.DSS0030,c](../../../../requirements/astm/f3411/v19.md)**, the DSS API must allow callers to create a subscription with either one or both of the start and end time missing, provided all the required parameters are valid. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_clustering.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_clustering.md index 6ec3c4ba1f..9a60a47a26 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_clustering.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_clustering.md @@ -2,28 +2,28 @@ This fragment is implemented in `display_data_evaluator.py:RIDObservationEvaluator._evaluate_clusters_observation`. -#### Minimal obfuscation distance of individual flights check +#### ⚠️ Minimal obfuscation distance of individual flights check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v22a.NET0490](../../../../requirements/astm/f3411/v22a.md)** requires that a Display provider shall obfuscate individual UAs within a cluster. If a cluster with a single flight has a width or height smaller than 2 * *NetMinObfuscationDistance* (meaning that the USS did not buffer the single point by *NetMinObfuscationDistance* in all directions when forming the cluster bounds), this test will fail. -#### Individual flights obfuscation check +#### ⚠️ Individual flights obfuscation check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v22a.NET0490](../../../../requirements/astm/f3411/v22a.md)** requires that a Display provider shall obfuscate individual UAs within a cluster. If a cluster with a single flight has its center equal to the position of the flight, this test will fail. If a cluster with a single flight does not actually encompass the flight, this test will fail. -#### Minimal obfuscation distance of multiple flights clusters check +#### ⚠️ Minimal obfuscation distance of multiple flights clusters check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v22a.NET0480](../../../../requirements/astm/f3411/v22a.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area. If a cluster with multiples flights has a width or height smaller than 2 * *NetMinObfuscationDistance*, this test will fail. A larger width or height might still be incorrect, but the test cannot at the moment determine this. -#### Clustering count check +#### ⚠️ Clustering count check For a display area with a diagonal greater than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v22a.NET0480](../../../../requirements/astm/f3411/v22a.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area. Taking into account the propagation time of the injected flights, if the total number of clustered UAs when this value is expected to be stable is not correct, this test will fail. -#### Minimal display area of clusters check +#### ⚠️ Minimal display area of clusters check For a display area with a diagonal greather than *NetDetailsMaxDisplayAreaDiagonal* and less than *NetMaxDisplayAreaDiagonal*, **[astm.f3411.v22a.NET0480](../../../../requirements/astm/f3411/v22a.md)** requires that a Display provider shall cluster UAs in close proximity to each other using a circular or polygonal area covering no less than *NetMinClusterSize* percent of the display area size. This check validates that the display area of a cluster, measured and provided in square meters by the test harness, is no less than *NetMinClusterSize* percent of the display area. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_flight_presence.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_flight_presence.md index 74848150af..8eb16f0e10 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_flight_presence.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/display_data_evaluator_flight_presence.md @@ -2,11 +2,11 @@ This fragment is implemented in `display_data_evaluator.py:RIDObservationEvaluator._evaluate_flight_presence`. -## Premature flight check +## ⚠️ Premature flight check The timestamps of the injected telemetry usually start in the future. If a flight with injected telemetry only in the future is observed prior to the timestamp of the first telemetry point, this check will fail because the SP does not satisfy **[interuss.automated_testing.rid.injection.ExpectedBehavior](../../../../requirements/interuss/automated_testing/rid/injection.md)**. -## Missing flight check +## ⚠️ Missing flight check **[astm.f3411.v22a.NET0610](../../../../requirements/astm/f3411/v22a.md)** requires that SPs make all UAS operations discoverable over the duration of the flight plus *NetMaxNearRealTimeDataPeriod*, so each injected flight should be observable during this time. If a flight is not observed during its appropriate time period, this check will fail. @@ -14,6 +14,6 @@ The timestamps of the injected telemetry usually start in the future. If a flig The identity of flights is determined by precisely matching the known injected positions. If the flight can be found, the USS may not have met **[astm.f3411.v22a.NET0260,Table1,10](../../../../requirements/astm/f3411/v22a.md)** or **[astm.f3411.v22a.NET0260,Table1,11](../../../../requirements/astm/f3411/v22a.md)** prescribing provision of position data consistent with the common data dictionary. -## Lingering flight check +## ⚠️ Lingering flight check **[astm.f3411.v22a.NET0260,NearRealTime](../../../../requirements/astm/f3411/v22a.md)** requires a SP to provide flights up to *NetMaxNearRealTimeDataPeriod* in the past, but an SP should preserve privacy and ensure relevancy by not sharing flights that are further in the past than this window. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md index 59c482fdf0..df509b6fea 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -95,14 +95,14 @@ The ISA search area parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove any created ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_expiry.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_expiry.md index 2ccccb976d..1aae88dbb3 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_expiry.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_expiry.md @@ -48,14 +48,14 @@ that ISAs that are in the searched area but have expired should not be returned. The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md index 221bfbd417..eb74e569d2 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_subscription_interactions.md @@ -148,7 +148,7 @@ and return the up-to-date subscription in the response to the query mutating the Failure to do so means that the DSS is not properly implementing **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. -#### Subscription that only barely overlaps the ISA contains the ISA check +#### 🛑 Subscription that only barely overlaps the ISA contains the ISA check A subscription that is created for a volume that only barely overlaps with the previously created ISA should still contain the ISA in the reply from the server, otherwise the DSS does not comply with **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)** @@ -182,22 +182,22 @@ Notifications to any subscriber to the created ISA need to be successful. If a The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely while also deleting any subscription it might have created for the ISA's area. -#### Successful ISA query check +#### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -#### Removed pre-existing ISA check +#### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -#### Notified subscriber check +#### ⚠️ Notified subscriber check When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. -#### Successful subscription search query check +#### ⚠️ Successful subscription search query check **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. -#### Subscription can be deleted check +#### ⚠️ Subscription can be deleted check **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md index ea79968595..eabc40b29b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_validation.md @@ -73,14 +73,14 @@ the DSS is expected to reject the request, otherwise it fails to meet **[astm.f3 The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### ⚠️ Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_simple.md index e8ce873117..cb693a925b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_simple.md @@ -109,11 +109,11 @@ The subscription version generated by the DSS must be a lower-case alphanumeric Query and search for the created subscription in various ways -#### Get Subscription by ID check +#### 🛑 Get Subscription by ID check If the freshly created subscription cannot be queried using its ID, the DSS is failing to meet **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)**. -#### Search for all subscriptions in ISA area check +#### 🛑 Search for all subscriptions in ISA area check If the DSS fails to let us search in the area for which the subscription was just created, it is failing to meet **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. @@ -121,7 +121,7 @@ If the DSS fails to let us search in the area for which the subscription was jus If the created subscription is not returned in a search that covers the area it was created for, the DSS is not properly implementing **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. -#### No huge search area allowed check +#### 🛑 No huge search area allowed check In accordance with **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**, the DSS should not allow searches for areas that are too big. @@ -200,7 +200,7 @@ The subscription version generated by the DSS must be a lower-case alphanumeric This test step mutates the previously created subscription to verify that the DSS reacts properly: notably, it checks that the subscription version is updated, including for changes that are not directly visible, such as changing the subscription's footprint. -#### Subscription can be mutated check +#### 🛑 Subscription can be mutated check If a subscription cannot be modified with a valid set of parameters, the DSS is failing to meet **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. @@ -359,15 +359,15 @@ The subscription version generated by the DSS must be a lower-case alphanumeric Attempt to query and search for the deleted subscription in various ways -#### Query by subscription ID should fail check +#### 🛑 Query by subscription ID should fail check If the DSS provides a successful reply to a direct query for the deleted subscription, it is in violation of **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)**. -#### Search for all subscriptions in ISA area check +#### 🛑 Search for all subscriptions in ISA area check If the DSS fails to let us search in the area for which the subscription was just created, it is failing to meet **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. -#### Search area that represents a loop is not allowed check +#### 🛑 Search area that represents a loop is not allowed check The DSS should not allow us to search for subscriptions using a list of vertices describing a loop (first and last points in the list of vertices are the same), otherwise it is failing to meet **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)**. @@ -380,10 +380,10 @@ If the DSS returns the deleted subscription in a search that covers the area it The cleanup phase of this test scenario removes the subscription with the known test ID if it has not been removed before. -#### Subscription can be queried by ID check +#### 🛑 Subscription can be queried by ID check If the DSS cannot be queried for the existing test ID, the DSS is likely not implementing **[astm.f3411.v22a.DSS0030,e](../../../../../requirements/astm/f3411/v22a.md)** correctly. -#### Subscription can be deleted check +#### 🛑 Subscription can be deleted check An attempt to delete a subscription when the correct version is provided should succeed, otherwise the DSS is in violation of **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md index 36fbdba1be..7eb5ea4fdc 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/subscription_validation.md @@ -30,7 +30,7 @@ This step ensures that we remove any subscription that may already exist for the The test will attempt to create 10 identical subscriptions for the same area and expect this to succeed. -#### Create up to the maximum allowed number of subscriptions in an area check +#### 🛑 Create up to the maximum allowed number of subscriptions in an area check As per **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**, the DSS API is expected to allow us to create multiple subscriptions. @@ -39,7 +39,7 @@ to create multiple subscriptions. Now, create an 11th one and expect it to fail. -#### Enforce maximum number of subscriptions for an area check +#### 🛑 Enforce maximum number of subscriptions for an area check If the DSS successfully creates an 11th Subscription in the same area instead of rejecting it, it will not have performed the Subscription count validation as defined in **[astm.f3411.v22a.DSS0050](../../../../../requirements/astm/f3411/v22a.md)** @@ -48,11 +48,11 @@ it will not have performed the Subscription count validation as defined in **[as Clean up any subscriptions created. -#### Successful subscription search query check +#### 🛑 Successful subscription search query check If the query for subscriptions fails, **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** was not met. -#### Subscription can be deleted check +#### 🛑 Subscription can be deleted check If the deletion attempt fails, **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** was not met. @@ -62,7 +62,7 @@ If the deletion attempt fails, **[astm.f3411.v22a.DSS0030,d](../../../../../requ The test will attempt to create a subscription for 24 hours and 10 minutes, and expect this to fail with an HTTP 400 error. -#### Too-long subscription creation rejected check +#### 🛑 Too-long subscription creation rejected check **[astm.f3411.v22a.DSS0060](../../../../../requirements/astm/f3411/v22a.md)** any subscription to the DSS may not exceed NetDSSMaxSubscriptionDuration (24 hours). @@ -73,11 +73,11 @@ If creation of the too-long-duration subscription succeeds, the test expects tha The test will attempt to create a valid subscription of 23 hours and 59 minutes, and then increase its duration to 24 hours and 10 minutes, expecting this update to fail. -#### Valid subscription created check +#### 🛑 Valid subscription created check The ability to create a valid subscription is required in **[astm.f3411.v22a.DSS0030,c](../../../../../requirements/astm/f3411/v22a.md)**. -#### Subscription duration limited during update check +#### 🛑 Subscription duration limited during update check If the DSS allows a user to extend an existing, valid subscription to a duration exceeding that specified in **[astm.f3411.v22a.DSS0060](../../../../../requirements/astm/f3411/v22a.md)**, this check will fail. @@ -93,10 +93,10 @@ The ability to delete an existing subscription is required in **[astm.f3411.v22a The cleanup phase of this test scenario will remove any subscription that may have been created during the test and that intersects with the test ISA. -### Successful subscription search query check +### 🛑 Successful subscription search query check If the query for subscriptions fails, the "GET Subscriptions" portion of **[astm.f3411.v22a.DSS0030,f](../../../../../requirements/astm/f3411/v22a.md)** was not met. -### Subscription can be deleted check +### 🛑 Subscription can be deleted check If the deletion attempt fails, the "DELETE Subscription" portion of **[astm.f3411.v22a.DSS0030,d](../../../../../requirements/astm/f3411/v22a.md)** was not met. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/clean_workspace.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/clean_workspace.md index 586a2ed6a4..cc4727a677 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/clean_workspace.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/clean_workspace.md @@ -2,26 +2,26 @@ This page describes the content of a common test step that ensures a clean workspace for testing interactions with a DSS -## Successful ISA query check +## 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -## Removed pre-existing ISA check +## ⚠️ Removed pre-existing ISA check If an ISA with the intended ID is already present in the DSS, it needs to be removed before proceeding with the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,d](../../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -## Notified subscriber check +## 🛑 Notified subscriber check When a pre-existing ISA needs to be deleted to ensure a clean workspace, any subscribers to ISAs in that area must be notified (as specified by the DSS). If a notification cannot be delivered, then the **[astm.f3411.v22a.NET0730](../../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the POST ISAs endpoint isn't met. -## Successful subscription search query check +## 🛑 Successful subscription search query check **[astm.f3411.v22a.DSS0030,f](../../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to retrieve the subscriptions they created. -## Subscription can be queried by ID check +## 🛑 Subscription can be queried by ID check If the DSS cannot be queried for the existing test ID, the DSS is likely not implementing **[astm.f3411.v22a.DSS0030,e](../../../../../../requirements/astm/f3411/v22a.md)** correctly. -## Subscription can be deleted check +## 🛑 Subscription can be deleted check **[astm.f3411.v22a.DSS0030,d](../../../../../../requirements/astm/f3411/v22a.md)** requires the implementation of the DSS endpoint to allow callers to delete subscriptions they created. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md index af51c360b8..ddf3b43d09 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas.md @@ -3,30 +3,30 @@ This page describes the content of a common test step where a search for ISAs should be successful. See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISAs search response format check +## ⚠️ ISAs search response format check While F3411-22a does not explicitly require the implementation of the ISA search endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -## ISA ID matches check +## ⚠️ ISA ID matches check The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -## ISA version format check +## ⚠️ ISA version format check Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. -## ISA version matches check +## ⚠️ ISA version matches check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -## ISA start time matches check +## ⚠️ ISA start time matches check The ISA creation request specified an exact start time slightly past now, so the DSS should have created an ISA starting at exactly that time. If the DSS response indicates the ISA start time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -## ISA end time matches check +## ⚠️ ISA end time matches check The ISA creation request specified an exact end time, so the DSS should have created an ISA ending at exactly that time. If the DSS response indicates the ISA end time is not this value, **[astm.f3411.v22a.DSS0030,a](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. -## ISA URL matches check +## ⚠️ ISA URL matches check When the ISA is created, the DSS returns the URL of the ISA in the response body. If this URL does not match the URL requested, **[astm.f3411.v22a.DSS0030,a](../../../../../../requirements/astm/f3411/v22a.md)** is not implemented correctly and this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas_miss.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas_miss.md index 6525b883b7..8cd4a3d1e9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas_miss.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/search_isas_miss.md @@ -5,10 +5,10 @@ relevant to the current scenario are not expected to be present. See `DSSWrapper.search_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISAs search response format check +## ⚠️ ISAs search response format check While F3411-22a does not explicitly require the implementation of the ISA search endpoint, Annex A4 specifies the explicit format for this endpoint. If this format is not followed, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -## ISA version format check +## ⚠️ ISA version format check Because the ISA version must be used in URLs, it must be URL-safe even though the ASTM standards do not explicitly require this. If the indicated ISA version is not URL-safe, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/token_validation.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/token_validation.md index b7e7356d96..72dbe5a774 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/token_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/token_validation.md @@ -32,74 +32,74 @@ part of the test. This step attempts to create and read ISAs by providing both the correct and incorrect scopes, omitting the token or providing an invalid one, and expects the DSS to properly behave in each case. -#### Read scope cannot create an ISA check +#### 🛑 Read scope cannot create an ISA check If an ISA can be created with a scope that does not provide write permission, the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. -#### Missing token prevents creating an ISA check +#### 🛑 Missing token prevents creating an ISA check If an ISA can be created without a token being present in the request, the DSS is in violation of **[astm.f3411.v22a.DSS0010](../../../../../requirements/astm/f3411/v22a.md)**. -#### Fake token prevents creating an ISA check +#### 🛑 Fake token prevents creating an ISA check If an ISA can be created with an incorrect token in the request, the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)**. -#### Correct token and scope can create ISA check +#### 🛑 Correct token and scope can create ISA check If the ISA cannot be created when the proper credentials are presented, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. -#### Missing token prevents reading an ISA check +#### 🛑 Missing token prevents reading an ISA check If the ISA that was created can be accessed without a token being present in the request, the DSS is in violation of **[astm.f3411.v22a.DSS0010](../../../../../requirements/astm/f3411/v22a.md)** -#### Fake token prevents reading an ISA check +#### 🛑 Fake token prevents reading an ISA check If the ISA that was created can be accessed using a token with an invalid signature, the DSS is in violation of **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** -#### Read scope cannot mutate an ISA check +#### 🛑 Read scope cannot mutate an ISA check If the existing ISA can be mutated by using a read-only scope, the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** -#### Missing token prevents mutating an ISA check +#### 🛑 Missing token prevents mutating an ISA check If the existing ISA can be mutated without a token being provided, the DSS is in violation of **[astm.f3411.v22a.DSS0010](../../../../../requirements/astm/f3411/v22a.md)** -#### Proper token is allowed to search for ISAs check +#### 🛑 Proper token is allowed to search for ISAs check If a valid token is presented as part of the search request, and the search parameters are valid, the DSS must return a 200 response, or be in violation of **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Fake token cannot mutate an ISA check +#### 🛑 Fake token cannot mutate an ISA check If the existing ISA can be mutated by using an invalid token, the DSS is in violation of **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** -#### Fake token cannot search for ISAs check +#### 🛑 Fake token cannot search for ISAs check If the DSS accepts search queries with an invalid token, it is in violation of **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Missing token cannot search for ISAs check +#### 🛑 Missing token cannot search for ISAs check If the DSS accepts search queries without a token, it is in violation of **[astm.f3411.v22a.DSS0010](../../../../../requirements/astm/f3411/v22a.md)**. -#### Read scope cannot delete an ISA check +#### 🛑 Read scope cannot delete an ISA check If the existing ISA can be deleted by using a read-only scope, the DSS is in violation of **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** -#### Missing token prevents ISA deletion check +#### 🛑 Missing token prevents ISA deletion check If the existing ISA can be deleted without a token being provided, the DSS is in violation of **[astm.f3411.v22a.DSS0010](../../../../../requirements/astm/f3411/v22a.md)** -#### Fake token cannot delete an ISA check +#### 🛑 Fake token cannot delete an ISA check If the existing ISA can be deleted by using an invalid token, the DSS is in violation of **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** -#### Correct token and scope can delete ISA check +#### 🛑 Correct token and scope can delete ISA check If the existing ISA cannot be deleted when the proper credentials are presented, the DSS is in violation of **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** -#### Notified subscriber check +#### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. @@ -107,14 +107,14 @@ When an ISA is deleted, subscribers must be notified. If a subscriber cannot be The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### 🛑 Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md index 3a98001732..d6b491cbd1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md @@ -54,7 +54,7 @@ As per **[astm.f3411.v22a.DSS0210](../../../../requirements/astm/f3411/v22a.md)* As such, this check will fail if the resolved IP of the DSS host is a private IP address. This check is skipped if the test exclusion `allow_private_addresses` is set to `True`. -#### DSS instance is reachable check +#### 🛑 DSS instance is reachable check As per **[astm.f3411.v22a.DSS0210](../../../../requirements/astm/f3411/v22a.md)** the DSS instance should be publicly addressable. As such, this check will fail if the DSS is not reachable with a dummy query, @@ -107,7 +107,7 @@ Action: USS2@DSS*n*: GET Subscription[*P*] by ID Qualitatively proves: Can retrieve specific Subscription emplaced in primary DSS from all DSSs -#### Subscription[*P*] returned with proper response check +#### 🛑 Subscription[*P*] returned with proper response check **[astm.f3411.v22a.DSS0130,A2-6-1,5](../../../../requirements/astm/f3411/v22a.md)** @@ -161,11 +161,11 @@ Action: USS1@DSS*P*: PUT ISA[*P*] setting end time to now + D seconds Qualitatively proves: ISA[*P*] modified with proper response, all Subscription[i] 1≤i≤n are returned in subscribers with proper response -#### Can get ISA from primary DSS check +#### 🛑 Can get ISA from primary DSS check **[astm.f3411.v22a.DSS0130,3,a](../../../../requirements/astm/f3411/v22a.md)** -#### Can modify ISA in primary DSS check +#### 🛑 Can modify ISA in primary DSS check **[astm.f3411.v22a.DSS0130,A2-6-1,1b](../../../../requirements/astm/f3411/v22a.md)** @@ -197,7 +197,7 @@ Action: USS2@DSS*n*: GET Subscription[*n*] by ID Qualitatively proves: Subscription deletion from ID index was effective from primary DSS -#### 404 with proper response check +#### 🛑 404 with proper response check **[astm.f3411.v22a.DSS0130,A2-6-1,4b](../../../../requirements/astm/f3411/v22a.md)** @@ -209,7 +209,7 @@ Action: USS2@DSS*n*: GET Subscriptions using ISA[*P*]’s area Qualitatively proves: Subscription deletion from geographic index was effective from primary DSS -#### Subscriptions queried successfully check +#### 🛑 Subscriptions queried successfully check **[astm.f3411.v22a.DSS0130,A2-6-1,6](../../../../requirements/astm/f3411/v22a.md)** @@ -255,7 +255,7 @@ Action: USS1@DSS*P*: DELETE ISA[*P*] Qualitatively proves: ISA deletion triggers subscription notification requests -#### ISA[*P*] deleted with proper response check +#### 🛑 ISA[*P*] deleted with proper response check **[astm.f3411.v22a.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v22a.md)** @@ -283,7 +283,7 @@ Action: USS2@DSS*n*: GET Subscriptions using ISA[*P*]’s area Qualitatively proves: Expired Subscription removed from geographic index on primary DSS -#### Subscriptions queried successfully check +#### 🛑 Subscriptions queried successfully check **[astm.f3411.v22a.DSS0130,A2-6-1,6](../../../../requirements/astm/f3411/v22a.md)** @@ -299,7 +299,7 @@ Action: USS2@DSS*n*: GET Subscription[*n*] by ID Qualitatively proves: Expired Subscription removed from ID index on primary DSS -#### 404 with proper response check +#### 🛑 404 with proper response check TODO: Investigate expected behavior and "404 with proper response" check @@ -311,7 +311,7 @@ Action: USS1@DSS*P*: DELETE ISA[*P*] Qualitatively proves: ISA deletion does not trigger subscription notification requests for expired Subscriptions -#### ISA[*P*] deleted with proper response check +#### 🛑 ISA[*P*] deleted with proper response check **[astm.f3411.v22a.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v22a.md)** @@ -347,14 +347,14 @@ Action: USS2@DSS*P*: DELETE Subscription[*n*] Any entities (ISAs or Subscriptions) not deleted normally will be deleted here. -### ISA deleted with proper response check +### ⚠️ ISA deleted with proper response check **[astm.f3411.v22a.DSS0130,A2-6-1,2a](../../../../requirements/astm/f3411/v22a.md)** -### Subscription can be queried by ID check +### ⚠️ Subscription can be queried by ID check **[astm.f3411.v22a.DSS0130,A2-6-1,4a](../../../../requirements/astm/f3411/v22a.md)** -### Subscription can be deleted check +### ⚠️ Subscription can be deleted check **[astm.f3411.v22a.DSS0130,A2-6-1,4a](../../../../requirements/astm/f3411/v22a.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/fragments/flight_injection.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/fragments/flight_injection.md index b4f4a767c4..5ceb315588 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/fragments/flight_injection.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/fragments/flight_injection.md @@ -16,7 +16,7 @@ This check will fail if the flight was not successfully injected. This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. -## Valid flight check +## 🛑 Valid flight check TODO: Validate injected flights, especially to make sure they contain the specified injection IDs diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md index 91c4fb6162..b684d1cd13 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/operator_interactions.md @@ -54,7 +54,7 @@ uss_qualifier retrieves the current (pre-existing) set of operator notifications uss_qualifier attempts to inject a flight into the Service Provider under test, knowing that the Service Provider will not be able to create an ISA. -#### Flight failed check +#### 🛑 Flight failed check Since the DSS is known to fail when attempting to create an ISA, if the Service Provider successfully creates the flight, they will have not met **[astm.f3411.v22a.NET0610](../../../../requirements/astm/f3411/v22a.md)**. @@ -64,7 +64,7 @@ TODO: Implement uss_qualifier retrieves the current (after failed flight) set of operator notifications. -#### Operator notified of discoverability failure check +#### 🛑 Operator notified of discoverability failure check The "after" set of operator notifications should contain at least one more entry than the "before" set of operator notifications. If there was no new operator notification, the Service Provider will not have met **[astm.f3411.v22a.NET0620](../../../../requirements/astm/f3411/v22a.md)**. @@ -84,13 +84,13 @@ uss_qualifier retrieves the current (pre-existing) set of operator notifications Throughout the duration of the flight, uss_qualifier periodically polls for operator notifications. -#### Insufficient telemetry operator notification check +#### 🛑 Insufficient telemetry operator notification check If the Service Provider under test does not provide an operator notification in Phase 2 (regarding the telemetry feed stopping), it will not have complied with **[astm.f3411.v22a.NET0040](../../../../requirements/astm/f3411/v22a.md)**. TODO: Implement -#### Missing data operator notification check +#### 🛑 Missing data operator notification check If the Service Provider under test does not provide an operator notification in Phase 4 (regarding missing data fields in reported telemetry), it will not have complied with **[astm.f3411.v22a.NET0030](../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/sp_notification_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/sp_notification_behavior.md index 3114a979b4..6d39d4e1c9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/sp_notification_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/sp_notification_behavior.md @@ -39,7 +39,7 @@ A [`DSSInstancesResource`](../../../../resources/astm/f3411/dss.py) from which a Before injecting the test flights, a subscription is created on the DSS for the configured mock USS to allow it to validate that the Service Providers under test correctly send out notifications. -#### Subscription creation succeeds check +#### 🛑 Subscription creation succeeds check As per **[astm.f3411.v22a.DSS0030,c](../../../../requirements/astm/f3411/v22a.md)**, the DSS API must allow callers to create a subscription with either one or both of the start and end time missing, provided all the required parameters are valid. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md index 17c470b4c7..af527cdd8e 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md @@ -98,6 +98,6 @@ direct retrieval by tested_uss from mock_uss. Teardown ## Cleanup -### Successful flight deletion check +### ⚠️ Successful flight deletion check This cleanup is for both - after testcase ends and after test scenario ends **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.md index cec95866fc..b3b38f886d 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.md @@ -57,7 +57,7 @@ Retrieve and validate synchronization of the created constraint at every DSS pro Check that read query succeeds. -#### Newly created CR can be consistently retrieved from all DSS instances check +#### 🛑 Newly created CR can be consistently retrieved from all DSS instances check If the constraint retrieved from a secondary DSS instance is not consistent with the newly created one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1a](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1f](../../../../../requirements/astm/f3548/v21.md)**, @@ -84,7 +84,7 @@ Search for and validate synchronization of the created constraint at every DSS p Check that search query succeeds and the response is well-formed. -#### Newly created CR can be consistently searched for from all DSS instances check +#### 🛑 Newly created CR can be consistently searched for from all DSS instances check If the constraint searched from a secondary DSS instance is not consistent with the newly created one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1a](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1e](../../../../../requirements/astm/f3548/v21.md)**, @@ -124,7 +124,7 @@ Retrieve and validate synchronization of the updated constraint at every DSS pro Check that read query succeeds and the response is well-formed. -#### Updated CR can be consistently retrieved from all DSS instances check +#### 🛑 Updated CR can be consistently retrieved from all DSS instances check If the constraint retrieved from a secondary DSS instance is not consistent with the updated one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1b](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1d](../../../../../requirements/astm/f3548/v21.md)**, @@ -147,7 +147,7 @@ Search for and validate synchronization of the updated constraint at every DSS p Check that search query succeeds and the response is well-formed. -#### Updated CR can be consistently searched for from all DSS instances check +#### 🛑 Updated CR can be consistently searched for from all DSS instances check If the constraint searched from a secondary DSS instance is not consistent with the updated one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1b](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1e](../../../../../requirements/astm/f3548/v21.md)**, diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/op_intent_ref_synchronization.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/op_intent_ref_synchronization.md index ea16e60760..cb417cc94c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/op_intent_ref_synchronization.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/op_intent_ref_synchronization.md @@ -57,7 +57,7 @@ Retrieve and validate synchronization of the created operational intent at every Check that read query succeeds. -#### Newly created OIR can be consistently retrieved from all DSS instances check +#### 🛑 Newly created OIR can be consistently retrieved from all DSS instances check If the operational intent retrieved from a secondary DSS instance is not consistent with the newly created one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1a](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1d](../../../../../requirements/astm/f3548/v21.md)**, @@ -76,7 +76,7 @@ Search for and validate synchronization of the created operational intent at eve Check that search query succeeds. -#### Newly created OIR can be consistently searched for from all DSS instances check +#### 🛑 Newly created OIR can be consistently searched for from all DSS instances check If the operational intent searched from a secondary DSS instance is not consistent with the newly created one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1a](../../../../../requirements/astm/f3548/v21.md)**, **[astm.f3548.v21.DSS0210,A2-7-2,1c](../../../../../requirements/astm/f3548/v21.md)**, @@ -112,7 +112,7 @@ Retrieve and validate synchronization of the updated operational intent at every Check that read query succeeds. -#### Updated OIR can be consistently retrieved from all DSS instances check +#### 🛑 Updated OIR can be consistently retrieved from all DSS instances check If the operational intent retrieved from a secondary DSS instance is not consistent with the updated one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1b](../../../../../requirements/astm/f3548/v21.md)** @@ -131,7 +131,7 @@ Search for and validate synchronization of the updated operational intent at eve Check that search query succeeds. -#### Updated OIR can be consistently searched for from all DSS instances check +#### 🛑 Updated OIR can be consistently searched for from all DSS instances check If the operational intent searched from a secondary DSS instance is not consistent with the updated one on the primary DSS instance, this check will fail per **[astm.f3548.v21.DSS0210,A2-7-2,1b](../../../../../requirements/astm/f3548/v21.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md index 147a7a1e7e..d451c0f2e7 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md @@ -133,7 +133,7 @@ If the tested USS successfully plans the flight or otherwise fails to indicate a to correctly compute the conflicting intersection. Therefore, this check will fail if the USS indicates success in planning the flight from the user flight intent, per **[astm.f3548.v21.GEN0500](../../../../requirements/astm/f3548/v21.md)**. -#### Failure check +#### 🛑 Failure check All flight intent data provided was complete and correct. It should have been processed successfully, allowing the USS to reject or accept the flight. If the USS indicates that the attempt failed, this check will fail per **[interuss.automated_testing.flight_planning.ExpectedBehavior](../../../../requirements/interuss/automated_testing/flight_planning.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md index 581b5ac29a..5cee80e703 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md @@ -127,6 +127,6 @@ Check a notification was received by tested_uss for Flight 2, with Flight 1's su ## Cleanup -### Successful flight deletion check +### ⚠️ Successful flight deletion check The flights are cleaned up at the end of the test scenario. **[interuss.automated_testing.flight_planning.DeleteFlightSuccess](../../../../../requirements/interuss/automated_testing/flight_planning.md)** diff --git a/monitoring/uss_qualifier/scenarios/documentation/autoformat.py b/monitoring/uss_qualifier/scenarios/documentation/autoformat.py index 289511745a..9ac2d26659 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/autoformat.py +++ b/monitoring/uss_qualifier/scenarios/documentation/autoformat.py @@ -8,7 +8,11 @@ from monitoring.uss_qualifier.documentation import text_of from monitoring.uss_qualifier.requirements.documentation import RequirementID -from monitoring.uss_qualifier.scenarios.documentation.definitions import TestCheckTree +from monitoring.uss_qualifier.scenarios.documentation.definitions import ( + TestCaseDocumentation, + TestCheckTree, + TestStepDocumentation, +) from monitoring.uss_qualifier.scenarios.documentation.parsing import ( TEST_STEP_SUFFIX, get_documentation, @@ -148,6 +152,15 @@ def update_checks_without_severity(test_scenarios: List[TestScenarioType]) -> No checks_without_severity.add_check( scenario=test_scenario, case=case, step=step, check=check ) + if "cleanup" in docs: + for check in docs.cleanup.checks: + if "severity" not in check or check.severity is None: + checks_without_severity.add_check( + scenario=test_scenario, + case=TestCaseDocumentation(name="Cleanup", steps=[]), + step=docs.cleanup, + check=check, + ) preexisting_checks_without_severity = TestCheckTree.preexisting( "checks_without_severity.json" ) diff --git a/monitoring/uss_qualifier/scenarios/documentation/checks_without_severity.json b/monitoring/uss_qualifier/scenarios/documentation/checks_without_severity.json index bef2127524..24bfbdf062 100644 --- a/monitoring/uss_qualifier/scenarios/documentation/checks_without_severity.json +++ b/monitoring/uss_qualifier/scenarios/documentation/checks_without_severity.json @@ -1,806 +1,3 @@ { - "scenarios": { - "scenarios.astm.netrid.v19.dp_behavior.DisplayProviderBehavior": { - "Setup": { - "Clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.dss.heavy_traffic_concurrent.HeavyTrafficConcurrent": { - "Concurrent Requests": { - "Search Available ISAs": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search Deleted ISAs": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.dss.isa_expiry.ISAExpiry": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.dss.isa_simple.ISASimple": { - "Delete ISA": { - "Search ISA": [ - "ISAs search response format", - "ISA version format" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Update and search ISA": { - "Search by area only": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search by earliest time (excluded)": [ - "ISAs search response format", - "ISA version format" - ], - "Search by earliest time (included)": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search by latest time (excluded)": [ - "ISAs search response format", - "ISA version format" - ], - "Search by latest time (included)": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ] - } - }, - "scenarios.astm.netrid.v19.dss.isa_subscription_interactions.ISASubscriptionInteractions": { - "ISA Subscription Interactions": { - "New subscription within ISA is mutated to ISA boundary": [ - "Subscription that only barely overlaps the ISA contains the ISA" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.dss.isa_validation.ISAValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.dss.subscription_simple.SubscriptionSimple": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Subscription Simple": { - "Mutate Subscription": [ - "Subscription can be mutated" - ], - "Query Deleted Subscription": [ - "Query by subscription ID should fail", - "Search for all subscriptions in ISA area", - "Search area that represents a loop is not allowed" - ], - "Query Existing Subscription": [ - "Get Subscription by ID", - "Search for all subscriptions in ISA area", - "No huge search area allowed" - ] - } - }, - "scenarios.astm.netrid.v19.dss.subscription_validation.SubscriptionValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Subscription duration limitations": { - "Try to create too-long subscription": [ - "Too-long subscription creation rejected" - ], - "Try to extend subscription": [ - "Valid subscription created", - "Subscription duration limited during update" - ] - }, - "Subscription quantity limitations": { - "Clean up subscriptions": [ - "Successful subscription search query", - "Subscription can be deleted" - ], - "Create maximum number of subscriptions": [ - "Create up to the maximum allowed number of subscriptions in an area" - ], - "Exceed maximum number of subscriptions": [ - "Enforce maximum number of subscriptions for an area" - ] - } - }, - "scenarios.astm.netrid.v19.dss.token_validation.TokenValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Token validation": { - "Token validation": [ - "Read scope cannot create an ISA", - "Missing token prevents creating an ISA", - "Fake token prevents creating an ISA", - "Correct token and scope can create ISA", - "Missing token prevents reading an ISA", - "Fake token prevents reading an ISA", - "Read scope cannot mutate an ISA", - "Missing token prevents mutating an ISA", - "Proper token is allowed to search for ISAs", - "Fake token cannot mutate an ISA", - "Fake token cannot search for ISAs", - "Missing token cannot search for ISAs", - "Read scope cannot delete an ISA", - "Missing token prevents ISA deletion", - "Fake token cannot delete an ISA", - "Correct token and scope can delete ISA", - "Notified subscriber" - ] - } - }, - "scenarios.astm.netrid.v19.dss_interoperability.DSSInteroperability": { - "Interoperability sequence": { - "S11": [ - "ISA[P] deleted with proper response" - ], - "S13": [ - "Subscriptions queried successfully" - ], - "S14": [ - "404 with proper response" - ], - "S15": [ - "ISA[P] deleted with proper response" - ], - "S3": [ - "Subscription[P] returned with proper response" - ], - "S5": [ - "Can get ISA from primary DSS", - "Can modify ISA in primary DSS" - ], - "S7": [ - "404 with proper response" - ], - "S8": [ - "Subscriptions queried successfully" - ] - }, - "Prerequisites": { - "Test environment requirements": [ - "DSS instance is reachable" - ] - } - }, - "scenarios.astm.netrid.v19.networked_uas_disconnect.NetworkedUASDisconnect": { - "Networked UAS disconnect": { - "Service Provider polling": [ - "Premature flight", - "Missing flight", - "Lingering flight", - "Flights data format", - "Recent positions timestamps", - "Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing", - "Successful observation", - "Minimal obfuscation distance of individual flights", - "Individual flights obfuscation", - "Minimal obfuscation distance of multiple flights clusters", - "Clustering count", - "Minimal display area of clusters", - "Duplicate flights", - "Premature flight", - "Missing flight", - "Lingering flight", - "Telemetry being used when present" - ] - } - }, - "scenarios.astm.netrid.v19.nominal_behavior.NominalBehavior": { - "Nominal flight": { - "Injection": [ - "Valid flight" - ], - "Observer polling": [ - "Minimal obfuscation distance of individual flights", - "Individual flights obfuscation", - "Minimal obfuscation distance of multiple flights clusters", - "Clustering count", - "Minimal display area of clusters", - "Premature flight", - "Missing flight", - "Lingering flight" - ], - "Service Provider polling": [ - "Premature flight", - "Missing flight", - "Lingering flight" - ] - } - }, - "scenarios.astm.netrid.v19.operator_interactions.OperatorInteractions": { - "Failed ISA": { - "Enumerate operator notifications": [ - "Operator notified of discoverability failure" - ], - "Inject flight": [ - "Flight failed" - ] - }, - "In-flight notifications": { - "Poll Service Provider": [ - "Insufficient telemetry operator notification", - "Missing data operator notification" - ] - } - }, - "scenarios.astm.netrid.v19.sp_notification_behavior.ServiceProviderNotificationBehavior": { - "Service Provider notification behavior": { - "Injection": [ - "Valid flight" - ], - "Mock USS Subscription": [ - "Subscription creation succeeds" - ] - }, - "Setup": { - "Clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v19.sp_operator_notify_missing_fields.SpOperatorNotifyMissingFields": { - "Missing fields flight": { - "Injection": [ - "Valid flight" - ] - } - }, - "scenarios.astm.netrid.v19.sp_operator_notify_slow_update.ServiceProviderNotifiesSlowUpdates": { - "Slow updates flight": { - "Injection": [ - "Valid flight" - ] - } - }, - "scenarios.astm.netrid.v22a.dp_behavior.DisplayProviderBehavior": { - "Setup": { - "Clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.heavy_traffic_concurrent.HeavyTrafficConcurrent": { - "Concurrent Requests": { - "Search Available ISAs": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search Deleted ISAs": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.isa_expiry.ISAExpiry": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.isa_simple.ISASimple": { - "Delete ISA": { - "Search ISA": [ - "ISAs search response format", - "ISA version format" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Update and search ISA": { - "Search by area only": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search by earliest time (excluded)": [ - "ISAs search response format", - "ISA version format" - ], - "Search by earliest time (included)": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ], - "Search by latest time (excluded)": [ - "ISAs search response format", - "ISA version format" - ], - "Search by latest time (included)": [ - "ISAs search response format", - "ISA ID matches", - "ISA version format", - "ISA version matches", - "ISA start time matches", - "ISA end time matches", - "ISA URL matches" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.isa_subscription_interactions.ISASubscriptionInteractions": { - "ISA Subscription Interactions": { - "New subscription within ISA is mutated to ISA boundary": [ - "Subscription that only barely overlaps the ISA contains the ISA" - ] - }, - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.isa_validation.ISAValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.subscription_simple.SubscriptionSimple": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Subscription Simple": { - "Mutate Subscription": [ - "Subscription can be mutated" - ], - "Query Deleted Subscription": [ - "Query by subscription ID should fail", - "Search for all subscriptions in ISA area", - "Search area that represents a loop is not allowed" - ], - "Query Existing Subscription": [ - "Get Subscription by ID", - "Search for all subscriptions in ISA area", - "No huge search area allowed" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.subscription_validation.SubscriptionValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Subscription duration limitations": { - "Try to create too-long subscription": [ - "Too-long subscription creation rejected" - ], - "Try to extend subscription": [ - "Valid subscription created", - "Subscription duration limited during update" - ] - }, - "Subscription quantity limitations": { - "Clean up subscriptions": [ - "Successful subscription search query", - "Subscription can be deleted" - ], - "Create maximum number of subscriptions": [ - "Create up to the maximum allowed number of subscriptions in an area" - ], - "Exceed maximum number of subscriptions": [ - "Enforce maximum number of subscriptions for an area" - ] - } - }, - "scenarios.astm.netrid.v22a.dss.token_validation.TokenValidation": { - "Setup": { - "Ensure clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - }, - "Token validation": { - "Token validation": [ - "Read scope cannot create an ISA", - "Missing token prevents creating an ISA", - "Fake token prevents creating an ISA", - "Correct token and scope can create ISA", - "Missing token prevents reading an ISA", - "Fake token prevents reading an ISA", - "Read scope cannot mutate an ISA", - "Missing token prevents mutating an ISA", - "Proper token is allowed to search for ISAs", - "Fake token cannot mutate an ISA", - "Fake token cannot search for ISAs", - "Missing token cannot search for ISAs", - "Read scope cannot delete an ISA", - "Missing token prevents ISA deletion", - "Fake token cannot delete an ISA", - "Correct token and scope can delete ISA", - "Notified subscriber" - ] - } - }, - "scenarios.astm.netrid.v22a.dss_interoperability.DSSInteroperability": { - "Interoperability sequence": { - "S11": [ - "ISA[P] deleted with proper response" - ], - "S13": [ - "Subscriptions queried successfully" - ], - "S14": [ - "404 with proper response" - ], - "S15": [ - "ISA[P] deleted with proper response" - ], - "S3": [ - "Subscription[P] returned with proper response" - ], - "S5": [ - "Can get ISA from primary DSS", - "Can modify ISA in primary DSS" - ], - "S7": [ - "404 with proper response" - ], - "S8": [ - "Subscriptions queried successfully" - ] - }, - "Prerequisites": { - "Test environment requirements": [ - "DSS instance is reachable" - ] - } - }, - "scenarios.astm.netrid.v22a.misbehavior.Misbehavior": { - "Invalid requests": { - "Injection": [ - "Valid flight" - ] - } - }, - "scenarios.astm.netrid.v22a.networked_uas_disconnect.NetworkedUASDisconnect": { - "Networked UAS disconnect": { - "Service Provider polling": [ - "Premature flight", - "Missing flight", - "Lingering flight" - ] - } - }, - "scenarios.astm.netrid.v22a.nominal_behavior.NominalBehavior": { - "Nominal flight": { - "Injection": [ - "Valid flight" - ], - "Observer polling": [ - "Minimal obfuscation distance of individual flights", - "Individual flights obfuscation", - "Minimal obfuscation distance of multiple flights clusters", - "Clustering count", - "Minimal display area of clusters", - "Premature flight", - "Missing flight", - "Lingering flight" - ], - "Service Provider polling": [ - "Premature flight", - "Missing flight", - "Lingering flight" - ] - } - }, - "scenarios.astm.netrid.v22a.operator_interactions.OperatorInteractions": { - "Failed ISA": { - "Enumerate operator notifications": [ - "Operator notified of discoverability failure" - ], - "Inject flight": [ - "Flight failed" - ] - }, - "In-flight notifications": { - "Poll Service Provider": [ - "Insufficient telemetry operator notification", - "Missing data operator notification" - ] - } - }, - "scenarios.astm.netrid.v22a.sp_notification_behavior.ServiceProviderNotificationBehavior": { - "Service Provider notification behavior": { - "Injection": [ - "Valid flight" - ], - "Mock USS Subscription": [ - "Subscription creation succeeds" - ] - }, - "Setup": { - "Clean workspace": [ - "Successful ISA query", - "Removed pre-existing ISA", - "Notified subscriber", - "Successful subscription search query", - "Subscription can be queried by ID", - "Subscription can be deleted" - ] - } - }, - "scenarios.astm.netrid.v22a.sp_operator_notify_missing_fields.SpOperatorNotifyMissingFields": { - "Missing fields flight": { - "Injection": [ - "Valid flight" - ] - } - }, - "scenarios.astm.netrid.v22a.sp_operator_notify_slow_update.ServiceProviderNotifiesSlowUpdates": { - "Slow updates flight": { - "Injection": [ - "Valid flight" - ] - } - }, - "scenarios.astm.utm.dss.synchronization.constraint_ref_synchronization.CRSynchronization": { - "CR synchronization": { - "Retrieve newly created CR": [ - "Newly created CR can be consistently retrieved from all DSS instances" - ], - "Retrieve updated CR": [ - "Updated CR can be consistently retrieved from all DSS instances" - ], - "Search for newly created CR": [ - "Newly created CR can be consistently searched for from all DSS instances" - ], - "Search for updated CR": [ - "Updated CR can be consistently searched for from all DSS instances" - ] - } - }, - "scenarios.astm.utm.dss.synchronization.op_intent_ref_synchronization.OIRSynchronization": { - "OIR synchronization": { - "Retrieve newly created OIR": [ - "Newly created OIR can be consistently retrieved from all DSS instances" - ], - "Retrieve updated OIR": [ - "Updated OIR can be consistently retrieved from all DSS instances" - ], - "Search for newly created OIR": [ - "Newly created OIR can be consistently searched for from all DSS instances" - ], - "Search for updated OIR": [ - "Updated OIR can be consistently searched for from all DSS instances" - ] - } - }, - "scenarios.astm.utm.flight_intent_validation.flight_intent_validation.FlightIntentValidation": { - "Validate precision of intersection computations": { - "Attempt to plan Tiny Overlap Conflict Flight": [ - "Failure" - ] - } - }, - "scenarios.eurocae.ed269.source_data_model.SourceDataModelValidation": { - "ED-269 data model compliance": { - "Valid source": [ - "Valid JSON", - "Valid schema and values" - ] - } - }, - "scenarios.faa.uft.message_signing_finalize.FinalizeMessageSigningReport": { - "Finalize message signing": { - "Signal mock USS": [ - "Successful finalization", - "All message signing headers present", - "Valid content digests", - "Valid signature" - ] - } - }, - "scenarios.faa.uft.message_signing_start.StartMessageSigningReport": { - "Start message signing": { - "Check mock USS readiness": [ - "Status ok", - "Ready" - ], - "Signal mock USS": [ - "Successful start" - ] - } - }, - "scenarios.interuss.mock_uss.configure_locality.ConfigureLocality": { - "Set locality": { - "Get current locality value": [ - "Query ok" - ], - "Set locality to desired value": [ - "Query ok" - ] - } - }, - "scenarios.interuss.mock_uss.unconfigure_locality.UnconfigureLocality": { - "Restore locality": { - "Set locality to old value": [ - "Query ok" - ] - } - } - } -} \ No newline at end of file + "scenarios": {} +} diff --git a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.md b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.md index 31f2d162da..c60f03dd89 100644 --- a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.md +++ b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.md @@ -14,10 +14,10 @@ The file or url of the document to be tested. ### Valid source test step -#### Valid JSON check +#### 🛑 Valid JSON check The JSON file is properly formatted and can be read successfully. -#### Valid schema and values check +#### 🛑 Valid schema and values check The file respects the ED-269 schema and values are valid. diff --git a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py index 3c61dae040..f1abcb0870 100644 --- a/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py +++ b/monitoring/uss_qualifier/scenarios/eurocae/ed269/source_data_model.py @@ -4,7 +4,6 @@ from implicitdict import ImplicitDict, StringBasedDateTime from uas_standards.eurocae_ed269 import UASZoneVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.eurocae.ed269.source_document import ( SourceDocument, ) @@ -46,7 +45,6 @@ def run(self, context: ExecutionContext): except json.decoder.JSONDecodeError as e: check.record_failed( summary="Unable to deserialize the document as JSON", - severity=Severity.High, details=str(e), ) @@ -59,7 +57,6 @@ def run(self, context: ExecutionContext): except ValueError as e: check.record_failed( summary="Invalid format error", - severity=Severity.High, details=str(e), ) diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md index bd637d06be..22759b5934 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.md @@ -13,18 +13,18 @@ The means to communicate with the mock USS that has been collecting message sign ### Signal mock USS test step -#### Successful finalization check +#### 🛑 Successful finalization check If the mock USS doesn't finalize the message signing report successfully, this check will fail. -#### All message signing headers present check +#### 🛑 All message signing headers present check The first step of message signing analysis is to check for the existence of all of the required message signing headers. These headers are `content-digest`, `utm-message-signature`, `utm-message-signature-input`, and `x-utm-jws-header`. If any of these headers are missing in incoming requests to the mock_uss, or are missing in received responses from the USS under test, then that will generate an issue. -#### Valid content digests check +#### 🛑 Valid content digests check -The sha-512 hash on either the request/response body (depending on what is being analyzed), is taken, and this value is compared to the one received in the `content-digest` header. If these values differ, this will generate an issue showing the difference in the values. +The sha-512 hash on either the request/response body (depending on what is being analyzed), is taken, and this value is compared to the one received in the `content-digest` header. If these values differ, this will generate an issue showing the difference in the values. -#### Valid signature check +#### 🛑 Valid signature check -The `utm-message-signature` is analyzed to ensure that it is a valid signature. To do this check, the signature base is created, and the public key served in the `x5u` field within the `x-utm-jws-header` is retreived. Using the value in the `utm-message-signature` header field, the public key, and the recreated signature base, it is determined whether or not the signature is valid. \ No newline at end of file +The `utm-message-signature` is analyzed to ensure that it is a valid signature. To do this check, the signature base is created, and the public key served in the `x5u` field within the `x-utm-jws-header` is retreived. Using the value in the `utm-message-signature` header field, the public key, and the recreated signature base, it is determined whether or not the signature is valid. diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py index 6f7179f382..69d9c52109 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_finalize.py @@ -1,4 +1,3 @@ -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSClient, MockUSSResource, @@ -29,7 +28,6 @@ def run(self, context: ExecutionContext): check.record_failed( summary="Failed to finalize message signing report", details="TODO", - severity=Severity.High, query_timestamps=[], ) return diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md index 2c1fe50e1e..93c87cba0b 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.md @@ -12,16 +12,16 @@ The means to communicate with the mock USS that will collect message signing dat ### Check mock USS readiness test step -#### Status ok check +#### 🛑 Status ok check If the mock USS doesn't respond properly to a request for its status, this check will fail. -#### Ready check +#### 🛑 Ready check If the mock USS doesn't indicate Ready for its scd functionality, this check will fail. ### Signal mock USS test step -#### Successful start check +#### 🛑 Successful start check If the mock USS doesn't start capturing message signing data successfully, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py index 5a2ed45358..3f01991d1e 100644 --- a/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py +++ b/monitoring/uss_qualifier/scenarios/faa/uft/message_signing_start.py @@ -1,4 +1,3 @@ -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSClient, MockUSSResource, @@ -31,7 +30,6 @@ def run(self, context: ExecutionContext): check.record_failed( summary="Failed to get status from mock USS", details=f"Status code {query.status_code}", - severity=Severity.High, query_timestamps=[query.request.timestamp], ) return # Return if this scenario cannot continue @@ -42,7 +40,6 @@ def run(self, context: ExecutionContext): check.record_failed( summary="Mock USS SCD functionality not ready", details=f"Status indicated as: {status}", - severity=Severity.High, query_timestamps=[query.request.timestamp], ) return @@ -59,7 +56,6 @@ def run(self, context: ExecutionContext): check.record_failed( summary="Failed to start message signing report", details="TODO", - severity=Severity.High, query_timestamps=[], ) return diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md index cf2a4ec27e..5208ed3381 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.md @@ -18,18 +18,18 @@ The locality to set all mock USS instances to. ### Get current locality value test step -#### Query ok check +#### 🛑 Query ok check If a mock USS instance doesn't respond properly to a request to get its current locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. ### Set locality to desired value test step -#### Query ok check +#### 🛑 Query ok check If a mock USS instance doesn't respond properly to a request to change its locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. ## Cleanup -### Restore locality check +### 🛑 Restore locality check If uss_qualifier cannot restore a mock_uss instance's locality to its old value when rolling back incomplete locality changes, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py index f4d613bd17..8296c2f07c 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py @@ -1,7 +1,6 @@ from typing import List from monitoring.monitorlib.locality import LocalityCode -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.interuss.mock_uss.client import ( MockUSSClient, MockUSSsResource, @@ -44,13 +43,11 @@ def run(self, context: ExecutionContext): if query.status_code != 200: check.record_failed( f"Get current locality returned {query.status_code}", - Severity.High, query_timestamps=[query.request.initiated_at.datetime], ) elif locality_code is None: check.record_failed( f"Missing current locality code", - Severity.High, "Query to get current locality value did not produce a valid locality code", query_timestamps=[query.request.initiated_at.datetime], ) @@ -68,7 +65,6 @@ def run(self, context: ExecutionContext): if query.status_code != 200: check.record_failed( f"Set locality returned {query.status_code}", - Severity.High, query_timestamps=[query.request.initiated_at.datetime], ) self.to_unconfigure.append( diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md index e679e7dde7..c9232b06e5 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.md @@ -12,6 +12,6 @@ No resources are needed because they are stored by the [ConfigureLocality test s The most recent ConfigureLocality test scenario instances recorded the old locality values and mock_uss instances. This test step consumes that information to restore localities to their old values. -#### Query ok check +#### ⚠️ Query ok check If a mock USS instance doesn't respond properly to a request to change its locality, **[interuss.mock_uss.hosted_instance.ExposeInterface](../../../requirements/interuss/mock_uss/hosted_instance.md)** is not met. diff --git a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py index 604bb4798d..651122f1c4 100644 --- a/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py +++ b/monitoring/uss_qualifier/scenarios/interuss/mock_uss/unconfigure_locality.py @@ -2,7 +2,6 @@ from typing import List from monitoring.monitorlib.locality import LocalityCode -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.interuss.mock_uss.client import MockUSSClient from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -41,7 +40,6 @@ def run(self, context: ExecutionContext): if query.status_code != 200: check.record_failed( f"Set locality returned {query.status_code}", - Severity.Medium, query_timestamps=[query.request.initiated_at.datetime], )