diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/cleanup_after_testcase_oir_subs.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/cleanup_after_testcase_oir_subs.md new file mode 100644 index 0000000000..ba7ee4766d --- /dev/null +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/cleanup_after_testcase_oir_subs.md @@ -0,0 +1,7 @@ +# Cleanup After Test Case test step fragment + +This fragment can be used by test cases that need to clean up after themselves by deleting the OIRs and subscriptions created during the test. + +## [Delete OIRs](./fragments/oir/crud/delete_query.md) + +## [Delete Subscriptions](./fragments/sub/crud/delete_query.md) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/__init__.py index 1e7c50549a..8d979efe9a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/oir/crud/__init__.py @@ -115,3 +115,25 @@ def delete_oir_query( ) # Failure of the query has a severity that will interrupt the test: # no need to return anything + + +def get_oir_query( + scenario: TestScenarioType, dss: DSSInstance, oir_id: EntityID +) -> Tuple[OperationalIntentReference, Query]: + """ + Issue a request to get an OIR by ID to the DSS instance, wrapped in a check documented in `get_query.md`. + """ + with scenario.check( + "Get operational intent reference by ID", dss.participant_id + ) as check: + try: + oir, query = dss.get_op_intent_reference(oir_id) + scenario.record_query(query) + return oir, query + except QueryError as qe: + scenario.record_queries(qe.queries) + check.record_failed( + summary="Could not get operational intent reference", + details=f"Failed to get operational intent reference with error code {qe.cause_status_code}: {qe.msg}", + query_timestamps=qe.query_timestamps, + ) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/__init__.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/__init__.py index 838a67eb80..4c1526ff10 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/__init__.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/sub/crud/__init__.py @@ -2,6 +2,7 @@ from uas_standards.astm.f3548.v21.api import OperationalIntentReference, Subscription +from monitoring.monitorlib.fetch.rid import FetchedSubscription from monitoring.monitorlib.mutate.scd import MutatedSubscription from monitoring.uss_qualifier.resources.astm.f3548.v21.dss import DSSInstance from monitoring.uss_qualifier.resources.astm.f3548.v21.subscription_params import ( @@ -33,3 +34,44 @@ def sub_create_query( query_timestamps=[sub.request.timestamp], ) return sub.subscription, sub.operational_intent_references, sub + + +def sub_get_query( + scenario: TestScenarioType, + dss: DSSInstance, + sub_id: str, +) -> Tuple[Subscription, FetchedSubscription]: + with scenario.check("Get Subscription by ID", dss.participant_id) as check: + fetched_sub = dss.get_subscription(sub_id) + scenario.record_query(fetched_sub) + if not fetched_sub.success: + check.record_failed( + summary="Subscription query failed", + details=f"Failed to query subscription {sub_id} referenced by oid {self._oir_a_id} with code {fetched_sub.response.status_code}. Message: {fetched_sub.error_message}", + query_timestamps=fetched_sub.query_timestamps, + ) + return None + + return fetched_sub.subscription, fetched_sub + + +def sub_delete_query( + scenario: TestScenarioType, + dss: DSSInstance, + sub_id: str, + sub_version: str, +) -> Tuple[Subscription, List[OperationalIntentReference], MutatedSubscription]: + """Implements check documented in `delete_query.md`.""" + with scenario.check( + "Subscription can be deleted", + [dss.participant_id], + ) as check: + sub = dss.delete_subscription(sub_id, sub_version) + scenario.record_query(sub) + if not sub.success: + check.record_failed( + summary="Delete subscription query failed", + details=f"Failed to delete a subscription on DSS instance with code {sub.status_code}: {sub.error_message}", + query_timestamps=[sub.request.timestamp], + ) + return sub.subscription, sub.operational_intent_references, sub diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.md b/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.md index fcb3117b2f..1e3114fbc6 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.md +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.md @@ -134,18 +134,12 @@ or **[astm.f3548.v21.DSS0005,5](../../../../requirements/astm/f3548/v21.md)**. #### [No implicit subscription was attached](./fragments/oir/oir_has_no_subscription.md) +### [Cleanup After Test Case test step](./fragments/oir/crud/delete_query.md) + ## Implicit subscriptions are properly deleted when required by OIR mutation test case This test case verifies that implicit subscriptions are properly removed if they become unnecessary following the mutation of an OIR. -### Ensure clean workspace test step - - - -#### [Clean any existing OIRs with known test IDs](clean_workspace_op_intents.md) - -#### [Clean any existing subscriptions with known test IDs](clean_workspace_subs.md) - ### Create two OIRs with implicit subscription test step Creates two OIRs with an implicit subscription, which will then be replaced by an explicitly created subscription @@ -186,18 +180,12 @@ This step updates the OIR to not use any subscription, and expects the implicit If the implicit subscription that was attached to the OIR is still present after the OIR is updated to use another subscription, the DSS is failing to properly manage implicit subscriptions for OIRs, and is therefore in violation of **[astm.f3548.v21.DSS0005,1](../../../../requirements/astm/f3548/v21.md)**. +### [Cleanup After Test Case test step](./cleanup_after_testcase_oir_subs.md) + ## Implicit subscriptions are expanded as needed test case This test case checks that a DSS will properly expand an implicit subscription to cover an OIR that is being attached to it. -### Ensure clean workspace test step - - - -#### [Clean any existing OIRs with known test IDs](clean_workspace_op_intents.md) - -#### [Clean any existing subscriptions with known test IDs](clean_workspace_subs.md) - ### Create an OIR with implicit subscription test step Create an OIR with which interactions will be tested and request an implicit @@ -225,18 +213,12 @@ in which case the DSS is in violation of **[astm.f3548.v21.DSS0005,1](../../../. Ensure that the attached implicit subscription has been expanded +### [Cleanup After Test Case test step](./fragments/oir/crud/delete_query.md) + ## Existing implicit subscription can replace an OIR's explicit subscription test case This test case verifies that an implicit subscription can be used to replace an explicit subscription attached to an OIR. -### Ensure clean workspace test step - -Reset the workspace for this test case. - -#### [Clean any existing OIRs with known test IDs](clean_workspace_op_intents.md) - -#### [Clean any existing subscriptions with known test IDs](clean_workspace_subs.md) - ### [Create an explicit subscription test step](./fragments/sub/crud/create_query.md) Create an explicit subscription to be initially set on the first OIR created in this test case @@ -267,18 +249,12 @@ Confirm that the query to replace the second OIR's explicit subscription with th #### [First OIR is now attached to the specified implicit subscription](fragments/oir/oir_has_expected_subscription.md) +### [Cleanup After Test Case test step](./cleanup_after_testcase_oir_subs.md) + ## Existing implicit subscription can be attached to OIR without subscription test case This test case verifies that an implicit subscription can be attached to an OIR that is not currently attached to any subscription. -### Ensure clean workspace test step - -Reset the workspace for this test case. - -#### [Clean any existing OIRs with known test IDs](clean_workspace_op_intents.md) - -#### [Clean any existing subscriptions with known test IDs](clean_workspace_subs.md) - ### [Create OIR with no subscription test step](./fragments/oir/crud/create_query.md) #### [OIR is not attached to an implicit subscription](./fragments/oir/oir_has_no_subscription.md) @@ -301,19 +277,13 @@ Confirms that the DSS properly attached the first OIR to the implicit subscripti #### [First OIR is now attached to the specified implicit subscription](fragments/oir/oir_has_expected_subscription.md) +### [Cleanup After Test Case test step](./fragments/oir/crud/delete_query.md) + ## OIR without subscription can be mutated without a new subscription being attached test case This test case ensures that, when a client mutates an OIR not attached to any subscription without specifiying either a subscription identifier nor parameters for an implicit subscription, the DSS under test will correctly keep the OIR unattached to any subscription. -### Ensure clean workspace test step - -Reset the workspace for this test case. - -#### [Clean any existing OIRs with known test IDs](clean_workspace_op_intents.md) - -#### [Clean any existing subscriptions with known test IDs](clean_workspace_subs.md) - ### Create OIR with no subscription test step #### [Create OIR](./fragments/oir/crud/create_query.md) @@ -326,6 +296,37 @@ Reset the workspace for this test case. #### [OIR is not attached to any subscription](./fragments/oir/oir_has_no_subscription.md) +### [Cleanup After Test Case test step](./fragments/oir/crud/delete_query.md) + +## Request new implicit subscription when mutating an OIR with existing explicit subscription test case + +This test case ensures that a DSS properly allows a client to request that a new implicit subscription be created for an existing OIR +with an explicit subscription attached. + +### [Create an explicit subscription test step](./fragments/sub/crud/create_query.md) + +### [Create OIR with explicit subscription test step](./fragments/oir/crud/create_query.md) + +#### [OIR is attached to the expected subscription](./fragments/oir/oir_has_expected_subscription.md) + +### [Mutate OIR to request new implicit subscription test step](./fragments/oir/crud/update_query.md) + +### Validate that the OIR is now attached to an implicit subscription test step + +#### [Get OIR](./fragments/oir/crud/read_query.md) + +#### 🛑 OIR is attached to a new subscription check + +If the DSS under test fails to attach the OIR to a subscription that is different from the one it is currently attached to when it is requested to do so, +it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../requirements/astm/f3548/v21.md)**. + +#### [Get subscription](./fragments/sub/crud/read_query.md) + +#### 🛑 OIR is now attached to an implicit subscription check + +If the DSS under test fails to attach the OIR to an implicit subscription (which may either already exist or be newly created) when it is requested to do so, +it is in violation of **[astm.f3548.v21.DSS0005,1](../../../../requirements/astm/f3548/v21.md)**. + ## Cleanup ### [Remove OIRs created during this test](clean_workspace_op_intents.md) diff --git a/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.py b/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.py index bbc8858dc1..1fdd6258f1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.py +++ b/monitoring/uss_qualifier/scenarios/astm/utm/dss/oir_implicit_sub_handling.py @@ -5,6 +5,7 @@ from uas_standards.astm.f3548.v21.api import ( OperationalIntentReference, OperationalIntentState, + PutOperationalIntentReferenceParameters, SubscriberToNotify, Subscription, SubscriptionID, @@ -23,8 +24,15 @@ from monitoring.uss_qualifier.scenarios.astm.utm.dss.fragments.oir import ( check_oir_has_correct_subscription, ) +from monitoring.uss_qualifier.scenarios.astm.utm.dss.fragments.oir.crud import ( + delete_oir_query, + get_oir_query, + update_oir_query, +) from monitoring.uss_qualifier.scenarios.astm.utm.dss.fragments.sub.crud import ( sub_create_query, + sub_delete_query, + sub_get_query, ) from monitoring.uss_qualifier.scenarios.scenario import TestScenario from monitoring.uss_qualifier.suites.suite import ExecutionContext @@ -101,7 +109,7 @@ def __init__( def run(self, context: ExecutionContext): self.begin_test_scenario(context) self.begin_test_case("Setup") - self._setup_case() + self._setup_scenario() self.end_test_case() self.begin_test_case( @@ -112,41 +120,48 @@ def run(self, context: ExecutionContext): self.begin_test_case("Implicit subscriptions always properly cover their OIR") self._case_2_implicit_subs_cover_their_oir() + self._clean_after_test_case() self.end_test_case() self.begin_test_case( "Implicit subscriptions are properly deleted when required by OIR mutation" ) - self._setup_case() self._case_3_implicit_sub_deleted_after_oir_mutation() + self._clean_after_test_case() self.end_test_case() self.begin_test_case("Implicit subscriptions are expanded as needed") - self._setup_case() self._case_4_implicit_sub_expansion() + self._clean_after_test_case() self.end_test_case() self.begin_test_case( "Existing implicit subscription can replace an OIR's explicit subscription" ) - self._setup_case() self._case_5_replace_explicit_sub_with_implicit() + self._clean_after_test_case() self.end_test_case() self.begin_test_case( "Existing implicit subscription can be attached to OIR without subscription" ) - self._setup_case() self._case_6_attach_implicit_sub_to_oir_without_subscription() + self._clean_after_test_case() self.end_test_case() self.begin_test_case( "OIR without subscription can be mutated without a new subscription being attached" ) - self._setup_case() + self._case_7_mutate_oir_without_subscription() + self._clean_after_test_case() self.end_test_case() + self.begin_test_case( + "Request new implicit subscription when mutating an OIR with existing explicit subscription" + ) + self._case_8_mutate_oir_explicit_request_implicit() + self.end_test_case() self.end_test_scenario() def _case_1_implicit_sub_cleanup_on_oir_deletion(self): @@ -272,6 +287,7 @@ def _case_2_step_mutate_oir_with_implicit_sub_specify_implicit_params(self): subscription_id=None, ) self.record_query(q) + self._oir_b_ovn = oir.ovn except QueryError as e: self.record_queries(e.queries) check.record_failed( @@ -482,6 +498,7 @@ def _case_1_step_delete_single_oir(self): self._oir_a_id, self._oir_a_ovn ) self.record_query(q) + self._oir_a_ovn = None except QueryError as e: self.record_queries(e.queries) check.record_failed( @@ -666,6 +683,7 @@ def _case_4_expand_oir_same_implicit_sub(self): subscription_id=self._implicit_sub_1.id, ) self.record_query(q) + self._oir_a_ovn = oir.ovn except QueryError as e: self.record_queries(e.queries) check.record_failed( @@ -711,8 +729,6 @@ def _case_4_expand_oir_same_implicit_sub(self): ) def _case_5_replace_explicit_sub_with_implicit(self): - # TODO explicit subscription creation could be moved to a common fragment to be shared - # with the OIRSimple scenario once it has been extended with new corner cases. self.begin_test_step("Create an explicit subscription") sub_params = self._planning_area.get_new_subscription_params( subscription_id=self._sub_id, @@ -721,29 +737,17 @@ def _case_5_replace_explicit_sub_with_implicit(self): notify_for_op_intents=True, notify_for_constraints=False, ) - with self.check("Create subscription query succeeds", self._pid) as check: - try: - sub_q = self._dss.upsert_subscription(**sub_params) - self.record_query(sub_q) - except QueryError as qe: - self.record_queries(qe.queries) - check.record_failed( - summary="Could not create subscription", - details=f"Failed to create subscription with error code {qe.cause_status_code}: {qe.msg}", - query_timestamps=qe.query_timestamps, - ) - sub_explicit = sub_q.subscription - self._explicit_sub = sub_explicit + self._explicit_sub, _, _ = sub_create_query(self, self._dss, sub_params) self.end_test_step() self.begin_test_step("Create first OIR with an explicit subscription") oir_explicit, _, _, _ = self._create_oir( oir_id=self._oir_a_id, - time_start=sub_explicit.time_start.value.datetime, - time_end=sub_explicit.time_end.value.datetime, + time_start=self._explicit_sub.time_start.value.datetime, + time_end=self._explicit_sub.time_end.value.datetime, relevant_ovns=[], with_implicit_sub=False, - subscription_id=sub_explicit.id, + subscription_id=self._explicit_sub.id, ) self._oir_a_ovn = oir_explicit.ovn self.end_test_step() @@ -751,8 +755,8 @@ def _case_5_replace_explicit_sub_with_implicit(self): self.begin_test_step("Create second OIR with an implicit subscription") oir_implicit, _, sub_implicit, _ = self._create_oir( oir_id=self._oir_b_id, - time_start=sub_explicit.time_start.value.datetime, - time_end=sub_explicit.time_end.value.datetime, + time_start=self._explicit_sub.time_start.value.datetime, + time_end=self._explicit_sub.time_end.value.datetime, relevant_ovns=[oir_explicit.ovn], with_implicit_sub=True, ) @@ -770,8 +774,8 @@ def _case_5_replace_explicit_sub_with_implicit(self): oir, subs, q = self._dss.put_op_intent( extents=[ self._planning_area.get_volume4d( - sub_explicit.time_start.value.datetime, - sub_explicit.time_end.value.datetime, + self._explicit_sub.time_start.value.datetime, + self._explicit_sub.time_end.value.datetime, ).to_f3548v21() ], key=[oir_implicit.ovn], @@ -943,6 +947,7 @@ def _case_7_mutate_oir_without_subscription(self): force_no_implicit_subscription=True, ) self.record_query(q) + self._oir_a_ovn = oir_mutated_no_sub.ovn except QueryError as e: self.record_queries(e.queries) check.record_failed( @@ -958,7 +963,103 @@ def _case_7_mutate_oir_without_subscription(self): ) self.end_test_step() - def _setup_case(self): + def _case_8_mutate_oir_explicit_request_implicit(self): + self.begin_test_step("Create an explicit subscription") + sub_params = self._planning_area.get_new_subscription_params( + subscription_id=self._sub_id, + start_time=self._time_2, + duration=self._time_3 - self._time_2, + notify_for_op_intents=True, + notify_for_constraints=False, + ) + self._explicit_sub, _, _ = sub_create_query(self, self._dss, sub_params) + self.end_test_step() + + self.begin_test_step("Create OIR with explicit subscription") + oir_explicit_sub, _, _, _ = self._create_oir( + oir_id=self._oir_a_id, + time_start=self._time_2, + time_end=self._time_3, + relevant_ovns=[], + subscription_id=self._sub_id, + with_implicit_sub=False, + ) + self._oir_a_ovn = oir_explicit_sub.ovn + check_oir_has_correct_subscription( + self, + self._dss, + self._oir_a_id, + expected_sub_id=self._sub_id, + ) + self.end_test_step() + + self.begin_test_step("Mutate OIR to request new implicit subscription") + oir_mutated_to_implicit_sub, _, q = update_oir_query( + scenario=self, + dss=self._dss, + oir_id=self._oir_a_id, + ovn=self._oir_a_ovn, + oir_params=PutOperationalIntentReferenceParameters( + extents=[ + self._planning_area.get_volume4d( + self._time_2, self._time_3 + ).to_f3548v21() + ], + key=[], + # Update to a state that requires a subscription + state=OperationalIntentState.Activated, + uss_base_url=DUMMY_BASE_URL, + subscription_id=None, + ), + ) + self._oir_a_ovn = oir_mutated_to_implicit_sub.ovn + self.end_test_step() + + self.begin_test_step( + "Validate that the OIR is now attached to an implicit subscription" + ) + + def check_sub_changed(oir, mutate_q): + with self.check( + "OIR is attached to a new subscription", self._pid + ) as check: + if oir.subscription_id is None: + check.record_failed( + summary="OIR not attached to any subscription", + details="OIR is not attached to any subscription, when a request for an implicit subscription was made.", + query_timestamps=[mutate_q.timestamp], + ) + if oir.subscription_id == self._explicit_sub.id: + check.record_failed( + summary="OIR is still attached to the explicit subscription", + details=f"OIR is still attached to the explicit subscription {self._explicit_sub.id}, when a request for an implicit subscription was made.", + query_timestamps=[mutate_q.timestamp], + ) + + # First check the OIR as it was returned by the mutation endpoint + check_sub_changed(oir_mutated_to_implicit_sub, q) + + # Then, do an actual query of the OIR + oir_queried, q = get_oir_query(self, self._dss, self._oir_a_id) + + check_sub_changed(oir_queried, q) + + # Now fetch the subscription and check it is implicit + fetched_sub, _ = sub_get_query(self, self._dss, oir_queried.subscription_id) + + with self.check( + "OIR is now attached to an implicit subscription", self._pid + ) as check: + if not fetched_sub.implicit_subscription: + check.record_failed( + summary="OIR is not attached to an implicit subscription", + details=f"Subscription {self._sub_id} referenced by OIR {self._oir_a_id} is not an implicit subscription.", + query_timestamps=[fetched_sub.request.timestamp], + ) + + self.end_test_step() + + def _setup_scenario(self): # T0 corresponds to 'now' self._time_0 = arrow.utcnow().datetime @@ -1000,6 +1101,35 @@ def _clean_workspace(self): ) test_step_fragments.cleanup_sub(self, self._dss, self._sub_id) + def _clean_after_test_case(self): + self.begin_test_step("Cleanup After Test Case") + + if self._oir_a_ovn: + delete_oir_query( + scenario=self, dss=self._dss, oir_id=self._oir_a_id, ovn=self._oir_a_ovn + ) + self._oir_a_ovn = None + + if self._oir_b_ovn: + delete_oir_query( + scenario=self, dss=self._dss, oir_id=self._oir_b_id, ovn=self._oir_b_ovn + ) + self._oir_b_ovn = None + + if self._oir_c_ovn: + delete_oir_query( + scenario=self, dss=self._dss, oir_id=self._oir_c_id, ovn=self._oir_c_ovn + ) + self._oir_c_ovn = None + + if self._explicit_sub: + sub_delete_query( + self, self._dss, self._explicit_sub.id, self._explicit_sub.version + ) + self._explicit_sub = None + + self.end_test_step() + def cleanup(self): self.begin_cleanup() self._clean_workspace()