From 4f177e3330d6faca068e72306fcead97f6759844 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 30 Sep 2025 15:51:01 +0200 Subject: [PATCH 01/23] Created product type, blocks, workflows and migration files --- docker/netbox/Dockerfile | 2 +- ..._0e8d17ce0f06_reconcile_workflows_l2vpn.py | 3 +- .../2025-09-30_a87d11eb8dd1_add_nsistp.py | 101 ++++++++++++++ products/__init__.py | 7 + products/product_blocks/nsistp.py | 52 ++++++++ products/product_types/nsistp.py | 23 ++++ templates/nsistp.yaml | 64 +++++++++ translations/en-GB.json | 126 +++++++++--------- workflows/__init__.py | 6 + workflows/nsistp/create_nsistp.py | 115 ++++++++++++++++ workflows/nsistp/modify_nsistp.py | 106 +++++++++++++++ workflows/nsistp/shared/forms.py | 0 workflows/nsistp/terminate_nsistp.py | 44 ++++++ workflows/nsistp/validate_nsistp.py | 21 +++ 14 files changed, 606 insertions(+), 64 deletions(-) create mode 100644 migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py create mode 100644 products/product_blocks/nsistp.py create mode 100644 products/product_types/nsistp.py create mode 100644 templates/nsistp.yaml create mode 100644 workflows/nsistp/create_nsistp.py create mode 100644 workflows/nsistp/modify_nsistp.py create mode 100644 workflows/nsistp/shared/forms.py create mode 100644 workflows/nsistp/terminate_nsistp.py create mode 100644 workflows/nsistp/validate_nsistp.py diff --git a/docker/netbox/Dockerfile b/docker/netbox/Dockerfile index 4f85e11..73ca5f6 100644 --- a/docker/netbox/Dockerfile +++ b/docker/netbox/Dockerfile @@ -1,4 +1,4 @@ -FROM netboxcommunity/netbox:v4.4.1 +FROM netboxcommunity/netbox:v4.0-2.9.1 # NOTE: when updating the Netbox version, remember to update the database snapshot. See docker/postgresql/README.md for details diff --git a/migrations/versions/schema/2025-08-28_0e8d17ce0f06_reconcile_workflows_l2vpn.py b/migrations/versions/schema/2025-08-28_0e8d17ce0f06_reconcile_workflows_l2vpn.py index fc540f1..08a919c 100644 --- a/migrations/versions/schema/2025-08-28_0e8d17ce0f06_reconcile_workflows_l2vpn.py +++ b/migrations/versions/schema/2025-08-28_0e8d17ce0f06_reconcile_workflows_l2vpn.py @@ -6,12 +6,11 @@ """ -import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "0e8d17ce0f06" -down_revision = "d946c20663d3" +down_revision = "bc54616fefcf" branch_labels = None depends_on = None diff --git a/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py b/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py new file mode 100644 index 0000000..ada9278 --- /dev/null +++ b/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py @@ -0,0 +1,101 @@ +"""Add nsistp product. + +Revision ID: a87d11eb8dd1 +Revises: 0e8d17ce0f06 +Create Date: 2025-09-30 15:50:36.882313 + +""" + +from uuid import uuid4 + +from alembic import op +from orchestrator.migrations.helpers import create, create_workflow, delete, delete_workflow, ensure_default_workflows +from orchestrator.targets import Target + +# revision identifiers, used by Alembic. +revision = "a87d11eb8dd1" +down_revision = "0e8d17ce0f06" +branch_labels = None +depends_on = None + +new_products = { + "products": { + "nsistp": { + "product_id": uuid4(), + "product_type": "Nsistp", + "description": "NSISTP", + "tag": "NSISTP", + "status": "active", + "root_product_block": "Nsistp", + "fixed_inputs": {}, + }, + }, + "product_blocks": { + "Nsistp": { + "product_block_id": uuid4(), + "description": "nsistp product block", + "tag": "NSISTP", + "status": "active", + "resources": { + "topology": "Topology type or identifier for the service instance", + "stp_id": "Unique identifier for the Service Termination Point", + "stp_description": "Description of the Service Termination Point", + "is_alias_in": "Indicates if the incoming SAP is an alias", + "is_alias_out": "Indicates if the outgoing SAP is an alias", + "expose_in_topology": "Whether to expose this STP in the topology view", + "bandwidth": "Requested bandwidth for the service instance (in Mbps)", + }, + "depends_on_block_relations": [ + "SAP", + ], + }, + }, + "workflows": {}, +} + +new_workflows = [ + { + "name": "create_nsistp", + "target": Target.CREATE, + "is_task": False, + "description": "Create nsistp", + "product_type": "Nsistp", + }, + { + "name": "modify_nsistp", + "target": Target.MODIFY, + "is_task": False, + "description": "Modify nsistp", + "product_type": "Nsistp", + }, + { + "name": "terminate_nsistp", + "target": Target.TERMINATE, + "is_task": False, + "description": "Terminate nsistp", + "product_type": "Nsistp", + }, + { + "name": "validate_nsistp", + "target": Target.VALIDATE, + "is_task": True, + "description": "Validate nsistp", + "product_type": "Nsistp", + }, +] + + +def upgrade() -> None: + conn = op.get_bind() + create(conn, new_products) + for workflow in new_workflows: + create_workflow(conn, workflow) + ensure_default_workflows(conn) + + +def downgrade() -> None: + conn = op.get_bind() + for workflow in new_workflows: + delete_workflow(conn, workflow["name"]) + + delete(conn, new_products) diff --git a/products/__init__.py b/products/__init__.py index 8d45115..9a0078f 100644 --- a/products/__init__.py +++ b/products/__init__.py @@ -32,3 +32,10 @@ "l2vpn": L2vpn, } ) +from products.product_types.nsistp import Nsistp + +SUBSCRIPTION_MODEL_REGISTRY.update( + { + "nsistp": Nsistp, + }, +) # fmt:skip diff --git a/products/product_blocks/nsistp.py b/products/product_blocks/nsistp.py new file mode 100644 index 0000000..3f6c6f3 --- /dev/null +++ b/products/product_blocks/nsistp.py @@ -0,0 +1,52 @@ +# products/product_blocks/nsistp.py +from typing import Annotated + +from annotated_types import Len +from orchestrator.domain.base import ProductBlockModel +from orchestrator.types import SI, SubscriptionLifecycle +from pydantic import computed_field + +from products.product_blocks.sap import SAPBlock, SAPBlockInactive, SAPBlockProvisioning + +ListOfSap = Annotated[list[SI], Len(min_length=2, max_length=8)] + + +class NsistpBlockInactive(ProductBlockModel, product_block_name="Nsistp"): + sap: ListOfSap[SAPBlockInactive] + topology: str | None = None + stp_id: str | None = None + stp_description: str | None = None + is_alias_in: str | None = None + is_alias_out: str | None = None + expose_in_topology: bool | None = None + bandwidth: int | None = None + + +class NsistpBlockProvisioning( + NsistpBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] +): + sap: ListOfSap[SAPBlockProvisioning] + topology: str + stp_id: str + stp_description: str | None = None + is_alias_in: str | None = None + is_alias_out: str | None = None + expose_in_topology: bool | None = None + bandwidth: int | None = None + + @computed_field + @property + def title(self) -> str: + # TODO: format correct title string + return f"{self.name}" + + +class NsistpBlock(NsistpBlockProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): + sap: ListOfSap[SAPBlock] + topology: str + stp_id: str + stp_description: str | None = None + is_alias_in: str | None = None + is_alias_out: str | None = None + expose_in_topology: bool | None = None + bandwidth: int | None = None diff --git a/products/product_types/nsistp.py b/products/product_types/nsistp.py new file mode 100644 index 0000000..a1ea2c4 --- /dev/null +++ b/products/product_types/nsistp.py @@ -0,0 +1,23 @@ +# products/product_types/nsistp.py +from orchestrator.domain.base import SubscriptionModel +from orchestrator.types import SubscriptionLifecycle + +from products.product_blocks.nsistp import ( + NsistpBlock, + NsistpBlockInactive, + NsistpBlockProvisioning, +) + + +class NsistpInactive(SubscriptionModel, is_base=True): + nsistp: NsistpBlockInactive + + +class NsistpProvisioning( + NsistpInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] +): + nsistp: NsistpBlockProvisioning + + +class Nsistp(NsistpProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): + nsistp: NsistpBlock diff --git a/templates/nsistp.yaml b/templates/nsistp.yaml new file mode 100644 index 0000000..e8510c6 --- /dev/null +++ b/templates/nsistp.yaml @@ -0,0 +1,64 @@ +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# This file describes the "L2VPN" product +# +config: + summary_forms: true +name: nsistp +type: Nsistp +tag: NSISTP +description: "NSISTP" +product_blocks: + - name: nsistp + type: Nsistp + tag: NSISTP + description: "nsistp product block" + fields: + - name: sap + type: list + description: "Virtual circuit service access points" + list_type: SAP + min_items: 2 + max_items: 8 + required: provisioning + - name: topology + description: "Topology type or identifier for the service instance" + type: str + required: provisioning + modifiable: + - name: stp_id + description: "Unique identifier for the Service Termination Point" + type: str + required: provisioning + - name: stp_description + description: "Description of the Service Termination Point" + type: str + modifiable: + - name: is_alias_in + description: "Indicates if the incoming SAP is an alias" + type: str + modifiable: + - name: is_alias_out + description: "Indicates if the outgoing SAP is an alias" + type: str + modifiable: + - name: expose_in_topology + description: "Whether to expose this STP in the topology view" + type: bool + modifiable: + - name: bandwidth + description: "Requested bandwidth for the service instance (in Mbps)" + type: int + modifiable: diff --git a/translations/en-GB.json b/translations/en-GB.json index 4e4ca4d..65636b0 100644 --- a/translations/en-GB.json +++ b/translations/en-GB.json @@ -1,63 +1,67 @@ { - "forms" : { - "fields" : { - "role_id" : "Node role", - "role_id_info" : "Functional role of the node in the network", - "type_id" : "Node type", - "type_id_info" : "Hardware make and model", - "site_id" : "Site", - "site_id_info" : "Location of the node", - "node_status" : "Node status", - "node_status_info" : "Operational status of the node", - "node_name" : "Node name", - "node_name_info" : "Unique name of node in IMS", - "node_description" : "Node description", - "node_description_info" : "Description of the node", - "node_subscription_id" : "Node", - "port_ims_id" : "Port", - "port_ims_id_info" : "Free port on node", - "port_description" : "Port description", - "port_description_info" : "Description of the port", - "port_mode" : "Port mode", - "port_mode_info" : "Mode of the port (tagged/untagged/link member)", - "auto_negotiation" : "Auto-Negotiation", - "auto_negotiation_info" : "Enable Ethernet Auto-Negotiation?", - "lldp" : "LLDP", - "lldp_info" : "Enable Link Layer Discovery Protocol?", - "number_of_ports" : "Number of ports", - "speed" : "Speed", - "speed_info" : "Speed in Mbit/s", - "speed_policer" : "Speed policer", - "speed_policer_info" : "Enforce the speed?", - "ports" : "Ports", - "vlan" : "VLAN", - "node_subscription_id_a" : "A side node", - "node_subscription_id_b" : "B side node", - "port_ims_id_a" : "A side port", - "port_ims_id_b" : "b side port", - "under_maintenance" : "Maintenance", - "under_maintenance_info" : "Enable maintenance mode?", - "annihilate": "Are you sure you want to wipe the complete administration from Netbox?", - "annihilate_info": "To continue, you have to check this box" - } - }, - "workflow" : { - "create_core_link" : "Create core_link", - "create_l2vpn" : "Create l2vpn", - "create_node" : "Create node", - "create_port" : "Create port", - "modify_core_link" : "Modify core_link", - "modify_l2vpn" : "Modify l2vpn", - "modify_node" : "Modify node", - "modify_port" : "Modify port", - "terminate_core_link" : "Terminate core_link", - "terminate_l2vpn" : "Terminate l2vpn", - "terminate_node" : "Terminate node", - "terminate_port" : "Terminate port", - "validate_core_link" : "Validate core_link", - "validate_l2vpn" : "Validate l2vpn", - "validate_node" : "Validate node", - "validate_port" : "Validate port", - "modify_sync_ports" : "Sync ports with IMS" - } + "forms": { + "fields": { + "annihilate": "Are you sure you want to wipe the complete administration from Netbox?", + "annihilate_info": "To continue, you have to check this box", + "auto_negotiation": "Auto-Negotiation", + "auto_negotiation_info": "Enable Ethernet Auto-Negotiation?", + "lldp": "LLDP", + "lldp_info": "Enable Link Layer Discovery Protocol?", + "node_description": "Node description", + "node_description_info": "Description of the node", + "node_name": "Node name", + "node_name_info": "Unique name of node in IMS", + "node_status": "Node status", + "node_status_info": "Operational status of the node", + "node_subscription_id": "Node", + "node_subscription_id_a": "A side node", + "node_subscription_id_b": "B side node", + "number_of_ports": "Number of ports", + "port_description": "Port description", + "port_description_info": "Description of the port", + "port_ims_id": "Port", + "port_ims_id_a": "A side port", + "port_ims_id_b": "b side port", + "port_ims_id_info": "Free port on node", + "port_mode": "Port mode", + "port_mode_info": "Mode of the port (tagged/untagged/link member)", + "ports": "Ports", + "role_id": "Node role", + "role_id_info": "Functional role of the node in the network", + "site_id": "Site", + "site_id_info": "Location of the node", + "speed": "Speed", + "speed_info": "Speed in Mbit/s", + "speed_policer": "Speed policer", + "speed_policer_info": "Enforce the speed?", + "type_id": "Node type", + "type_id_info": "Hardware make and model", + "under_maintenance": "Maintenance", + "under_maintenance_info": "Enable maintenance mode?", + "vlan": "VLAN" + } + }, + "workflow": { + "create_core_link": "Create core_link", + "create_l2vpn": "Create l2vpn", + "create_node": "Create node", + "create_nsistp": "Create nsistp", + "create_port": "Create port", + "modify_core_link": "Modify core_link", + "modify_l2vpn": "Modify l2vpn", + "modify_node": "Modify node", + "modify_nsistp": "Modify nsistp", + "modify_port": "Modify port", + "modify_sync_ports": "Sync ports with IMS", + "terminate_core_link": "Terminate core_link", + "terminate_l2vpn": "Terminate l2vpn", + "terminate_node": "Terminate node", + "terminate_nsistp": "Terminate nsistp", + "terminate_port": "Terminate port", + "validate_core_link": "Validate core_link", + "validate_l2vpn": "Validate l2vpn", + "validate_node": "Validate node", + "validate_nsistp": "Validate nsistp", + "validate_port": "Validate port" + } } diff --git a/workflows/__init__.py b/workflows/__init__.py index 4f865bc..c80d4d2 100644 --- a/workflows/__init__.py +++ b/workflows/__init__.py @@ -40,5 +40,11 @@ LazyWorkflowInstance("workflows.l2vpn.modify_l2vpn", "reconcile_l2vpn") +LazyWorkflowInstance("workflows.nsistp.create_nsistp", "create_nsistp") +LazyWorkflowInstance("workflows.nsistp.modify_nsistp", "modify_nsistp") +LazyWorkflowInstance("workflows.nsistp.terminate_nsistp", "terminate_nsistp") +LazyWorkflowInstance("workflows.nsistp.validate_nsistp", "validate_nsistp") + + LazyWorkflowInstance("workflows.tasks.bootstrap_netbox", "task_bootstrap_netbox") LazyWorkflowInstance("workflows.tasks.wipe_netbox", "task_wipe_netbox") diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py new file mode 100644 index 0000000..9834d05 --- /dev/null +++ b/workflows/nsistp/create_nsistp.py @@ -0,0 +1,115 @@ +# workflows/nsistp/create_nsistp.py +import structlog +from orchestrator.domain import SubscriptionModel +from orchestrator.forms import FormPage +from orchestrator.forms.validators import CustomerId, Divider, Label +from orchestrator.targets import Target +from orchestrator.types import SubscriptionLifecycle +from orchestrator.workflow import StepList, begin, step +from orchestrator.workflows.steps import store_process_subscription +from orchestrator.workflows.utils import create_workflow +from pydantic import ConfigDict +from pydantic_forms.types import FormGenerator, State, UUIDstr + +from products.product_types.nsistp import NsistpInactive, NsistpProvisioning +from workflows.shared import create_summary_form + + +def subscription_description(subscription: SubscriptionModel) -> str: + """Generate subscription description. + + The suggested pattern is to implement a subscription service that generates a subscription specific + description, in case that is not present the description will just be set to the product name. + """ + return f"{subscription.product.name} subscription" + + +logger = structlog.get_logger(__name__) + + +def initial_input_form_generator(product_name: str) -> FormGenerator: + # TODO add additional fields to form if needed + + class CreateNsistpForm(FormPage): + model_config = ConfigDict(title=product_name) + + customer_id: CustomerId + + nsistp_settings: Label + divider_1: Divider + + topology: str + stp_id: str + stp_description: str | None = None + is_alias_in: str | None = None + is_alias_out: str | None = None + expose_in_topology: bool | None = None + bandwidth: int | None = None + + user_input = yield CreateNsistpForm + user_input_dict = user_input.dict() + + summary_fields = [ + "topology", + "stp_id", + "stp_description", + "is_alias_in", + "is_alias_out", + "expose_in_topology", + "bandwidth", + ] + yield from create_summary_form(user_input_dict, product_name, summary_fields) + + return user_input_dict + + +@step("Construct Subscription model") +def construct_nsistp_model( + product: UUIDstr, + customer_id: UUIDstr, + topology: str, + stp_id: str, + stp_description: str | None, + is_alias_in: str | None, + is_alias_out: str | None, + expose_in_topology: bool | None, + bandwidth: int | None, +) -> State: + nsistp = NsistpInactive.from_product_id( + product_id=product, + customer_id=customer_id, + status=SubscriptionLifecycle.INITIAL, + ) + nsistp.nsistp.topology = topology + nsistp.nsistp.stp_id = stp_id + nsistp.nsistp.stp_description = stp_description + nsistp.nsistp.is_alias_in = is_alias_in + nsistp.nsistp.is_alias_out = is_alias_out + nsistp.nsistp.expose_in_topology = expose_in_topology + nsistp.nsistp.bandwidth = bandwidth + + nsistp = NsistpProvisioning.from_other_lifecycle( + nsistp, SubscriptionLifecycle.PROVISIONING + ) + nsistp.description = subscription_description(nsistp) + + return { + "subscription": nsistp, + "subscription_id": nsistp.subscription_id, # necessary to be able to use older generic step functions + "subscription_description": nsistp.description, + } + + +additional_steps = begin + + +@create_workflow( + "Create nsistp", + initial_input_form=initial_input_form_generator, + additional_steps=additional_steps, +) +def create_nsistp() -> StepList: + return ( + begin >> construct_nsistp_model >> store_process_subscription(Target.CREATE) + # TODO add provision step(s) + ) diff --git a/workflows/nsistp/modify_nsistp.py b/workflows/nsistp/modify_nsistp.py new file mode 100644 index 0000000..fd4af42 --- /dev/null +++ b/workflows/nsistp/modify_nsistp.py @@ -0,0 +1,106 @@ +# workflows/nsistp/modify_nsistp.py +import structlog +from orchestrator.domain import SubscriptionModel +from orchestrator.forms import FormPage +from orchestrator.forms.validators import CustomerId, Divider +from orchestrator.types import SubscriptionLifecycle +from orchestrator.workflow import StepList, begin, step +from orchestrator.workflows.steps import set_status +from orchestrator.workflows.utils import modify_workflow +from pydantic_forms.types import FormGenerator, State, UUIDstr +from pydantic_forms.validators import read_only_field + +from products.product_types.nsistp import Nsistp, NsistpProvisioning +from workflows.shared import modify_summary_form + + +def subscription_description(subscription: SubscriptionModel) -> str: + """The suggested pattern is to implement a subscription service that generates a subscription specific + description, in case that is not present the description will just be set to the product name. + """ + return f"{subscription.product.name} subscription" + + +logger = structlog.get_logger(__name__) + + +def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator: + subscription = Nsistp.from_subscription(subscription_id) + nsistp = subscription.nsistp + + # TODO fill in additional fields if needed + + class ModifyNsistpForm(FormPage): + customer_id: CustomerId = subscription.customer_id # type: ignore + + divider_1: Divider + + stp_id: read_only_field(nsistp.stp_id) + topology: str = nsistp.topology + stp_description: str | None = nsistp.stp_description + is_alias_in: str | None = nsistp.is_alias_in + is_alias_out: str | None = nsistp.is_alias_out + expose_in_topology: bool | None = nsistp.expose_in_topology + bandwidth: int | None = nsistp.bandwidth + + user_input = yield ModifyNsistpForm + user_input_dict = user_input.dict() + + summary_fields = [ + "topology", + "stp_id", + "stp_description", + "is_alias_in", + "is_alias_out", + "expose_in_topology", + "bandwidth", + ] + yield from modify_summary_form(user_input_dict, subscription.nsistp, summary_fields) + + return user_input_dict | {"subscription": subscription} + + +@step("Update subscription") +def update_subscription( + subscription: NsistpProvisioning, + topology: str, + stp_description: str | None, + is_alias_in: str | None, + is_alias_out: str | None, + expose_in_topology: bool | None, + bandwidth: int | None, +) -> State: + # TODO: get all modified fields + subscription.nsistp.topology = topology + subscription.nsistp.stp_description = stp_description + subscription.nsistp.is_alias_in = is_alias_in + subscription.nsistp.is_alias_out = is_alias_out + subscription.nsistp.expose_in_topology = expose_in_topology + subscription.nsistp.bandwidth = bandwidth + + return {"subscription": subscription} + + +@step("Update subscription description") +def update_subscription_description(subscription: Nsistp) -> State: + subscription.description = subscription_description(subscription) + return {"subscription": subscription} + + +additional_steps = begin + + +@modify_workflow( + "Modify nsistp", + initial_input_form=initial_input_form_generator, + additional_steps=additional_steps, +) +def modify_nsistp() -> StepList: + return ( + begin + >> set_status(SubscriptionLifecycle.PROVISIONING) + >> update_subscription + >> update_subscription_description + # TODO add additional steps if needed + >> set_status(SubscriptionLifecycle.ACTIVE) + ) diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py new file mode 100644 index 0000000..e69de29 diff --git a/workflows/nsistp/terminate_nsistp.py b/workflows/nsistp/terminate_nsistp.py new file mode 100644 index 0000000..a7a4900 --- /dev/null +++ b/workflows/nsistp/terminate_nsistp.py @@ -0,0 +1,44 @@ +# workflows/nsistp/terminate_nsistp.py +import structlog +from orchestrator.forms import FormPage +from orchestrator.forms.validators import DisplaySubscription +from orchestrator.workflow import StepList, begin, step +from orchestrator.workflows.utils import terminate_workflow +from pydantic_forms.types import InputForm, State, UUIDstr + +from products.product_types.nsistp import Nsistp + +logger = structlog.get_logger(__name__) + + +def terminate_initial_input_form_generator( + subscription_id: UUIDstr, customer_id: UUIDstr +) -> InputForm: + temp_subscription_id = subscription_id + + class TerminateNsistpForm(FormPage): + subscription_id: DisplaySubscription = temp_subscription_id # type: ignore + + return TerminateNsistpForm + + +@step("Delete subscription from OSS/BSS") +def delete_subscription_from_oss_bss(subscription: Nsistp) -> State: + # TODO: add actual call to OSS/BSS to delete subscription + + return {} + + +additional_steps = begin + + +@terminate_workflow( + "Terminate nsistp", + initial_input_form=terminate_initial_input_form_generator, + additional_steps=additional_steps, +) +def terminate_nsistp() -> StepList: + return ( + begin >> delete_subscription_from_oss_bss + # TODO: fill in additional steps if needed + ) diff --git a/workflows/nsistp/validate_nsistp.py b/workflows/nsistp/validate_nsistp.py new file mode 100644 index 0000000..d863a49 --- /dev/null +++ b/workflows/nsistp/validate_nsistp.py @@ -0,0 +1,21 @@ +# workflows/nsistp/validate_nsistp.py +import structlog +from orchestrator.workflow import StepList, begin, step +from orchestrator.workflows.utils import validate_workflow +from pydantic_forms.types import State + +from products.product_types.nsistp import Nsistp + +logger = structlog.get_logger(__name__) + + +@step("Load initial state") +def load_initial_state_nsistp(subscription: Nsistp) -> State: + return { + "subscription": subscription, + } + + +@validate_workflow("Validate nsistp") +def validate_nsistp() -> StepList: + return begin >> load_initial_state_nsistp From 0d58be8f7ccaaa7c2ae06a80bafae28c0bc90792 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Thu, 2 Oct 2025 09:41:02 +0200 Subject: [PATCH 02/23] Initial working version of NSI --- README.md | 266 +++++++------- docker-compose.yml | 1 + forms/__init__.py | 1 + forms/types.py | 66 ++++ forms/validator/__init__.py | 0 forms/validator/service_port.py | 337 ++++++++++++++++++ forms/validator/service_port_tags.py | 41 +++ forms/validator/shared.py | 93 +++++ forms/validator/subscription_bandwidth.py | 60 ++++ forms/validator/subscription_customer.py | 52 +++ .../subscription_exclude_subscriptions.py | 49 +++ forms/validator/subscription_id.py | 138 +++++++ forms/validator/subscription_in_sync.py | 42 +++ forms/validator/subscription_is_port.py | 44 +++ forms/validator/subscription_port_mode.py | 66 ++++ forms/validator/subscription_product_id.py | 56 +++ forms/validator/subscription_status.py | 55 +++ forms/validator/subscription_tag.py | 53 +++ forms/validator/vlan_ranges.py | 35 ++ products/product_blocks/nsistp.py | 13 +- utils/exceptions.py | 137 +++++++ uv.lock | 16 +- workflows/l2vpn/create_l2vpn.py | 14 +- workflows/l2vpn/shared/forms.py | 13 +- workflows/nsistp/create_nsistp.py | 50 ++- workflows/nsistp/shared/__init__.py | 0 workflows/nsistp/shared/forms.py | 242 +++++++++++++ workflows/nsistp/shared/helpers.py | 108 ++++++ workflows/nsistp/shared/nsistp.py | 104 ++++++ workflows/port/create_port.py | 23 +- workflows/shared.py | 52 ++- 31 files changed, 2051 insertions(+), 176 deletions(-) create mode 100644 forms/__init__.py create mode 100644 forms/types.py create mode 100644 forms/validator/__init__.py create mode 100644 forms/validator/service_port.py create mode 100644 forms/validator/service_port_tags.py create mode 100644 forms/validator/shared.py create mode 100644 forms/validator/subscription_bandwidth.py create mode 100644 forms/validator/subscription_customer.py create mode 100644 forms/validator/subscription_exclude_subscriptions.py create mode 100644 forms/validator/subscription_id.py create mode 100644 forms/validator/subscription_in_sync.py create mode 100644 forms/validator/subscription_is_port.py create mode 100644 forms/validator/subscription_port_mode.py create mode 100644 forms/validator/subscription_product_id.py create mode 100644 forms/validator/subscription_status.py create mode 100644 forms/validator/subscription_tag.py create mode 100644 forms/validator/vlan_ranges.py create mode 100644 utils/exceptions.py create mode 100644 workflows/nsistp/shared/__init__.py create mode 100644 workflows/nsistp/shared/helpers.py create mode 100644 workflows/nsistp/shared/nsistp.py diff --git a/README.md b/README.md index f5f0743..409e061 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ Example workflow orchestrator implementation based on the - [Example orchestrator](#example-orchestrator) - [Folder layout](#folder-layout) - [migrations/versions/schema](#migrationsversionsschema) - - [products/product\_types](#productsproduct_types) - - [products/product\_blocks](#productsproduct_blocks) + - [products/product_types](#productsproduct_types) + - [products/product_blocks](#productsproduct_blocks) - [products/services](#productsservices) - [services](#services) - [templates](#templates) @@ -95,14 +95,12 @@ To access the new v2 `orchestrator-ui`, point your browser to: http://localhost:3000/ ``` - To access `netbox` (admin/admin), point your browser to: ``` http://localhost:8000/ ``` - To access `federation`, point your browser to: ``` @@ -114,43 +112,43 @@ http://localhost:4000 Use the following steps to see the example orchestrator in action: 1. Bootstrap NetBox - 1. from the `Tasks` page click `New Task` - 2. select `NetBox Bootstrap` and click `Start task` - 3. select `Expand all` on the following page to see the step details + 1. from the `Tasks` page click `New Task` + 2. select `NetBox Bootstrap` and click `Start task` + 3. select `Expand all` on the following page to see the step details 2. Create a network node (need at least two to create a core link) - 1. in the left-above corner, click on `New subscription` - 2. select either the `Node Cisco` or `Node Nokia` - 3. fill in the needed fields, click `Start workflow` and view the summary form - 4. click `Start workflow` again to start the workflow, or click `Previous` to modify fields + 1. in the left-above corner, click on `New subscription` + 2. select either the `Node Cisco` or `Node Nokia` + 3. fill in the needed fields, click `Start workflow` and view the summary form + 4. click `Start workflow` again to start the workflow, or click `Previous` to modify fields 3. Add interfaces to a node (needed by the other products) - 1. on the `Subscriptions` page, click on the subscription description of the node to show the details - 2. select `Update node interfaces` from the `Actions` pulldown + 1. on the `Subscriptions` page, click on the subscription description of the node to show the details + 2. select `Update node interfaces` from the `Actions` pulldown 4. Create a core link - 1. in the left-above corner, click on `New subscription` - 2. select either the `core link 10G` or `core link 100G` - 3. fill in the forms and finally click on `Start workflow` to start the workflow + 1. in the left-above corner, click on `New subscription` + 2. select either the `core link 10G` or `core link 100G` + 3. fill in the forms and finally click on `Start workflow` to start the workflow 5. Create a customer port (need at least two **tagged** ports to create a l2vpn) - 1. use `New subscription` for either a `port 10G` or a `port 100G` - 3. fill in the forms and click on `Start workflow` to start the workflow + 1. use `New subscription` for either a `port 10G` or a `port 100G` + 2. fill in the forms and click on `Start workflow` to start the workflow 6. Create a l2vpn - 1. use `New subscription` for a `l2vpn`, fill in the forms, and `Start workflow` + 1. use `New subscription` for a `l2vpn`, fill in the forms, and `Start workflow` While running the different workflows, have a look at the following netbox pages to see the orchestrator interact with netbox: - Devices - - Devices - - Interfaces + - Devices + - Interfaces - Connections - - Cables - - Interface Connections + - Cables + - Interface Connections - IPAM - - IP Addresses - - Prefixes - - VLANs + - IP Addresses + - Prefixes + - VLANs - Overlay - - L2VPNs - - Terminations + - L2VPNs + - Terminations ## Summary @@ -212,9 +210,9 @@ based on a simple fictional NREN that has the following characteristics: - The network nodes are connected to each other through core links - On top of this substrate a set of services like Internet Access, L3VPN and L2VPN are offered - The Operations Support Systems (OSS) used are: - - An IP Administration Management (IPAM) tool - - A network Inventory Management System (IMS) - - A Network Resource Manager (NRM) to provision the network + - An IP Administration Management (IPAM) tool + - A network Inventory Management System (IMS) + - A Network Resource Manager (NRM) to provision the network - There is no Business Support System (BSS) yet This NREN decided on a phased introduction of automation in their @@ -222,15 +220,15 @@ organisation, only automating some of the procedures and flows of information while leaving others unautomated for the moment: - Automated administration and provisioning of: - - Network nodes including loopback IP addresses - - Core links in between network nodes including point-to-point IP addresses - - Customer ports - - Customer L2VPN’s + - Network nodes including loopback IP addresses + - Core links in between network nodes including point-to-point IP addresses + - Customer ports + - Customer L2VPN’s - Not automated administration and provisioning of: - - Role, make and model of the network nodes - - Sites where network nodes are installed - - Customer services like Internet Access, L3VPN, … - - Internet peering + - Role, make and model of the network nodes + - Sites where network nodes are installed + - Customer services like Internet Access, L3VPN, … + - Internet peering NetBox[^3] is used as IMS and IPAM, and serves as the source of truth for the complete IP address administration and physical and logical @@ -417,6 +415,7 @@ When this example orchestrator is deployed, it can create a growing graph of product blocks as is shown below. ### Product Hiearchy Diagram +
Product block graph
### How to use @@ -496,6 +495,7 @@ is unique per customer. In other words a Subscription contains all the informati resource owned by a user/customer that conforms to a certain definition, namely a Product. ### Product description in Python + Products are described in Python classes called Domain Models. These classes are designed to help the developer manage complex subscription models and interact with the objects in a developer-friendly way. Domain models use Pydantic[^6] with some @@ -507,6 +507,7 @@ type checkers, already helps to make the code more robust, furthermore the use o variables at runtime which greatly improves reliability. #### Example of "Runtime typecasting/safety" + In the example below we attempt to access a resource that has been stored in an instance of a product (subscription instance). It shows how it can be done directly through the ORM and it shows the added value of Domain Models on top of the ORM. @@ -526,6 +527,7 @@ Models on top of the ORM. ``` **Serialisation using domain models** + ```python >>> class ProductBlock(ProductBlockModel): ... instance_from_db: bool @@ -548,10 +550,12 @@ False ... print("False") "False" ``` + As you can see in the example above, interacting with the data stored in the database rows, helps with some of the heavy lifting, and makes sure the database remains generic and it's schema remains stable. #### Product Structure + A Product definition has two parts in its structure. The Higher order product type that contains information describing the product in a more general sense, and multiple layers of product blocks that logically describe the set of resources that make up the product definition. The product type describes the fixed inputs and the top-level product blocks. @@ -563,13 +567,14 @@ product blocks as well. If a fixed input needs a custom type, then it is defined here together with fixed input definition. #### Terminology - * **Product:** A definition of what can be instantiated through a Subscription. - * **Product Type:** The higher order definition of a Product. Many different Products can exist within a Product Type. - * **Fixed Input:** Product attributes that discriminate the different Products that adhere to the same Product Type definition. - * **Product Block:** A (logical) construct that contain references to other Product Blocks or Resource Types. It gives - structure to the product definition and defines what resources are related to other resources - * **Resource Types:** Customer facing attributes that are the result of choices made by the user whilst filling an - input form. This can be a value the user chose, or an identifier towards a different system. + +- **Product:** A definition of what can be instantiated through a Subscription. +- **Product Type:** The higher order definition of a Product. Many different Products can exist within a Product Type. +- **Fixed Input:** Product attributes that discriminate the different Products that adhere to the same Product Type definition. +- **Product Block:** A (logical) construct that contain references to other Product Blocks or Resource Types. It gives + structure to the product definition and defines what resources are related to other resources +- **Resource Types:** Customer facing attributes that are the result of choices made by the user whilst filling an + input form. This can be a value the user chose, or an identifier towards a different system. ### Product types @@ -584,6 +589,7 @@ before it ends up terminated. The terminated state does not have its own type definition, but will default to initial unless otherwise defined. #### Domain Model a.k.a Product Type Definition + ```python class PortInactive(SubscriptionModel, is_base=True): speed: PortSpeed @@ -608,6 +614,7 @@ product block, but it is totally fine to use product blocks from different lifecycle states if that suits your use case. #### Fixed Input + Because a port is only available in a limited number of speeds, a separate type is declared with the allowed values, see below. @@ -628,6 +635,7 @@ choices, and in the database migration to register the speed variant of this product. #### Wiring it up in the Orchestrator +
This section contains advanced information about how to configure the Orchestrator. It is also possible to use a more user friendly tool available State: @@ -839,10 +853,12 @@ def my_ugly_step(state: State) -> State: state["subscription"] = subscription return state ``` + In the above example you see we do a simple calculation based on `variable_1`. When computing with even more variables, you van imagine how unreadable the function will be. Now consider the next example. **Good use of the step decorator** + ```python @step("Good use of the input params functionality") def my_beautiful_step(variable_1: int, variable_2: str, subscription: SubscriptionModel) -> State: @@ -859,11 +875,11 @@ def my_beautiful_step(variable_1: int, variable_2: str, subscription: Subscripti As you can see the Orchestrator the orchestrator helps you a lot to condense the logic in your function. The `@step` decorator does the following: -* Loads the previous steps state from the database. -* Inspects the step functions signature -* Finds the arguments in the state and injects them as function arguments to the step function -* It casts them to the correct type by using the type hints of the step function. -* Finally it updates the state of the workflow and persists all model changes to the database upon reaching the +- Loads the previous steps state from the database. +- Inspects the step functions signature +- Finds the arguments in the state and injects them as function arguments to the step function +- It casts them to the correct type by using the type hints of the step function. +- Finally it updates the state of the workflow and persists all model changes to the database upon reaching the `return` of the step function. ### Forms @@ -904,19 +920,21 @@ subscription with minimal or no impact to the customer.
#### Form _Magic_ + As mentioned before, forms are dynamically created from the backend. This means, **little to no** frontend coding is needed to make complex wizard like input forms available to the user. When selecting an action in the UI. The first thing the frontend does is make an api call to load a form from the backend. The resulting `JSONschema` is parsed and the correct widgets are loaded in the frontend. Upon submit this is posted to the backend that does all validation and signals to the user if there are any errors. The following forms are supported: -* Multiselect -* Drop-down -* Text field (restricted) -* Number (float and dec) -* Radio +- Multiselect +- Drop-down +- Text field (restricted) +- Number (float and dec) +- Radio ## Workflow examples + What follows are a few examples of how workflows implement the best common practices implemented by SURF. It explains in detail what a typical workflow could look like for provision in network element. These examples can be examined in greater detail by exploring the `.workflows.node` directory. @@ -945,20 +963,21 @@ def create_node() -> StepList: 1. Collect input from user (`initial_input_form`) 2. Instantiate subscription (`construct_node_model`): - 1. Create inactive subscription model - 2. assign user input to subscription - 3. transition to subscription to provisioning + 1. Create inactive subscription model + 2. assign user input to subscription + 3. transition to subscription to provisioning 3. Register create process for this subscription (`store_process_subscription`) 4. Interact with OSS and/or BSS, in this example - 1. Administer subscription in IMS (`create_node_in ims`) - 2. Reserve IP addresses in IPAM (`reserve_loopback_addresses`) - 3. Provision subscription in the network (`provision_node_in_nrm`) + 1. Administer subscription in IMS (`create_node_in ims`) + 2. Reserve IP addresses in IPAM (`reserve_loopback_addresses`) + 3. Provision subscription in the network (`provision_node_in_nrm`) 5. Transition subscription to active and ‘in sync’ (`@create_workflow`) As long as every step remains as idempotent as possible, the work can be divided over fewer or more steps as desired. #### Input Form + The input form is created by subclassing the `FormPage` and add the input fields together with the type and indication if they are optional or not. Additional form settings can be changed via the Config class, @@ -1014,6 +1033,7 @@ PortsChoiceList: TypeAlias = cast(type[Choice], ports_selector(2)) ``` #### Extra Validation between dependant fields + Validations between multiple fields is also possible by making use of the Pydantic `@model_validator` decorator that gives access to all fields. To check if the A and B side of a point-to-point service are not @@ -1062,13 +1082,13 @@ def modify_node() -> StepList: 1. Collect input from user (`initial_input_form`) 2. Necessary subscription administration (`@modify_workflow`): - 1. Register modify process for this subscription - 2. Set subscription ‘out of sync’ to prevent the start of other processes + 1. Register modify process for this subscription + 2. Set subscription ‘out of sync’ to prevent the start of other processes 3. Transition subscription to Provisioning (`set_status`) 4. Update subscription with the user input 5. Interact with OSS and/or BSS, in this example - 1. Update subscription in IMS (`update_node_in ims`) - 2. Update subscription in NRM (`update_node_in nrm`) + 1. Update subscription in IMS (`update_node_in ims`) + 2. Update subscription in NRM (`update_node_in nrm`) 6. Transition subscription to active (`set_status`) 7. Set subscription ‘in sync’ (`@modify_workflow`) @@ -1116,15 +1136,15 @@ def terminate_node() -> StepList: 1. Show subscription details and ask user to confirm termination (`initial_input_form`) 2. Necessary subscription administration (`@terminate_workflow`): - 1. Register terminate process for this subscription - 2. Set subscription ‘out of sync’ to prevent the start of other processes + 1. Register terminate process for this subscription + 2. Set subscription ‘out of sync’ to prevent the start of other processes 3. Get subscription and add information for following steps to the State (`load_initial_state`) 4. Interact with OSS and/or BSS, in this example - 1. Delete node in IMS (`delete_node_in ims`) - 2. Deprovision node in NRM (`deprovision_node_in_nrm`) + 1. Delete node in IMS (`delete_node_in ims`) + 2. Deprovision node in NRM (`deprovision_node_in_nrm`) 5. Necessary subscription administration (`@terminate_workflow`) - 1. Transition subscription to terminated - 2. Set subscription ‘in sync’ + 1. Transition subscription to terminated + 2. Set subscription ‘in sync’ The initial input form for the terminate workflow is very simple, it only has to show the details of the subscription: @@ -1158,13 +1178,13 @@ def validate_l2vpn() -> StepList: ``` 1. Necessary subscription administration (`@validate_workflow`): - 1. Register validate process for this subscription - 2. Set subscription ‘out of sync’, even when subscription is already out of sync + 1. Register validate process for this subscription + 2. Set subscription ‘out of sync’, even when subscription is already out of sync 2. One or more steps to validate the subscription against all OSS and BSS: - 1. Validate subscription against IMS: - 1. `validate_l2vpn_in_ims` - 2. `validate_l2vpn_terminations_in_ims` - 3. `validate_vlans_on_ports_in_ims` + 1. Validate subscription against IMS: + 1. `validate_l2vpn_in_ims` + 2. `validate_l2vpn_terminations_in_ims` + 3. `validate_vlans_on_ports_in_ims` 3. Set subscription ‘in sync’ again (`@validate_workflow`) When one of the validation steps fail, the subscription will stay ‘out @@ -1207,12 +1227,12 @@ def reconcile_l2vpn() -> StepList: ``` 1. Minimal required information of the subscription is collected and consists of the subscriptions -existing configuration. + existing configuration. 2. Necessary subscription administration (`@reconcile_workflow`): - 1. Register reconcile process for this subscription - 2. Set subscription ‘out of sync’ to prevent the start of other processes + 1. Register reconcile process for this subscription + 2. Set subscription ‘out of sync’ to prevent the start of other processes 3. Interact with OSS and/or BSS, in this example - 1. Update subscription in external systems (OSS and/or BSS) (`update_l2vpn_in_external_systems`) + 1. Update subscription in external systems (OSS and/or BSS) (`update_l2vpn_in_external_systems`) 4. Set subscription ‘in sync’ (`@reconcile_workflow`) Because both a `@modify_workflows` and `@reconcile_workflow` need to have the same update steps for @@ -1258,7 +1278,7 @@ parameter will be taken into account to decide which one of the functions need to be execute. A helper function called `single_dispatch_base()` is used to keep track -of all registered functions and the type of their first argument. This +of all registered functions and the type of their first argument. This allows for more informative error messages when the single dispatch function is called with an unsupported parameter. @@ -1465,40 +1485,41 @@ nodes. ### Federation -WFO and NetBox both use the GraphQL framework Strawberry[^9] which supports Apollo Federation[^8]. This allows to expose both GraphQL backends as a single *supergraph*. WFO can be integrated with any other GraphQL backend that supports[^10] federation and of which you can modify the code. In case of NetBox we don't have direct control over the source code, so we patched it for purposes of demonstration. +WFO and NetBox both use the GraphQL framework Strawberry[^9] which supports Apollo Federation[^8]. This allows to expose both GraphQL backends as a single _supergraph_. WFO can be integrated with any other GraphQL backend that supports[^10] federation and of which you can modify the code. In case of NetBox we don't have direct control over the source code, so we patched it for purposes of demonstration. #### Requirements The following is required to facilitate GraphQL federation on top of WFO and other GraphQL backend(s): -* WFO must be configured with `FEDERATION_ENABLED=True` - * [`docker/orchestrator/orchestrator.env`](docker/orchestrator/orchestrator.env) -* The other backend must also enable federation - * NetBox: [`docker/netbox/Dockerfile`](docker/netbox/Dockerfile) -* In both backends set a federation key on the GraphQL types to join - * WFO: [`graphql_federation.py`](graphql_federation.py) - * NetBox: [`docker/netbox/patch_federation.py`](docker/netbox/patch_federation.py) -* Define the supergraph config with both backends - * [`docker/federation/supergraph-config.yaml`](docker/federation/supergraph-config.yaml) -* Compile the supergraph schema with rover[^12] - * `rover-compose` startup service in [`docker-compose.yml`](docker-compose.yml) -* Run Apollo Router to serve the supergraph - * `federation` service in [`docker-compose.yml`](docker-compose.yml) +- WFO must be configured with `FEDERATION_ENABLED=True` + - [`docker/orchestrator/orchestrator.env`](docker/orchestrator/orchestrator.env) +- The other backend must also enable federation + - NetBox: [`docker/netbox/Dockerfile`](docker/netbox/Dockerfile) +- In both backends set a federation key on the GraphQL types to join + - WFO: [`graphql_federation.py`](graphql_federation.py) + - NetBox: [`docker/netbox/patch_federation.py`](docker/netbox/patch_federation.py) +- Define the supergraph config with both backends + - [`docker/federation/supergraph-config.yaml`](docker/federation/supergraph-config.yaml) +- Compile the supergraph schema with rover[^12] + - `rover-compose` startup service in [`docker-compose.yml`](docker-compose.yml) +- Run Apollo Router to serve the supergraph + - `federation` service in [`docker-compose.yml`](docker-compose.yml) For more information on federating new GraphQL types, or the existing WFO GraphQL types, please refer to our reference documentation[^11]. #### Example queries > **Note:** -> The following queries assume a running `docker-compose` environment with: -> - Initial seed of NetBox via running a `Netbox bootstrap` Task -> - Two newly configured nodes -> +> The following queries assume a running `docker-compose` environment with: +> +> - Initial seed of NetBox via running a `Netbox bootstrap` Task +> - Two newly configured nodes +> > See section [Using the example orchestrator](#using-the-example-orchestrator) on how to run Tasks and create nodes in the [Workflow Orchestrator UI](http://localhost:3000/) - + We'll demonstrate how two separate GraphQL queries can now be performed in one federated query. -**NetBox**: NetBox device details can be queried from the NetBox GraphQL endpoint at +**NetBox**: NetBox device details can be queried from the NetBox GraphQL endpoint at http://localhost:8000/graphql/ (be sure to authenticate first with admin/admin in [NetBox](http://localhost:8000/)) ```graphql @@ -1524,9 +1545,7 @@ query GetNetboxDevices { ```graphql query GetSubscriptions { - subscriptions(filterBy: - {field: "type", value: "Node"} - ) { + subscriptions(filterBy: { field: "type", value: "Node" }) { page { ... on NodeSubscription { subscriptionId @@ -1547,9 +1566,7 @@ query GetSubscriptions { ```graphql query GetEnrichedSubscriptions { - subscriptions(filterBy: - {field: "type", value: "Node"} - ) { + subscriptions(filterBy: { field: "type", value: "Node" }) { page { ... on NodeSubscription { subscriptionId @@ -1604,31 +1621,30 @@ Environment variables and orchestrator-core can be overridden for development pu
WFO
WorkFlow Orchestrator
-[^1]: M7.3 Common NREN Network Service Product Models - -https://resources.geant.org/wp-content/uploads/2023/06/M7.3_Common-NREN-Network-Service-Product-Models.pdf +[^1]: + M7.3 Common NREN Network Service Product Models - + https://resources.geant.org/wp-content/uploads/2023/06/M7.3_Common-NREN-Network-Service-Product-Models.pdf -[^2]: Workflow Orchestrator website - -https://workfloworchestrator.org/orchestrator-core/ +[^2]: + Workflow Orchestrator website - + https://workfloworchestrator.org/orchestrator-core/ -[^3]: NetBox is a tool for data center infrastructure management and IP -address management - https://netbox.dev +[^3]: + NetBox is a tool for data center infrastructure management and IP + address management - https://netbox.dev -[^4]: The Python SQL Toolkit and Object Relational Mapper - -https://www.sqlalchemy.org +[^4]: + The Python SQL Toolkit and Object Relational Mapper - + https://www.sqlalchemy.org [^5]: ASGI server Uvicorn - https://www.uvicorn.org - -[^6]: Pydantic is a data validation library for Python - -https://pydantic.dev/ +[^6]: + Pydantic is a data validation library for Python - + https://pydantic.dev/ [^7]: Pynetbox Python API - https://github.com/netbox-community/pynetbox - [^8]: Apollo Federation - https://www.apollographql.com/docs/federation/ - [^9]: Strawberry Federation - https://strawberry.rocks/docs/federation/introduction - [^10]: Apollo Federation support - https://www.apollographql.com/docs/federation/building-supergraphs/supported-subgraphs - [^11]: WFO GraphQL Documentation - https://workfloworchestrator.org/orchestrator-core/reference-docs/graphql/ - [^12]: Apollo Rover - https://www.apollographql.com/docs/rover/ diff --git a/docker-compose.yml b/docker-compose.yml index 51eb286..947dc65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -193,6 +193,7 @@ services: - 5678 #Enable Python debugger volumes: - ./workflows:/home/orchestrator/workflows + - ./forms:/home/orchestrator/forms - ./products:/home/orchestrator/products - ./migrations:/home/orchestrator/migrations - ./docker:/home/orchestrator/etc diff --git a/forms/__init__.py b/forms/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/forms/__init__.py @@ -0,0 +1 @@ + diff --git a/forms/types.py b/forms/types.py new file mode 100644 index 0000000..6d2aa05 --- /dev/null +++ b/forms/types.py @@ -0,0 +1,66 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network +from typing import Literal + +Tags = Literal[ + "SP", + "SPNL", + "MSC", + "MSCNL", + "AGGSP", + "LightPath", + "LP", + "LR", + "NSILP", + "IPS", + "IPBGP", + "AGGSPNL", + "LRNL", + "LIR_PREFIX", + "SUB_PREFIX", + "Node", + "IRBSP", + "FW", + "Corelink", + "L2VPN", + "L3VPN", + "LPNLNSI", + "IPPG", + "IPPP", + "Wireless", + "OS", +] + +# IMSStatus = Literal["RFS", "PL", "IS", "MI", "RFC", "OOS"] +TransitionType = Literal["speed", "upgrade", "downgrade", "replace"] +VisiblePortMode = Literal["all", "normal", "tagged", "untagged", "link_member"] + + +# class MailAddress(TypedDict): +# email: EmailStr +# name: str + + +# class ConfirmationMail(TypedDict): +# message: str +# subject: str +# language: str +# to: list[MailAddress] +# cc: list[MailAddress] +# bcc: list[MailAddress] + + +IPAddress = IPv4Address | IPv6Address +IPNetwork = IPv4Network | IPv6Network diff --git a/forms/validator/__init__.py b/forms/validator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/forms/validator/service_port.py b/forms/validator/service_port.py new file mode 100644 index 0000000..37699e9 --- /dev/null +++ b/forms/validator/service_port.py @@ -0,0 +1,337 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import operator +import random +import string +from collections import defaultdict +from collections.abc import Iterable +from typing import Any +from uuid import UUID + +import structlog +from more_itertools import flatten +from nwastdlib.vlans import VlanRanges +from orchestrator.db import ( + SubscriptionTable, +) +from orchestrator.services import subscriptions +from orchestrator.types import SubscriptionLifecycle +from pydantic import BaseModel, Field, create_model, field_validator +from pydantic_core.core_schema import ValidationInfo +from pydantic_forms.types import State, UUIDstr + +from forms.types import Tags, VisiblePortMode +from forms.validator.shared import ( + PortTag, + _get_port_mode, + get_port_speed_for_port_subscription, +) +from forms.validator.subscription_id import subscription_id +from forms.validator.vlan_ranges import NsiVlanRanges +from products.product_blocks.port import PortMode +from utils.exceptions import PortsValueError, VlanValueError +from workflows.nsistp.shared.nsistp import ( + get_available_vlans_by_port_id, + nsistp_get_by_port_id, +) + +logger = structlog.get_logger(__name__) + +_port_tag_values = [str(value) for value in PortTag] + + +class ServicePortNoVlan(BaseModel): + subscription_id: subscription_id(allowed_tags=_port_tag_values) # type: ignore # noqa: F821 + + def __repr__(self) -> str: + return f"FormServicePortNoVlan({self.subscription_id=})" + + +class ServicePort(BaseModel): + # By default we don't want constraints but having ports in the allowed taglist also signals the frontend + # This way we get te correct behavior even though we don't constrain.. (Well we are sure we want ports here..) + subscription_id: subscription_id(allowed_tags=_port_tag_values) # type: ignore # noqa: F821 + vlan: VlanRanges + + def __repr__(self) -> str: + # Help distinguish this from the ServicePort product type.. + print("subscription_id", self.subscription_id) + + return f"FormServicePort({self.subscription_id=}, {self.vlan=})" + + @field_validator("vlan") + @classmethod + def check_vlan(cls, vlan: VlanRanges, info: ValidationInfo) -> VlanRanges: + # We assume an empty string is untagged and thus 0 + if not vlan: + vlan = VlanRanges(0) + + subscription_id = info.data.get("subscription_id") + if not subscription_id: + return vlan + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + port_mode = _get_port_mode(subscription) + + if port_mode == PortMode.tagged and vlan == VlanRanges(0): + raise VlanValueError( + f"{port_mode} {subscription.product.tag} must have a vlan" + ) + elif port_mode == PortMode.untagged and vlan != VlanRanges(0): # noqa: RET506 + raise VlanValueError( + f"{port_mode} {subscription.product.tag} can not have a vlan" + ) + + return vlan + + +class NsiServicePort(ServicePort): + vlan: NsiVlanRanges + + +def _random_service_port_str() -> str: + return "".join( + random.choice(string.ascii_letters + string.digits) for _ in range(8) + ) # noqa: S311 + + +def service_port( + visible_port_mode: VisiblePortMode | None = None, + customer_id: UUIDstr | None = None, + customer_key: str | None = None, + customer_ports_only: bool = False, + bandwidth: int | None = None, + bandwidth_key: str | None = None, + current: list[State] | None = None, + allowed_tags: list[Tags] | None = None, + disabled_ports: bool | None = None, + excluded_subscriptions: list[UUID] | None = None, + allowed_statuses: list[SubscriptionLifecycle] | None = None, + nsi_vlans_only: bool = False, +) -> type[ServicePort]: + """Extend the normal validator with configurable constraints.""" + + @field_validator("vlan") # type: ignore[misc] + @classmethod + def check_vlan_in_use( + cls: ServicePort, v: VlanRanges, info: ValidationInfo + ) -> VlanRanges: + """Check if vlan value is already in use by service port. + + Args: + cls: class + v: Vlan range of the form input. + info: validation info, contains other fields in info.data + + 1. Get all used vlans in a service port. + 2. Get all nsi reserved vlans and add them to the used_vlans. + 3. Filter out vlans used in current subscription from used_vlans. + 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. + 5. checks if input value uses already used vlans. errors if true. + 6. return input value. + + + """ + if not (subscription_id := info.data.get("subscription_id")): + return v + + # used_vlans = VlanRanges(ims.get_vlans_by_subscription_id(subscription_id)) + # return available_vlans - used_vlans + used_vlans = nsistp_get_by_port_id(subscription_id) + print("used_vlans", used_vlans) + nsistp_reserved_vlans = get_available_vlans_by_port_id(subscription_id) + used_vlans = VlanRanges( + flatten([list(used_vlans), list(nsistp_reserved_vlans)]) + ) + + # Remove currently chosen vlans for this port to prevent tripping on in used by itself + current_selected_vlan_ranges: list[str] = [] + if current: + current_selected_service_port = filter( + lambda c: str(c["subscription_id"]) == str(subscription_id), current + ) + current_selected_vlans = list( + map(operator.itemgetter("vlan"), current_selected_service_port) + ) + for current_selected_vlan in current_selected_vlans: + # We assume an empty string is untagged and thus 0 + if not current_selected_vlan: + current_selected_vlan = "0" + + current_selected_vlan_range = VlanRanges(current_selected_vlan) + used_vlans -= current_selected_vlan_range + current_selected_vlan_ranges = [ + *current_selected_vlan_ranges, + *list(current_selected_vlan_range), + ] + + # TODO (#1842): probably better to have a separate type/validator for this + if nsi_vlans_only: + vlan_list = list(v) + invalid_ranges = [ + vlan + for vlan in vlan_list + if vlan not in list(nsistp_reserved_vlans) + and vlan not in current_selected_vlan_ranges + ] + used_vlans -= nsistp_reserved_vlans + + if invalid_ranges: + raise VlanValueError( + f"Vlan(s) {VlanRanges(invalid_ranges)} not valid nsi vlan range" + ) + + logger.info( + "Validation info for current chosen vlans vs vlan already in use", + current=current, + used_vlans=used_vlans, + subscription_id=subscription_id, + ) + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + print("subscription check", subscription) + + if v & used_vlans: + port_mode = _get_port_mode(subscription) + + # for tagged only; for link_member/untagged say "SP already in use" + if port_mode == PortMode.untagged or port_mode == PortMode.link_member: + raise PortsValueError("Service Port already in use") + raise VlanValueError(f"Vlan(s) {used_vlans} already in use") + + return v + + # Choose needed extra validators + validators: dict[str, Any] = { + "check_vlan_in_use": check_vlan_in_use, + } + + # Choose Base Model + base_model = NsiServicePort if nsi_vlans_only else ServicePort + + print("allowed_tags", allowed_tags) + print("base_model", base_model) + + return create_model( + f"{base_model.__name__}{_random_service_port_str()}Value", + __base__=base_model, + __validators__=validators, + subscription_id=( + subscription_id( + visible_port_mode=visible_port_mode, + customer_id=customer_id if customer_ports_only else None, + customer_key=customer_key if customer_ports_only else None, + bandwidth=bandwidth, + bandwidth_key=bandwidth_key, + allowed_tags=allowed_tags, + excluded_subscriptions=excluded_subscriptions, + allowed_statuses=allowed_statuses, + ), + Field(...), + ), + ) + + +def service_port_no_vlan( + visible_port_mode: VisiblePortMode | None = None, + customer_id: UUIDstr | None = None, + customer_key: str | None = None, + customer_ports_only: bool = False, + bandwidth: int | None = None, + bandwidth_key: str | None = None, + allowed_tags: list[Tags] | None = None, + excluded_subscriptions: list[UUID] | None = None, + allowed_statuses: list[SubscriptionLifecycle] | None = None, +) -> type[ServicePortNoVlan]: + """Extend the normal validator with configurable constraints.""" + + base_model = ServicePortNoVlan + + return create_model( + f"{base_model.__name__}{_random_service_port_str()}Value", + __base__=base_model, + subscription_id=( + subscription_id( + visible_port_mode=visible_port_mode, + customer_id=customer_id if customer_ports_only else None, + customer_key=customer_key if customer_ports_only else None, + bandwidth=bandwidth, + bandwidth_key=bandwidth_key, + allowed_tags=allowed_tags, + excluded_subscriptions=excluded_subscriptions, + allowed_statuses=allowed_statuses, + ), + Field(...), + ), + ) + + +def service_port_values(values: Iterable[dict] | None) -> list[ServicePort] | None: + if values is None: + return None + + return [ + ServicePort.model_construct( + subscription_id=UUID(v["subscription_id"]), vlan=VlanRanges(v["vlan"]) + ) + for v in values + ] + + +def validate_single_vlan(v: list[ServicePort]) -> list[ServicePort]: + if not all(sp.vlan.is_single_vlan for sp in v): + raise VlanValueError("This product only supports a single vlan") + return v + + +def validate_service_ports_bandwidth( + v: ServicePort, info: ValidationInfo, *, bandwidth_key: str +) -> ServicePort: + values = info.data + + if bandwidth := values.get(bandwidth_key): + port_speed = get_port_speed_for_port_subscription(v.subscription_id) + + if int(bandwidth) > port_speed: + raise PortsValueError( + f"The port speed is lower than the desired speed {bandwidth}" + ) + + return v + + +def validate_owner_of_service_port(customer: str | None, v: ServicePort) -> ServicePort: + if customer: + subscription = subscriptions.get_subscription(v.subscription_id) + + if subscription.customer_id != str(customer): + raise PortsValueError(f"Port subscription is not of customer {customer}") + + return v + + +def validate_service_ports_unique_vlans(v: list[ServicePort]) -> list[ServicePort]: + vlans: dict[UUID, list[VlanRanges]] = defaultdict(list) + for sap in v: + for vlan in vlans[sap.subscription_id]: + if not sap.vlan.isdisjoint(vlan): + raise VlanValueError("Vlans are already in use") + + vlans[sap.subscription_id].append(sap.vlan) + + return v diff --git a/forms/validator/service_port_tags.py b/forms/validator/service_port_tags.py new file mode 100644 index 0000000..de4e8e1 --- /dev/null +++ b/forms/validator/service_port_tags.py @@ -0,0 +1,41 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from forms.types import Tags + +PORT_TAG_GENERAL: list[Tags] = ["PORT"] + +# TODO: these tags can probably be removed +PORT_TAGS_AGGSP: list[Tags] = ["AGGSP", "AGGSPNL"] +PORT_TAGS_IRBSP: list[Tags] = ["IRBSP"] +PORT_TAGS_MSC: list[Tags] = ["MSC", "MSCNL"] +PORT_TAGS_SPNL: list[Tags] = ["SPNL"] +PORT_TAGS_SP: list[Tags] = ["SP"] +PORT_TAGS_SP_ALL: list[Tags] = PORT_TAGS_SPNL + PORT_TAGS_SP +PORT_TAGS_ALL: list[Tags] = ( + PORT_TAGS_SP_ALL + + PORT_TAGS_AGGSP + + PORT_TAGS_MSC + + PORT_TAGS_IRBSP + + PORT_TAG_GENERAL +) +SERVICES_TAGS_FOR_IMS_REDEPLOY: list[Tags] = [ + "IPBGP", + "IPS", + "L2VPN", + "L3VPN", + "LP", + "LR", + "NSILP", +] +TAGS_FOR_IMS_REDEPLOY: list[Tags] = SERVICES_TAGS_FOR_IMS_REDEPLOY + PORT_TAGS_ALL diff --git a/forms/validator/shared.py b/forms/validator/shared.py new file mode 100644 index 0000000..713a262 --- /dev/null +++ b/forms/validator/shared.py @@ -0,0 +1,93 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Callable +from enum import StrEnum, auto +from functools import partial +from typing import Any +from uuid import UUID + +import structlog +from annotated_types import BaseMetadata +from orchestrator.db import ( + SubscriptionTable, +) +from orchestrator.services import subscriptions +from pydantic import Field + +from products.product_types.port import Port as ServicePort + +logger = structlog.get_logger(__name__) + +GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] + +PORT_SPEED = "port_speed" +MAX_SPEED_POSSIBLE = 400_000 + + +class PortMode(StrEnum): + tagged = auto() + untagged = auto() + link_member = auto() + + +class PortTag(StrEnum): + SP = "SP" + SPNL = "SPNL" + AGGSP = "AGGSP" + AGGSPNL = "AGGSPNL" + MSC = "MSC" + MSCNL = "MSCNL" + IRBSP = "IRBSP" + + +def _get_port_mode(subscription: SubscriptionTable) -> PortMode: + if subscription.product.tag in [PortTag.AGGSP + PortTag.SP]: + return subscription.port_mode + return PortMode.tagged + + +def get_port_speed_for_port_subscription( + subscription_id: UUID, get_subscription: GetSubscriptionByIdFunc | None = None +) -> int: + print("HELLO from get_port_speed_for_port_subscription") + if get_subscription: + subscription = get_subscription(subscription_id) + print("subscription from get_subscription", subscription) + else: + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + if subscription.tag in [PortTag.MSC + PortTag.AGGSP]: + port_speed = ServicePort.from_subscription( + subscription.subscription_id + ).get_port_speed() + elif subscription.tag in PortTag.IRBSP: + port_speed = MAX_SPEED_POSSIBLE + else: + port_speed = int(subscription.product.fixed_input_value(PORT_SPEED)) + + logger.info( + "Validation determined speed for port", + product_tag=subscription.tag, + port_speed=port_speed, + ) + return port_speed + + +def merge_uniforms(schema: dict[str, Any], *, to_merge: dict[str, Any]) -> None: + schema["uniforms"] = schema.get("uniforms", {}) | to_merge + + +def uniforms_field(to_merge: dict[str, Any]) -> BaseMetadata: + return Field(json_schema_extra=partial(merge_uniforms, to_merge=to_merge)) diff --git a/forms/validator/subscription_bandwidth.py b/forms/validator/subscription_bandwidth.py new file mode 100644 index 0000000..8fadc6d --- /dev/null +++ b/forms/validator/subscription_bandwidth.py @@ -0,0 +1,60 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.validator.shared import ( + GetSubscriptionByIdFunc, + get_port_speed_for_port_subscription, + uniforms_field, +) +from utils.exceptions import PortsValueError + + +def validate_service_port_bandwidth( + v: UUID, *, bandwidth: int, get_subscription: GetSubscriptionByIdFunc +) -> UUID: + port_speed = get_port_speed_for_port_subscription( + v, get_subscription=get_subscription + ) + + if bandwidth > port_speed: + raise PortsValueError( + f"The port speed is lower than the desired speed {bandwidth}" + ) + + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionBandwidthValidator(GroupedMetadata): + bandwidth: int + get_subscription: GetSubscriptionByIdFunc + bandwidth_key: str | None = None + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_service_port_bandwidth, + bandwidth=self.bandwidth, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field( + {"bandwidth": self.bandwidth, "bandwidthKey": self.bandwidth_key} + ) diff --git a/forms/validator/subscription_customer.py b/forms/validator/subscription_customer.py new file mode 100644 index 0000000..63a7c55 --- /dev/null +++ b/forms/validator/subscription_customer.py @@ -0,0 +1,52 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator +from pydantic_forms.types import UUIDstr + +from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field +from utils.exceptions import CustomerValueError + + +def validate_customer( + v: UUID, *, customer_id: UUIDstr, get_subscription: GetSubscriptionByIdFunc +) -> UUID: + subscription = get_subscription(v) + + if subscription.customer_id != str(customer_id): + raise CustomerValueError(f"Subscription is not of customer {customer_id}") + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionCustomerValidator(GroupedMetadata): + customer_id: UUIDstr + get_subscription: GetSubscriptionByIdFunc + customer_key: str | None = None + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_customer, + customer_id=self.customer_id, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field( + {"customerId": self.customer_id, "customerKey": self.customer_key} + ) diff --git a/forms/validator/subscription_exclude_subscriptions.py b/forms/validator/subscription_exclude_subscriptions.py new file mode 100644 index 0000000..8636f3f --- /dev/null +++ b/forms/validator/subscription_exclude_subscriptions.py @@ -0,0 +1,49 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.validator.shared import uniforms_field +from utils.exceptions import ProductValueError + + +def validate_excluded_subscriptions( + v: UUID, *, excluded_subscriptions: list[UUID] +) -> UUID: + if excluded_subscriptions and v in excluded_subscriptions: + raise ProductValueError( + "Subscription is in the excluded list and cannot be chosen" + ) + + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionExcludeSubscriptionsValidator(GroupedMetadata): + excluded_subscriptions: list[UUID] + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_excluded_subscriptions, + excluded_subscriptions=self.excluded_subscriptions, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field( + {"excludedSubscriptionIds": list(map(str, self.excluded_subscriptions))} + ) diff --git a/forms/validator/subscription_id.py b/forms/validator/subscription_id.py new file mode 100644 index 0000000..b0a966d --- /dev/null +++ b/forms/validator/subscription_id.py @@ -0,0 +1,138 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random +import string +from collections.abc import Generator +from types import new_class +from typing import Annotated, Any +from uuid import UUID + +import structlog +from orchestrator.db import ( + SubscriptionTable, +) +from orchestrator.services import subscriptions +from orchestrator.types import SubscriptionLifecycle +from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema +from pydantic_forms.types import UUIDstr + +from forms.types import Tags, VisiblePortMode # move to other location +from forms.validator.shared import GetSubscriptionByIdFunc +from forms.validator.subscription_bandwidth import SubscriptionBandwidthValidator +from forms.validator.subscription_customer import SubscriptionCustomerValidator +from forms.validator.subscription_exclude_subscriptions import ( + SubscriptionExcludeSubscriptionsValidator, +) +from forms.validator.subscription_in_sync import SubscriptionInSyncValidator +from forms.validator.subscription_is_port import SubscriptionIsPortValidator +from forms.validator.subscription_port_mode import SubscriptionPortModeValidator +from forms.validator.subscription_product_id import SubscriptionProductIdValidator +from forms.validator.subscription_status import SubscriptionStatusValidator +from forms.validator.subscription_tag import SubscriptionTagValidator + +logger = structlog.get_logger(__name__) + + +class SubscriptionId: # TODO #1983 change to Annotated Type + @classmethod + def __get_pydantic_json_schema__( + cls, schema: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + json_schema_extra = {"type": "string", "format": "subscriptionId"} + json_schema = handler(schema) + json_schema.update(json_schema_extra) + return json_schema + + @classmethod + def __get_pydantic_core_schema__( + cls, _source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return handler(UUID) + + +def default_get_subscription(v: UUID) -> SubscriptionTable: + print("uuid v", v) + print("subscription", subscriptions.get_subscription(v, model=SubscriptionTable)) + + return subscriptions.get_subscription(v, model=SubscriptionTable) + + +def subscription_id( + product_ids: list[UUID] | None = None, + visible_port_mode: VisiblePortMode | None = None, + customer_id: UUIDstr | None = None, + customer_key: str | None = None, + bandwidth: int | None = None, + bandwidth_key: str | None = None, + allowed_tags: list[Tags] | None = None, + excluded_subscriptions: list[UUID] | None = None, + allowed_statuses: list[SubscriptionLifecycle] | None = None, + allow_out_of_sync: bool = False, + get_subscription: GetSubscriptionByIdFunc | None = None, +) -> type: + # Create type name + org_string = f"O{str(customer_id)[0:8]}" if customer_id else "" + bandwidth_str = f"B{bandwidth}" if bandwidth else "" + random_str = "".join( + random.choice(string.ascii_letters + string.digits) for _ in range(8) + ) # noqa: S311 + class_name = ( + f"SubscriptionId{visible_port_mode}{org_string}{bandwidth_str}{random_str}Value" + ) + + # TODO: caching is disabled here since it was in the wrong place. First #1983 has to be fixed + get_subscription = ( + get_subscription if get_subscription else default_get_subscription + ) + + def get_validators() -> Generator[Any, Any, None]: + yield SubscriptionStatusValidator( + allowed_statuses=allowed_statuses, get_subscription=get_subscription + ) + if product_ids: + yield SubscriptionProductIdValidator( + allowed_product_ids=product_ids, get_subscription=get_subscription + ) + if allowed_tags: + yield SubscriptionTagValidator( + allowed_product_tags=allowed_tags, get_subscription=get_subscription + ) + if visible_port_mode: + yield SubscriptionIsPortValidator(get_subscription=get_subscription) + yield SubscriptionPortModeValidator( + visible_port_mode=visible_port_mode, get_subscription=get_subscription + ) + if excluded_subscriptions: + yield SubscriptionExcludeSubscriptionsValidator( + excluded_subscriptions=excluded_subscriptions + ) + if customer_id: + print("customer_id in get_validators", customer_id) + yield SubscriptionCustomerValidator( + customer_id=customer_id, + customer_key=customer_key, + get_subscription=get_subscription, + ) + if not allow_out_of_sync: + yield SubscriptionInSyncValidator(get_subscription=get_subscription) + if bandwidth is not None: + yield SubscriptionBandwidthValidator( + bandwidth=bandwidth, + bandwidth_key=bandwidth_key, + get_subscription=get_subscription, + ) + + validator = new_class(class_name, (SubscriptionId,)) + return Annotated[validator, *get_validators()] # type: ignore diff --git a/forms/validator/subscription_in_sync.py b/forms/validator/subscription_in_sync.py new file mode 100644 index 0000000..eb7fa19 --- /dev/null +++ b/forms/validator/subscription_in_sync.py @@ -0,0 +1,42 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from orchestrator.db.models import SubscriptionTable +from pydantic import AfterValidator + +from utils.exceptions import InSyncValueError + +GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] + + +def validate_in_sync(v: UUID, *, get_subscription: GetSubscriptionByIdFunc) -> UUID: + subscription = get_subscription(v) + if not subscription.insync: + raise InSyncValueError("Subscription is not in sync") + + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionInSyncValidator(GroupedMetadata): + get_subscription: GetSubscriptionByIdFunc + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial(validate_in_sync, get_subscription=self.get_subscription) + yield cast(BaseMetadata, AfterValidator(validator)) diff --git a/forms/validator/subscription_is_port.py b/forms/validator/subscription_is_port.py new file mode 100644 index 0000000..a28ed58 --- /dev/null +++ b/forms/validator/subscription_is_port.py @@ -0,0 +1,44 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.validator.service_port_tags import PORT_TAGS_ALL +from forms.validator.shared import GetSubscriptionByIdFunc +from utils.exceptions import PortsValueError + + +def validate_subscription_is_port( + v: UUID, *, get_subscription: GetSubscriptionByIdFunc +) -> UUID: + subscription = get_subscription(v) + if subscription.product.tag not in PORT_TAGS_ALL: + raise PortsValueError("Not a service port subscription") + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionIsPortValidator(GroupedMetadata): + get_subscription: GetSubscriptionByIdFunc + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_subscription_is_port, get_subscription=self.get_subscription + ) + yield cast(BaseMetadata, AfterValidator(validator)) diff --git a/forms/validator/subscription_port_mode.py b/forms/validator/subscription_port_mode.py new file mode 100644 index 0000000..9f6f438 --- /dev/null +++ b/forms/validator/subscription_port_mode.py @@ -0,0 +1,66 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.types import VisiblePortMode +from forms.validator.shared import ( + GetSubscriptionByIdFunc, + PortMode, + _get_port_mode, + uniforms_field, +) +from utils.exceptions import PortsModeValueError + + +def validate_port_mode( + v: UUID, + *, + visible_port_mode: VisiblePortMode, + get_subscription: GetSubscriptionByIdFunc, +) -> UUID: + subscription = get_subscription(v) + port_mode = _get_port_mode(subscription) + + if visible_port_mode == "normal" and port_mode == PortMode.link_member: + raise PortsModeValueError("normal", "Port mode should be 'untagged' or 'tagged") + elif visible_port_mode == "link_member" and port_mode != PortMode.link_member: # noqa: RET506 + raise PortsModeValueError( + PortMode.link_member, "Port mode should be 'link_member'" + ) + elif visible_port_mode == "untagged" and port_mode != PortMode.untagged: + raise PortsModeValueError(PortMode.untagged, "Port mode should be 'untagged'") + elif visible_port_mode == "tagged" and port_mode != PortMode.tagged: + raise PortsModeValueError(PortMode.tagged, "Port mode should be 'tagged'") + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionPortModeValidator(GroupedMetadata): + visible_port_mode: VisiblePortMode + get_subscription: GetSubscriptionByIdFunc + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_port_mode, + visible_port_mode=self.visible_port_mode, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field({"visiblePortMode": self.visible_port_mode}) diff --git a/forms/validator/subscription_product_id.py b/forms/validator/subscription_product_id.py new file mode 100644 index 0000000..2287161 --- /dev/null +++ b/forms/validator/subscription_product_id.py @@ -0,0 +1,56 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field +from utils.exceptions import ProductValueError + + +def validate_product_id( + v: UUID, + *, + allowed_product_ids: list[UUID], + get_subscription: GetSubscriptionByIdFunc, +) -> UUID: + subscription = get_subscription(v) + if ( + allowed_product_ids + and subscription.product.product_id not in allowed_product_ids + ): + raise ProductValueError( + f"Subscription is not of products: {allowed_product_ids}" + ) + + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionProductIdValidator(GroupedMetadata): + allowed_product_ids: list[UUID] + get_subscription: GetSubscriptionByIdFunc + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_product_id, + allowed_product_ids=self.allowed_product_ids, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field({"productIds": list(map(str, self.allowed_product_ids))}) diff --git a/forms/validator/subscription_status.py b/forms/validator/subscription_status.py new file mode 100644 index 0000000..8fe17fd --- /dev/null +++ b/forms/validator/subscription_status.py @@ -0,0 +1,55 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from orchestrator.types import SubscriptionLifecycle +from pydantic import AfterValidator + +from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field +from utils.exceptions import AllowedStatusValueError + + +def validate_allowed_statuses( + v: UUID, + *, + allowed_statuses: list[SubscriptionLifecycle] | None = None, + get_subscription: GetSubscriptionByIdFunc, +) -> UUID: + allowed_statuses = allowed_statuses or [SubscriptionLifecycle.ACTIVE] + subscription = get_subscription(v) + if subscription.status not in allowed_statuses: + raise AllowedStatusValueError( + f"Subscription has status {subscription.status}. Allowed statuses: {list(map(str, allowed_statuses))}" + ) + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionStatusValidator(GroupedMetadata): + get_subscription: GetSubscriptionByIdFunc + allowed_statuses: list[SubscriptionLifecycle] | None = None + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_allowed_statuses, + allowed_statuses=self.allowed_statuses, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + allowed_statuses = self.allowed_statuses or [SubscriptionLifecycle.ACTIVE] + yield uniforms_field({"statuses": list(map(str, allowed_statuses))}) diff --git a/forms/validator/subscription_tag.py b/forms/validator/subscription_tag.py new file mode 100644 index 0000000..610a125 --- /dev/null +++ b/forms/validator/subscription_tag.py @@ -0,0 +1,53 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from typing import cast +from uuid import UUID + +from annotated_types import SLOTS, BaseMetadata, GroupedMetadata +from pydantic import AfterValidator + +from forms.types import Tags +from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field +from utils.exceptions import ProductValueError + + +def validate_product_tags( + v: UUID, + *, + allowed_product_tags: list[Tags], + get_subscription: GetSubscriptionByIdFunc, +) -> UUID: + subscription = get_subscription(v) + if subscription.product.tag not in allowed_product_tags: + raise ProductValueError( + f"Subscription is of the wrong product. Allowed product tags: {allowed_product_tags}" + ) + return v + + +@dataclass(frozen=True, **SLOTS) +class SubscriptionTagValidator(GroupedMetadata): + allowed_product_tags: list[Tags] + get_subscription: GetSubscriptionByIdFunc + + def __iter__(self) -> Iterator[BaseMetadata]: + validator = partial( + validate_product_tags, + allowed_product_tags=self.allowed_product_tags, + get_subscription=self.get_subscription, + ) + yield cast(BaseMetadata, AfterValidator(validator)) + yield uniforms_field({"tags": list(map(str, self.allowed_product_tags))}) diff --git a/forms/validator/vlan_ranges.py b/forms/validator/vlan_ranges.py new file mode 100644 index 0000000..b7abda0 --- /dev/null +++ b/forms/validator/vlan_ranges.py @@ -0,0 +1,35 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Annotated + +from nwastdlib.vlans import VlanRanges +from pydantic import AfterValidator, Field, TypeAdapter + +from utils.exceptions import VlanValueError + + +def validate_vlan_range(vlan_ranges: VlanRanges) -> VlanRanges: + for vlan in vlan_ranges: + if vlan == 0 or (2 <= vlan <= 4094): + continue + raise VlanValueError("VLAN range must be between 2 and 4094") + return vlan_ranges + + +_vlan_ranges_schema = TypeAdapter(VlanRanges).json_schema() + +NsiVlanRanges = Annotated[ + VlanRanges, + Field(json_schema_extra=_vlan_ranges_schema | {"uniforms": {"nsiVlansOnly": True}}), + AfterValidator(validate_vlan_range), +] diff --git a/products/product_blocks/nsistp.py b/products/product_blocks/nsistp.py index 3f6c6f3..df35c4e 100644 --- a/products/product_blocks/nsistp.py +++ b/products/product_blocks/nsistp.py @@ -1,18 +1,17 @@ # products/product_blocks/nsistp.py -from typing import Annotated -from annotated_types import Len from orchestrator.domain.base import ProductBlockModel -from orchestrator.types import SI, SubscriptionLifecycle +from orchestrator.types import SubscriptionLifecycle from pydantic import computed_field from products.product_blocks.sap import SAPBlock, SAPBlockInactive, SAPBlockProvisioning -ListOfSap = Annotated[list[SI], Len(min_length=2, max_length=8)] +# NOTE: ListOfSap is not required here??? +# ListOfSap = Annotated[list[SI], Len(min_length=2, max_length=8)] class NsistpBlockInactive(ProductBlockModel, product_block_name="Nsistp"): - sap: ListOfSap[SAPBlockInactive] + sap: SAPBlockInactive topology: str | None = None stp_id: str | None = None stp_description: str | None = None @@ -25,7 +24,7 @@ class NsistpBlockInactive(ProductBlockModel, product_block_name="Nsistp"): class NsistpBlockProvisioning( NsistpBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] ): - sap: ListOfSap[SAPBlockProvisioning] + sap: SAPBlockProvisioning topology: str stp_id: str stp_description: str | None = None @@ -42,7 +41,7 @@ def title(self) -> str: class NsistpBlock(NsistpBlockProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): - sap: ListOfSap[SAPBlock] + sap: SAPBlock topology: str stp_id: str stp_description: str | None = None diff --git a/utils/exceptions.py b/utils/exceptions.py new file mode 100644 index 0000000..6da2ac9 --- /dev/null +++ b/utils/exceptions.py @@ -0,0 +1,137 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides ValueError Exception classes.""" + + +class AllowedStatusValueError(ValueError): + pass + + +class ASNValueError(ValueError): + pass + + +class BGPPolicyValueError(ValueError): + pass + + +class BlackHoleCommunityValueError(ValueError): + pass + + +class ChoiceValueError(ValueError): + pass + + +class DuplicateValueError(ValueError): + pass + + +class EndpointTypeValueError(ValueError): + pass + + +class FieldValueError(ValueError): + pass + + +class FreeSpaceValueError(ValueError): + pass + + +class InSyncValueError(ValueError): + pass + + +class IPAddressValueError(ValueError): + pass + + +class IPPrefixValueError(ValueError): + pass + + +class LocationValueError(ValueError): + pass + + +class NodesValueError(ValueError): + pass + + +class CustomerValueError(ValueError): + pass + + +class PeeringValueError(ValueError): + pass + + +class PeerGroupNameError(ValueError): + pass + + +class PeerNameValueError(ValueError): + pass + + +class PeerPortNameValueError(ValueError): + pass + + +class PeerPortValueError(ValueError): + pass + + +class PortsModeValueError(ValueError): + def __init__(self, mode: str = "", message: str = ""): + super().__init__(message) + self.message = message + self.mode = mode + + +class PortsValueError(ValueError): + pass + + +class ProductValueError(ValueError): + pass + + +class ServicesActiveValueError(ValueError): + pass + + +class SubscriptionTypeValueError(ValueError): + pass + + +class UnsupportedSpeedValueError(ValueError): + pass + + +class UnsupportedTypeValueError(ValueError): + pass + + +class VlanRetaggingValueError(ValueError): + pass + + +class VlanValueError(ValueError): + pass + + +class InUseByAzError(ValueError): + pass diff --git a/uv.lock b/uv.lock index e041399..52ceba5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.13'", @@ -206,14 +206,14 @@ wheels = [ [[package]] name = "deepdiff" -version = "8.0.1" +version = "8.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "orderly-set" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/ba/aced1d6a7d988ca1b6f9b274faed7dafc7356a733e45a457819bddcf2dbc/deepdiff-8.0.1.tar.gz", hash = "sha256:245599a4586ab59bb599ca3517a9c42f3318ff600ded5e80a3432693c8ec3c4b", size = 427721, upload-time = "2024-08-28T20:24:09.286Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/76/36c9aab3d5c19a94091f7c6c6e784efca50d87b124bf026c36e94719f33c/deepdiff-8.6.1.tar.gz", hash = "sha256:ec56d7a769ca80891b5200ec7bd41eec300ced91ebcc7797b41eb2b3f3ff643a", size = 634054, upload-time = "2025-09-03T19:40:41.461Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/46/01673060e83277a863baf0909b387cd809865cba2d5e7213db76516bedd9/deepdiff-8.0.1-py3-none-any.whl", hash = "sha256:42e99004ce603f9a53934c634a57b04ad5900e0d8ed0abb15e635767489cbc05", size = 82741, upload-time = "2024-08-28T20:24:07.645Z" }, + { url = "https://files.pythonhosted.org/packages/f7/e6/efe534ef0952b531b630780e19cabd416e2032697019d5295defc6ef9bd9/deepdiff-8.6.1-py3-none-any.whl", hash = "sha256:ee8708a7f7d37fb273a541fa24ad010ed484192cd0c4ffc0fa0ed5e2d4b9e78b", size = 91378, upload-time = "2025-09-03T19:40:39.679Z" }, ] [[package]] @@ -272,7 +272,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "deepdiff", specifier = "==8.0.1" }, + { name = "deepdiff", specifier = "==8.6.1" }, { name = "orchestrator-core", specifier = "==4.0.4" }, { name = "pynetbox", specifier = "==7.4.1" }, { name = "rich", specifier = "==13.9.4" }, @@ -644,11 +644,11 @@ wheels = [ [[package]] name = "orderly-set" -version = "5.2.2" +version = "5.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c8/71/5408fee86ce5408132a3ece6eff61afa2c25d5b37cd76bc100a9a4a4d8dd/orderly_set-5.2.2.tar.gz", hash = "sha256:52a18b86aaf3f5d5a498bbdb27bf3253a4e5c57ab38e5b7a56fa00115cd28448", size = 19103, upload-time = "2024-08-28T20:12:57.618Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/88/39c83c35d5e97cc203e9e77a4f93bf87ec89cf6a22ac4818fdcc65d66584/orderly_set-5.5.0.tar.gz", hash = "sha256:e87185c8e4d8afa64e7f8160ee2c542a475b738bc891dc3f58102e654125e6ce", size = 27414, upload-time = "2025-07-10T20:10:55.885Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/71/6f9554919da608cb5bcf709822a9644ba4785cc7856e01ea375f6d808774/orderly_set-5.2.2-py3-none-any.whl", hash = "sha256:f7a37c95a38c01cdfe41c3ffb62925a318a2286ea0a41790c057fc802aec54da", size = 11621, upload-time = "2024-08-28T20:12:56.407Z" }, + { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, ] [[package]] diff --git a/workflows/l2vpn/create_l2vpn.py b/workflows/l2vpn/create_l2vpn.py index 4988f0c..517d6e2 100644 --- a/workflows/l2vpn/create_l2vpn.py +++ b/workflows/l2vpn/create_l2vpn.py @@ -12,7 +12,6 @@ # limitations under the License. -from pydantic_forms.types import UUIDstr import uuid from random import randrange from typing import TypeAlias, cast @@ -24,7 +23,7 @@ from orchestrator.workflows.utils import create_workflow from pydantic import ConfigDict from pydantic_forms.core import FormPage -from pydantic_forms.types import FormGenerator, State +from pydantic_forms.types import FormGenerator, State, UUIDstr from pydantic_forms.validators import Choice from products.product_blocks.sap import SAPBlockInactive @@ -48,7 +47,8 @@ class CreateL2vpnForm(FormPage): user_input = yield CreateL2vpnForm user_input_dict = user_input.model_dump() PortsChoiceList: TypeAlias = cast( - type[Choice], ports_selector(AllowedNumberOfL2vpnPorts(user_input_dict["number_of_ports"])) # noqa: F821 + type[Choice], + ports_selector(AllowedNumberOfL2vpnPorts(user_input_dict["number_of_ports"])), # noqa: F821 ) class SelectPortsForm(FormPage): @@ -90,7 +90,9 @@ def to_sap(port: UUIDstr) -> SAPBlockInactive: subscription.virtual_circuit.saps = [to_sap(port) for port in ports] - subscription = L2vpnProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) + subscription = L2vpnProvisioning.from_other_lifecycle( + subscription, SubscriptionLifecycle.PROVISIONING + ) subscription.description = description(subscription) return { @@ -125,7 +127,9 @@ def ims_create_l2vpn_terminations(subscription: L2vpnProvisioning) -> State: l2vpn = netbox.get_l2vpn(id=subscription.virtual_circuit.ims_id) for sap in subscription.virtual_circuit.saps: vlan = netbox.get_vlan(id=sap.ims_id) - payload = netbox.L2vpnTerminationPayload(l2vpn=l2vpn.id, assigned_object_id=vlan.id) + payload = netbox.L2vpnTerminationPayload( + l2vpn=l2vpn.id, assigned_object_id=vlan.id + ) netbox.create(payload) payloads.append(payload) diff --git a/workflows/l2vpn/shared/forms.py b/workflows/l2vpn/shared/forms.py index 54645fc..173be0e 100644 --- a/workflows/l2vpn/shared/forms.py +++ b/workflows/l2vpn/shared/forms.py @@ -16,16 +16,23 @@ from pydantic_forms.validators import Choice, choice_list from products.product_blocks.port import PortMode -from workflows.shared import AllowedNumberOfL2vpnPorts, subscriptions_by_product_type_and_instance_value +from workflows.shared import ( + AllowedNumberOfL2vpnPorts, + subscriptions_by_product_type_and_instance_value, +) -def ports_selector(number_of_ports: AllowedNumberOfL2vpnPorts) -> type[list[Choice]]: +def ports_selector( + number_of_ports: AllowedNumberOfL2vpnPorts, +) -> type[list[Choice]]: port_subscriptions = subscriptions_by_product_type_and_instance_value( "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] ) ports = { str(subscription.subscription_id): subscription.description - for subscription in sorted(port_subscriptions, key=lambda port: port.description) + for subscription in sorted( + port_subscriptions, key=lambda port: port.description + ) } return choice_list( Choice("PortsEnum", zip(ports.keys(), ports.items())), # type: ignore diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 9834d05..8b891ae 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -1,4 +1,7 @@ # workflows/nsistp/create_nsistp.py +# from typing import TypeAlias, cast + + import structlog from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage @@ -12,6 +15,17 @@ from pydantic_forms.types import FormGenerator, State, UUIDstr from products.product_types.nsistp import NsistpInactive, NsistpProvisioning +from products.services.netbox.netbox import build_payload +from services import netbox +from workflows.nsistp.shared.forms import ( + IsAlias, + ServiceSpeed, + StpDescription, + StpId, + Topology, + nsistp_fill_sap, +) +from workflows.nsistp.shared.helpers import FormNsistpPort from workflows.shared import create_summary_form @@ -33,24 +47,29 @@ def initial_input_form_generator(product_name: str) -> FormGenerator: class CreateNsistpForm(FormPage): model_config = ConfigDict(title=product_name) + # TODO: check whether this should be removed customer_id: CustomerId nsistp_settings: Label divider_1: Divider - topology: str - stp_id: str - stp_description: str | None = None - is_alias_in: str | None = None - is_alias_out: str | None = None - expose_in_topology: bool | None = None - bandwidth: int | None = None + # TODO: could this be multiple service ports?? + service_port: FormNsistpPort + + topology: Topology + stp_id: StpId + stp_description: StpDescription | None = None + is_alias_in: IsAlias | None = None + is_alias_out: IsAlias | None = None + expose_in_topology: bool = True + bandwidth: ServiceSpeed user_input = yield CreateNsistpForm user_input_dict = user_input.dict() summary_fields = [ "topology", + # "vlan", "stp_id", "stp_description", "is_alias_in", @@ -67,6 +86,7 @@ class CreateNsistpForm(FormPage): def construct_nsistp_model( product: UUIDstr, customer_id: UUIDstr, + service_port: list[dict], topology: str, stp_id: str, stp_description: str | None, @@ -75,6 +95,7 @@ def construct_nsistp_model( expose_in_topology: bool | None, bandwidth: int | None, ) -> State: + print("service Port in construct", service_port) nsistp = NsistpInactive.from_product_id( product_id=product, customer_id=customer_id, @@ -88,6 +109,8 @@ def construct_nsistp_model( nsistp.nsistp.expose_in_topology = expose_in_topology nsistp.nsistp.bandwidth = bandwidth + nsistp_fill_sap(nsistp, service_port) + nsistp = NsistpProvisioning.from_other_lifecycle( nsistp, SubscriptionLifecycle.PROVISIONING ) @@ -100,6 +123,14 @@ def construct_nsistp_model( } +@step("Create VLANs in IMS (Netbox)") +def ims_create_vlans(subscription: NsistpProvisioning) -> State: + payload = build_payload(subscription.nsistp.sap, subscription) + subscription.nsistp.sap.ims_id = netbox.create(payload) + + return {"subscription": subscription, "payloads": [payload]} + + additional_steps = begin @@ -110,6 +141,9 @@ def construct_nsistp_model( ) def create_nsistp() -> StepList: return ( - begin >> construct_nsistp_model >> store_process_subscription(Target.CREATE) + begin + >> construct_nsistp_model + >> store_process_subscription(Target.CREATE) + >> ims_create_vlans # TODO add provision step(s) ) diff --git a/workflows/nsistp/shared/__init__.py b/workflows/nsistp/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index e69de29..ec0c952 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -0,0 +1,242 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from collections.abc import Iterator +from datetime import datetime +from functools import partial +from typing import Annotated +from uuid import UUID + +from annotated_types import Ge, Le +from more_itertools import one +from orchestrator.db import ProductTable, db +from orchestrator.db.models import ( + SubscriptionTable, +) +from orchestrator.domain.base import SubscriptionModel +from orchestrator.types import SubscriptionLifecycle +from pydantic import AfterValidator, ValidationInfo +from pydantic_forms.types import State +from pydantic_forms.validators import Choice, choice_list +from sqlalchemy import select +from typing_extensions import Doc + +from forms.validator.service_port import service_port +from forms.validator.service_port_tags import ( + PORT_TAG_GENERAL, +) +from forms.validator.shared import MAX_SPEED_POSSIBLE +from products.product_blocks.port import PortMode +from products.product_types.nsistp import Nsistp, NsistpInactive +from utils.exceptions import DuplicateValueError, FieldValueError +from workflows.shared import ( + subscriptions_by_product_type_and_instance_value, +) + +TOPOLOGY_REGEX = r"^[-a-z0-9+,.;=_]+$" +STP_ID_REGEX = r"^[-a-z0-9+,.;=_:]+$" +NURN_REGEX = r"^urn:ogf:network:([^:]+):([0-9]+):([a-z0-9+,-.:;_!$()*@~&]*)$" +FQDN_REQEX = r"^(?!.{255}|.{253}[^.])([a-z0-9](?:[-a-z-0-9]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?[.]?$" + + +def nsistp_service_port(current: list[State] | None = None) -> type: + print("current", current) + return service_port( + visible_port_mode="tagged", + allowed_tags=PORT_TAG_GENERAL, + current=current, + ) + + +def is_fqdn(hostname: str) -> bool: + return re.match(FQDN_REQEX, hostname, re.IGNORECASE) is not None + + +def valid_date(date: str) -> tuple[bool, str | None]: + def valid_month() -> tuple[bool, str | None]: + month_str = date[4:6] + month = int(month_str) + if month < 1 or month > 12: + return False, f"{month_str} is not a valid month number" + return True, None + + def valid_day() -> tuple[bool, str | None]: + try: + datetime.fromisoformat(f"{date[0:4]}-{date[4:6]}-{date[6:8]}") + except ValueError: + return False, f"`{date}` is not a valid date" + return True, None + + length = len(date) + if length == 4: # year + pass # No checks on reasonable year, so 9999 is allowed + elif length in (6, 8): + valid, message = valid_month() + if not valid: + return valid, message + if length == 8: # year + month + day + return valid_day() + else: + return False, f"date `{date}` has invalid length" + + return True, None + + +def valid_nurn(nurn: str) -> tuple[bool, str | None]: + if not (match := re.match(NURN_REGEX, nurn, re.IGNORECASE)): + return False, "not a valid NSI STP identifier (urn:ogf:network:...)" + + hostname = match.group(1) + if not is_fqdn(hostname): + return False, f"{hostname} is not a valid fqdn" + + date = match.group(2) + valid, message = valid_date(date) + + return valid, message + + +def validate_regex( + regex: str, + message: str, + field: str | None, +) -> str | None: + if field is None: + return field + + if not re.match(regex, field, re.IGNORECASE): + raise FieldValueError(f"{message} must match: {regex}") + + return field + + +def _get_nsistp_subscriptions(subscription_id: UUID | None) -> Iterator[Nsistp]: + query = ( + select(SubscriptionTable.subscription_id) + .join(ProductTable) + .filter( + ProductTable.product_type == "NSISTP", + SubscriptionTable.status == SubscriptionLifecycle.ACTIVE, + SubscriptionTable.subscription_id != subscription_id, + ) + ) + result = db.session.scalars(query).all() + return (Nsistp.from_subscription(subscription_id) for subscription_id in result) + + +def validate_stp_id_uniqueness( + subscription_id: UUID | None, stp_id: str, info: ValidationInfo +) -> str: + values = info.data + + customer_id = values.get("customer_id") + topology = values.get("topology") + + if customer_id and topology: + + def is_not_unique(nsistp: Nsistp) -> bool: + return ( + nsistp.settings.stp_id.casefold() == stp_id.casefold() + and nsistp.settings.topology.casefold() == topology.casefold() + ) + + subscriptions = _get_nsistp_subscriptions(subscription_id) + if any(is_not_unique(nsistp) for nsistp in subscriptions): + raise DuplicateValueError( + f"STP identifier `{stp_id}` already exists for topology `{topology}`" + ) + + return stp_id + + +StpId = Annotated[ + str, + AfterValidator(partial(validate_regex, STP_ID_REGEX, "STP identifier")), + Doc("must be unique along the set of NSISTP's in the same TOPOLOGY"), +] + + +def validate_both_aliases_empty_or_not( + is_alias_in: str | None, is_alias_out: str | None +) -> None: + if bool(is_alias_in) != bool(is_alias_out): + raise FieldValueError( + "NSI inbound and outbound isAlias should either both have a value or be empty" + ) + + +def validate_nurn(nurn: str | None) -> str | None: + if nurn: + valid, message = valid_nurn(nurn) + if not valid: + raise FieldValueError(message) + + return nurn + + +def nsistp_fill_sap(subscription: NsistpInactive, service_ports: list[dict]) -> None: + print("nsi_fill_sap", service_ports) + sp = one(service_ports) + subscription.nsistp.sap.vlan = sp["vlan"] + # SubscriptionModel can be any type of ServicePort + subscription.nsistp.sap.port = SubscriptionModel.from_subscription( + sp["port_id"] + ).port # type: ignore + + +IsAlias = Annotated[ + str, + AfterValidator(validate_nurn), + Doc("ISALIAS conform https://www.ogf.org/documents/GFD.202.pdf"), +] + +StpDescription = Annotated[ + str, + AfterValidator(partial(validate_regex, r"^[^<>&]*$", "STP description")), + Doc("STP description may not contain characters from the set [<>&]"), +] + +Topology = Annotated[ + str, + AfterValidator(partial(validate_regex, TOPOLOGY_REGEX, "Topology")), + Doc("topology string may only consist of characters from the set [-a-z+,.;=_]"), +] + + +Bandwidth = Annotated[ + int, + Ge(1), + Le(MAX_SPEED_POSSIBLE), + Doc(f"Bandwidth between {1} and {MAX_SPEED_POSSIBLE}"), +] + +ServiceSpeed = Bandwidth + + +# NOTE: currently in helpers.py +def ports_selector() -> type[list[Choice]]: + port_subscriptions = subscriptions_by_product_type_and_instance_value( + "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] + ) + ports = { + str(subscription.subscription_id): subscription.description + for subscription in sorted( + port_subscriptions, key=lambda port: port.description + ) + } + return choice_list( + Choice("PortsEnum", zip(ports.keys(), ports.items())), # type: ignore + unique_items=True, + min_items=1, + max_items=1, + ) diff --git a/workflows/nsistp/shared/helpers.py b/workflows/nsistp/shared/helpers.py new file mode 100644 index 0000000..714dd6e --- /dev/null +++ b/workflows/nsistp/shared/helpers.py @@ -0,0 +1,108 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import structlog +from nwastdlib.vlans import VlanRanges +from orchestrator.db import ( + SubscriptionTable, +) +from orchestrator.services import subscriptions +from orchestrator.types import SubscriptionLifecycle +from pydantic import BaseModel, GetJsonSchemaHandler, TypeAdapter, field_validator +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema +from pydantic_core.core_schema import ValidationInfo +from pydantic_forms.validators import Choice + +from forms.validator.shared import ( + _get_port_mode, +) +from products.product_blocks.port import PortMode +from utils.exceptions import VlanValueError +from workflows.shared import subscriptions_by_product_type_and_instance_value + +logger = structlog.get_logger(__name__) + + +# Custom VlanRanges needed to avoid matching conflict with SURF orchestrator-ui components +class CustomVlanRanges(VlanRanges): + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + parent_schema = super().__get_pydantic_json_schema__(core_schema_, handler) + parent_schema["format"] = "custom-vlan" + + return parent_schema + + +# Add this after your CustomVlanRanges class definition +adapter = TypeAdapter(CustomVlanRanges) +print("CustomVlanRanges schema:", adapter.json_schema()) + + +def ports_selector() -> type[list[Choice]]: + port_subscriptions = subscriptions_by_product_type_and_instance_value( + "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] + ) + ports = { + str(subscription.subscription_id): subscription.description + for subscription in sorted( + port_subscriptions, key=lambda port: port.description + ) + } + + return Choice("ServicePort", zip(ports.keys(), ports.items())) + + +class FormNsistpPort(BaseModel): + # NOTE: subscription_id and vlan are pydantic_forms fields which render ui components + port_id: ports_selector() # type: ignore # noqa: F821 + vlan: CustomVlanRanges + + def __repr__(self) -> str: + # Help distinguish this from the ServicePort product type.. + return f"FormServicePort({self.port_id=}, {self.vlan=})" + + @field_validator("vlan") + @classmethod + def check_vlan( + cls, vlan: CustomVlanRanges, info: ValidationInfo + ) -> CustomVlanRanges: + print("hallo_vlan", vlan) + # We assume an empty string is untagged and thus 0 + if not vlan: + vlan = CustomVlanRanges(0) + + subscription_id = info.data.get("port_id") + print("port:", subscription_id) + if not subscription_id: + return vlan + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + port_mode = _get_port_mode(subscription) + print("port_mode:", port_mode) + + if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): + raise VlanValueError( + f"{port_mode} {subscription.product.tag} must have a vlan" + ) + elif port_mode == PortMode.UNTAGGED and vlan != CustomVlanRanges(0): # noqa: RET506 + raise VlanValueError( + f"{port_mode} {subscription.product.tag} can not have a vlan" + ) + + return vlan diff --git a/workflows/nsistp/shared/nsistp.py b/workflows/nsistp/shared/nsistp.py new file mode 100644 index 0000000..c0a50a2 --- /dev/null +++ b/workflows/nsistp/shared/nsistp.py @@ -0,0 +1,104 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Sequence +from uuid import UUID + +from more_itertools import flatten +from nwastdlib.vlans import VlanRanges +from orchestrator.db import db +from orchestrator.db.models import ( + ProductTable, + SubscriptionInstanceRelationTable, + SubscriptionInstanceTable, + SubscriptionTable, +) +from orchestrator.types import SubscriptionLifecycle +from sqlalchemy import select +from sqlalchemy.orm import aliased + +# from surf.products.product_types.nsi_lp import NsiLightPath +from products.product_types.nsistp import Nsistp + + +def _get_subscriptions_inuseby_port_id( + port_id: UUID, product_type: str, statuses: list[SubscriptionLifecycle] +) -> Sequence[UUID]: + relations = aliased(SubscriptionInstanceRelationTable) + instances = aliased(SubscriptionInstanceTable) + query = ( + select(SubscriptionTable.subscription_id) + .join(SubscriptionInstanceTable) + .join(ProductTable) + .join( + relations, + relations.in_use_by_id + == SubscriptionInstanceTable.subscription_instance_id, + ) + .join(instances, relations.depends_on_id == instances.subscription_instance_id) + .filter(instances.subscription_id == port_id) + .filter(ProductTable.product_type == product_type) + .filter(SubscriptionTable.status.in_(statuses)) + ) + return db.session.scalars(query).all() + + +def nsistp_get_by_port_id(port_id: UUID) -> list[Nsistp]: + """Get Nsistps by service port id. + + Args: + port_id: ID of the service port for which you want all nsistps of. + """ + statuses = [ + SubscriptionLifecycle.ACTIVE, + SubscriptionLifecycle.PROVISIONING, + SubscriptionLifecycle.MIGRATING, + ] + result = _get_subscriptions_inuseby_port_id(port_id, "NSISTP", statuses) + + return [Nsistp.from_subscription(id) for id in list(set(result))] + + +# def nsi_lp_get_by_port_id(port_id: UUID) -> list[NsiLightPath]: +# """Get NsiLightPaths by service port id. + +# Args: +# port_id: ID of the service port for which you want all NsiLightPaths of. +# """ +# statuses = [SubscriptionLifecycle.ACTIVE] +# result = _get_subscriptions_inuseby_port_id(port_id, "NSILP", statuses) + +# return [NsiLightPath.from_subscription(id) for id in list(set(result))] + + +def get_available_vlans_by_port_id(port_id: UUID) -> VlanRanges: + """Get available vlans by service port id. + + This will get all NSISTPs and adds their vlan ranges to a single VlanRanges to get the available vlans by nsistps. + + Then filters out the vlans that are already in use by NSI light paths and returns the available vlans. + + Args: + port_id: ID of the service port to find available vlans. + """ + nsistps = nsistp_get_by_port_id(port_id) + available_vlans = VlanRanges(flatten(nsistp.vlan_range for nsistp in nsistps)) + + # NOTE: Lightpad can be ommited? + # nsi_lps = nsi_lp_get_by_port_id(port_id) + # used_vlans = VlanRanges( + # flatten(sap.vlanrange for nsi_lp in nsi_lps for sap in nsi_lp.vc.saps) + # ) + + # return available_vlans - used_vlans + + return available_vlans diff --git a/workflows/port/create_port.py b/workflows/port/create_port.py index 5f6d13f..be96008 100644 --- a/workflows/port/create_port.py +++ b/workflows/port/create_port.py @@ -12,22 +12,21 @@ # limitations under the License. -from pydantic_forms.types import UUIDstr -import uuid import json +import uuid from random import randrange from typing import TypeAlias, cast from orchestrator.services.products import get_product_by_id from orchestrator.targets import Target from orchestrator.types import SubscriptionLifecycle +from orchestrator.utils.json import json_dumps from orchestrator.workflow import StepList, begin, step from orchestrator.workflows.steps import store_process_subscription from orchestrator.workflows.utils import create_workflow -from orchestrator.utils.json import json_dumps from pydantic import ConfigDict from pydantic_forms.core import FormPage -from pydantic_forms.types import FormGenerator, State +from pydantic_forms.types import FormGenerator, State, UUIDstr from pydantic_forms.validators import Choice, Label from products.product_blocks.port import PortMode @@ -57,7 +56,9 @@ class SelectNodeForm(FormPage): _product = get_product_by_id(product) speed = int(_product.fixed_input_value("speed")) - FreePortChoice: TypeAlias = cast(type[Choice], free_port_selector(node_subscription_id, speed)) + FreePortChoice: TypeAlias = cast( + type[Choice], free_port_selector(node_subscription_id, speed) + ) class CreatePortForm(FormPage): model_config = ConfigDict(title=product_name) @@ -75,7 +76,13 @@ class CreatePortForm(FormPage): user_input = yield CreatePortForm user_input_dict = user_input.model_dump() - summary_fields = ["port_ims_id", "port_description", "port_mode", "auto_negotiation", "lldp"] + summary_fields = [ + "port_ims_id", + "port_description", + "port_mode", + "auto_negotiation", + "lldp", + ] yield from create_summary_form(user_input_dict, product_name, summary_fields) return user_input_dict | {"node_subscription_id": node_subscription_id} @@ -108,7 +115,9 @@ def construct_port_model( subscription.port.enabled = False subscription.port.ims_id = port_ims_id - subscription = PortProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) + subscription = PortProvisioning.from_other_lifecycle( + subscription, SubscriptionLifecycle.PROVISIONING + ) subscription.description = description(subscription) return { diff --git a/workflows/shared.py b/workflows/shared.py index 45e5e44..5c8dff4 100644 --- a/workflows/shared.py +++ b/workflows/shared.py @@ -10,8 +10,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pydantic_forms.types import UUIDstr -from pydantic_forms.types import SummaryData from pprint import pformat from typing import Annotated, Generator, List, TypeAlias, cast @@ -28,6 +26,7 @@ from orchestrator.types import SubscriptionLifecycle from pydantic import ConfigDict from pydantic_forms.core import FormPage +from pydantic_forms.types import SummaryData, UUIDstr from pydantic_forms.validators import Choice, MigrationSummary, migration_summary from products.product_types.node import Node @@ -35,10 +34,18 @@ Vlan = Annotated[int, Ge(2), Le(4094), doc("VLAN ID.")] -AllowedNumberOfL2vpnPorts = Annotated[int, Ge(2), Le(8), doc("Allowed number of L2vpn ports.")] +AllowedNumberOfL2vpnPorts = Annotated[ + int, Ge(2), Le(8), doc("Allowed number of L2vpn ports.") +] +AllowedNumberOfNsistpPorts = Annotated[ + int, Ge(1), Le(1), doc("Allowed number of Nsistp ports.") +] -def subscriptions_by_product_type(product_type: str, status: List[SubscriptionLifecycle]) -> List[SubscriptionTable]: + +def subscriptions_by_product_type( + product_type: str, status: List[SubscriptionLifecycle] +) -> List[SubscriptionTable]: """ retrieve_subscription_list_by_product This function lets you retreive a list of all subscriptions of a given product type. For example, you could @@ -69,7 +76,10 @@ def subscriptions_by_product_type(product_type: str, status: List[SubscriptionLi def subscriptions_by_product_type_and_instance_value( - product_type: str, resource_type: str, value: str, status: List[SubscriptionLifecycle] + product_type: str, + resource_type: str, + value: str, + status: List[SubscriptionLifecycle], ) -> List[SubscriptionTable]: """Retrieve a list of Subscriptions by product_type, resource_type and value. @@ -96,25 +106,35 @@ def subscriptions_by_product_type_and_instance_value( def node_selector(enum: str = "NodesEnum") -> type[Choice]: - node_subscriptions = subscriptions_by_product_type("Node", [SubscriptionLifecycle.ACTIVE]) + node_subscriptions = subscriptions_by_product_type( + "Node", [SubscriptionLifecycle.ACTIVE] + ) nodes = { str(subscription.subscription_id): subscription.description - for subscription in sorted(node_subscriptions, key=lambda node: node.description) + for subscription in sorted( + node_subscriptions, key=lambda node: node.description + ) } return Choice(enum, zip(nodes.keys(), nodes.items())) # type:ignore -def free_port_selector(node_subscription_id: UUIDstr, speed: int, enum: str = "PortsEnum") -> type[Choice]: +def free_port_selector( + node_subscription_id: UUIDstr, speed: int, enum: str = "PortsEnum" +) -> type[Choice]: node = Node.from_subscription(node_subscription_id) interfaces = { str(interface.id): interface.name - for interface in netbox.get_interfaces(device=node.node.node_name, speed=speed * 1000, enabled=False) + for interface in netbox.get_interfaces( + device=node.node.node_name, speed=speed * 1000, enabled=False + ) } return Choice(enum, zip(interfaces.keys(), interfaces.items())) # type:ignore def summary_form(product_name: str, summary_data: SummaryData) -> Generator: - ProductSummary: TypeAlias = cast(type[MigrationSummary], migration_summary(summary_data)) + ProductSummary: TypeAlias = cast( + type[MigrationSummary], migration_summary(summary_data) + ) class SummaryForm(FormPage): model_config = ConfigDict(title=f"{product_name} summary") @@ -124,17 +144,23 @@ class SummaryForm(FormPage): yield SummaryForm -def create_summary_form(user_input: dict, product_name: str, fields: List[str]) -> Generator: +def create_summary_form( + user_input: dict, product_name: str, fields: List[str] +) -> Generator: columns = [[str(user_input[nm]) for nm in fields]] yield from summary_form(product_name, SummaryData(labels=fields, columns=columns)) # type: ignore -def modify_summary_form(user_input: dict, block: ProductBlockModel, fields: List[str]) -> Generator: +def modify_summary_form( + user_input: dict, block: ProductBlockModel, fields: List[str] +) -> Generator: before = [str(getattr(block, nm)) for nm in fields] # type: ignore[attr-defined] after = [str(user_input[nm]) for nm in fields] yield from summary_form( block.subscription.product.name, - SummaryData(labels=fields, headers=["Before", "After"], columns=[before, after]), + SummaryData( + labels=fields, headers=["Before", "After"], columns=[before, after] + ), ) From 1d62263f81062bc0b989deaa5b8263a7deb722f3 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Fri, 3 Oct 2025 17:37:17 +0200 Subject: [PATCH 03/23] Restructered with working create_form --- forms/__init__.py | 1 - forms/validator/__init__.py | 0 forms/validator/service_port.py | 337 ------------------ forms/validator/service_port_tags.py | 41 --- forms/validator/shared.py | 93 ----- forms/validator/subscription_bandwidth.py | 60 ---- forms/validator/subscription_customer.py | 52 --- .../subscription_exclude_subscriptions.py | 49 --- forms/validator/subscription_id.py | 138 ------- forms/validator/subscription_in_sync.py | 42 --- forms/validator/subscription_is_port.py | 44 --- forms/validator/subscription_port_mode.py | 66 ---- forms/validator/subscription_product_id.py | 56 --- forms/validator/subscription_status.py | 55 --- forms/validator/subscription_tag.py | 53 --- forms/validator/vlan_ranges.py | 35 -- products/product_blocks/sap.py | 16 +- products/product_types/nsistp.py | 5 + {forms => utils}/types.py | 24 +- workflows/nsistp/create_nsistp.py | 43 ++- workflows/nsistp/shared/forms.py | 124 +++---- workflows/nsistp/shared/helpers.py | 108 ------ .../shared/{nsistp.py => nsistp_services.py} | 31 +- workflows/nsistp/shared/shared.py | 67 ++++ workflows/nsistp/shared/vlan.py | 232 ++++++++++++ 25 files changed, 418 insertions(+), 1354 deletions(-) delete mode 100644 forms/__init__.py delete mode 100644 forms/validator/__init__.py delete mode 100644 forms/validator/service_port.py delete mode 100644 forms/validator/service_port_tags.py delete mode 100644 forms/validator/shared.py delete mode 100644 forms/validator/subscription_bandwidth.py delete mode 100644 forms/validator/subscription_customer.py delete mode 100644 forms/validator/subscription_exclude_subscriptions.py delete mode 100644 forms/validator/subscription_id.py delete mode 100644 forms/validator/subscription_in_sync.py delete mode 100644 forms/validator/subscription_is_port.py delete mode 100644 forms/validator/subscription_port_mode.py delete mode 100644 forms/validator/subscription_product_id.py delete mode 100644 forms/validator/subscription_status.py delete mode 100644 forms/validator/subscription_tag.py delete mode 100644 forms/validator/vlan_ranges.py rename {forms => utils}/types.py (60%) delete mode 100644 workflows/nsistp/shared/helpers.py rename workflows/nsistp/shared/{nsistp.py => nsistp_services.py} (70%) create mode 100644 workflows/nsistp/shared/shared.py create mode 100644 workflows/nsistp/shared/vlan.py diff --git a/forms/__init__.py b/forms/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/forms/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/forms/validator/__init__.py b/forms/validator/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/forms/validator/service_port.py b/forms/validator/service_port.py deleted file mode 100644 index 37699e9..0000000 --- a/forms/validator/service_port.py +++ /dev/null @@ -1,337 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import operator -import random -import string -from collections import defaultdict -from collections.abc import Iterable -from typing import Any -from uuid import UUID - -import structlog -from more_itertools import flatten -from nwastdlib.vlans import VlanRanges -from orchestrator.db import ( - SubscriptionTable, -) -from orchestrator.services import subscriptions -from orchestrator.types import SubscriptionLifecycle -from pydantic import BaseModel, Field, create_model, field_validator -from pydantic_core.core_schema import ValidationInfo -from pydantic_forms.types import State, UUIDstr - -from forms.types import Tags, VisiblePortMode -from forms.validator.shared import ( - PortTag, - _get_port_mode, - get_port_speed_for_port_subscription, -) -from forms.validator.subscription_id import subscription_id -from forms.validator.vlan_ranges import NsiVlanRanges -from products.product_blocks.port import PortMode -from utils.exceptions import PortsValueError, VlanValueError -from workflows.nsistp.shared.nsistp import ( - get_available_vlans_by_port_id, - nsistp_get_by_port_id, -) - -logger = structlog.get_logger(__name__) - -_port_tag_values = [str(value) for value in PortTag] - - -class ServicePortNoVlan(BaseModel): - subscription_id: subscription_id(allowed_tags=_port_tag_values) # type: ignore # noqa: F821 - - def __repr__(self) -> str: - return f"FormServicePortNoVlan({self.subscription_id=})" - - -class ServicePort(BaseModel): - # By default we don't want constraints but having ports in the allowed taglist also signals the frontend - # This way we get te correct behavior even though we don't constrain.. (Well we are sure we want ports here..) - subscription_id: subscription_id(allowed_tags=_port_tag_values) # type: ignore # noqa: F821 - vlan: VlanRanges - - def __repr__(self) -> str: - # Help distinguish this from the ServicePort product type.. - print("subscription_id", self.subscription_id) - - return f"FormServicePort({self.subscription_id=}, {self.vlan=})" - - @field_validator("vlan") - @classmethod - def check_vlan(cls, vlan: VlanRanges, info: ValidationInfo) -> VlanRanges: - # We assume an empty string is untagged and thus 0 - if not vlan: - vlan = VlanRanges(0) - - subscription_id = info.data.get("subscription_id") - if not subscription_id: - return vlan - - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) - - port_mode = _get_port_mode(subscription) - - if port_mode == PortMode.tagged and vlan == VlanRanges(0): - raise VlanValueError( - f"{port_mode} {subscription.product.tag} must have a vlan" - ) - elif port_mode == PortMode.untagged and vlan != VlanRanges(0): # noqa: RET506 - raise VlanValueError( - f"{port_mode} {subscription.product.tag} can not have a vlan" - ) - - return vlan - - -class NsiServicePort(ServicePort): - vlan: NsiVlanRanges - - -def _random_service_port_str() -> str: - return "".join( - random.choice(string.ascii_letters + string.digits) for _ in range(8) - ) # noqa: S311 - - -def service_port( - visible_port_mode: VisiblePortMode | None = None, - customer_id: UUIDstr | None = None, - customer_key: str | None = None, - customer_ports_only: bool = False, - bandwidth: int | None = None, - bandwidth_key: str | None = None, - current: list[State] | None = None, - allowed_tags: list[Tags] | None = None, - disabled_ports: bool | None = None, - excluded_subscriptions: list[UUID] | None = None, - allowed_statuses: list[SubscriptionLifecycle] | None = None, - nsi_vlans_only: bool = False, -) -> type[ServicePort]: - """Extend the normal validator with configurable constraints.""" - - @field_validator("vlan") # type: ignore[misc] - @classmethod - def check_vlan_in_use( - cls: ServicePort, v: VlanRanges, info: ValidationInfo - ) -> VlanRanges: - """Check if vlan value is already in use by service port. - - Args: - cls: class - v: Vlan range of the form input. - info: validation info, contains other fields in info.data - - 1. Get all used vlans in a service port. - 2. Get all nsi reserved vlans and add them to the used_vlans. - 3. Filter out vlans used in current subscription from used_vlans. - 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. - 5. checks if input value uses already used vlans. errors if true. - 6. return input value. - - - """ - if not (subscription_id := info.data.get("subscription_id")): - return v - - # used_vlans = VlanRanges(ims.get_vlans_by_subscription_id(subscription_id)) - # return available_vlans - used_vlans - used_vlans = nsistp_get_by_port_id(subscription_id) - print("used_vlans", used_vlans) - nsistp_reserved_vlans = get_available_vlans_by_port_id(subscription_id) - used_vlans = VlanRanges( - flatten([list(used_vlans), list(nsistp_reserved_vlans)]) - ) - - # Remove currently chosen vlans for this port to prevent tripping on in used by itself - current_selected_vlan_ranges: list[str] = [] - if current: - current_selected_service_port = filter( - lambda c: str(c["subscription_id"]) == str(subscription_id), current - ) - current_selected_vlans = list( - map(operator.itemgetter("vlan"), current_selected_service_port) - ) - for current_selected_vlan in current_selected_vlans: - # We assume an empty string is untagged and thus 0 - if not current_selected_vlan: - current_selected_vlan = "0" - - current_selected_vlan_range = VlanRanges(current_selected_vlan) - used_vlans -= current_selected_vlan_range - current_selected_vlan_ranges = [ - *current_selected_vlan_ranges, - *list(current_selected_vlan_range), - ] - - # TODO (#1842): probably better to have a separate type/validator for this - if nsi_vlans_only: - vlan_list = list(v) - invalid_ranges = [ - vlan - for vlan in vlan_list - if vlan not in list(nsistp_reserved_vlans) - and vlan not in current_selected_vlan_ranges - ] - used_vlans -= nsistp_reserved_vlans - - if invalid_ranges: - raise VlanValueError( - f"Vlan(s) {VlanRanges(invalid_ranges)} not valid nsi vlan range" - ) - - logger.info( - "Validation info for current chosen vlans vs vlan already in use", - current=current, - used_vlans=used_vlans, - subscription_id=subscription_id, - ) - - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) - print("subscription check", subscription) - - if v & used_vlans: - port_mode = _get_port_mode(subscription) - - # for tagged only; for link_member/untagged say "SP already in use" - if port_mode == PortMode.untagged or port_mode == PortMode.link_member: - raise PortsValueError("Service Port already in use") - raise VlanValueError(f"Vlan(s) {used_vlans} already in use") - - return v - - # Choose needed extra validators - validators: dict[str, Any] = { - "check_vlan_in_use": check_vlan_in_use, - } - - # Choose Base Model - base_model = NsiServicePort if nsi_vlans_only else ServicePort - - print("allowed_tags", allowed_tags) - print("base_model", base_model) - - return create_model( - f"{base_model.__name__}{_random_service_port_str()}Value", - __base__=base_model, - __validators__=validators, - subscription_id=( - subscription_id( - visible_port_mode=visible_port_mode, - customer_id=customer_id if customer_ports_only else None, - customer_key=customer_key if customer_ports_only else None, - bandwidth=bandwidth, - bandwidth_key=bandwidth_key, - allowed_tags=allowed_tags, - excluded_subscriptions=excluded_subscriptions, - allowed_statuses=allowed_statuses, - ), - Field(...), - ), - ) - - -def service_port_no_vlan( - visible_port_mode: VisiblePortMode | None = None, - customer_id: UUIDstr | None = None, - customer_key: str | None = None, - customer_ports_only: bool = False, - bandwidth: int | None = None, - bandwidth_key: str | None = None, - allowed_tags: list[Tags] | None = None, - excluded_subscriptions: list[UUID] | None = None, - allowed_statuses: list[SubscriptionLifecycle] | None = None, -) -> type[ServicePortNoVlan]: - """Extend the normal validator with configurable constraints.""" - - base_model = ServicePortNoVlan - - return create_model( - f"{base_model.__name__}{_random_service_port_str()}Value", - __base__=base_model, - subscription_id=( - subscription_id( - visible_port_mode=visible_port_mode, - customer_id=customer_id if customer_ports_only else None, - customer_key=customer_key if customer_ports_only else None, - bandwidth=bandwidth, - bandwidth_key=bandwidth_key, - allowed_tags=allowed_tags, - excluded_subscriptions=excluded_subscriptions, - allowed_statuses=allowed_statuses, - ), - Field(...), - ), - ) - - -def service_port_values(values: Iterable[dict] | None) -> list[ServicePort] | None: - if values is None: - return None - - return [ - ServicePort.model_construct( - subscription_id=UUID(v["subscription_id"]), vlan=VlanRanges(v["vlan"]) - ) - for v in values - ] - - -def validate_single_vlan(v: list[ServicePort]) -> list[ServicePort]: - if not all(sp.vlan.is_single_vlan for sp in v): - raise VlanValueError("This product only supports a single vlan") - return v - - -def validate_service_ports_bandwidth( - v: ServicePort, info: ValidationInfo, *, bandwidth_key: str -) -> ServicePort: - values = info.data - - if bandwidth := values.get(bandwidth_key): - port_speed = get_port_speed_for_port_subscription(v.subscription_id) - - if int(bandwidth) > port_speed: - raise PortsValueError( - f"The port speed is lower than the desired speed {bandwidth}" - ) - - return v - - -def validate_owner_of_service_port(customer: str | None, v: ServicePort) -> ServicePort: - if customer: - subscription = subscriptions.get_subscription(v.subscription_id) - - if subscription.customer_id != str(customer): - raise PortsValueError(f"Port subscription is not of customer {customer}") - - return v - - -def validate_service_ports_unique_vlans(v: list[ServicePort]) -> list[ServicePort]: - vlans: dict[UUID, list[VlanRanges]] = defaultdict(list) - for sap in v: - for vlan in vlans[sap.subscription_id]: - if not sap.vlan.isdisjoint(vlan): - raise VlanValueError("Vlans are already in use") - - vlans[sap.subscription_id].append(sap.vlan) - - return v diff --git a/forms/validator/service_port_tags.py b/forms/validator/service_port_tags.py deleted file mode 100644 index de4e8e1..0000000 --- a/forms/validator/service_port_tags.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from forms.types import Tags - -PORT_TAG_GENERAL: list[Tags] = ["PORT"] - -# TODO: these tags can probably be removed -PORT_TAGS_AGGSP: list[Tags] = ["AGGSP", "AGGSPNL"] -PORT_TAGS_IRBSP: list[Tags] = ["IRBSP"] -PORT_TAGS_MSC: list[Tags] = ["MSC", "MSCNL"] -PORT_TAGS_SPNL: list[Tags] = ["SPNL"] -PORT_TAGS_SP: list[Tags] = ["SP"] -PORT_TAGS_SP_ALL: list[Tags] = PORT_TAGS_SPNL + PORT_TAGS_SP -PORT_TAGS_ALL: list[Tags] = ( - PORT_TAGS_SP_ALL - + PORT_TAGS_AGGSP - + PORT_TAGS_MSC - + PORT_TAGS_IRBSP - + PORT_TAG_GENERAL -) -SERVICES_TAGS_FOR_IMS_REDEPLOY: list[Tags] = [ - "IPBGP", - "IPS", - "L2VPN", - "L3VPN", - "LP", - "LR", - "NSILP", -] -TAGS_FOR_IMS_REDEPLOY: list[Tags] = SERVICES_TAGS_FOR_IMS_REDEPLOY + PORT_TAGS_ALL diff --git a/forms/validator/shared.py b/forms/validator/shared.py deleted file mode 100644 index 713a262..0000000 --- a/forms/validator/shared.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Callable -from enum import StrEnum, auto -from functools import partial -from typing import Any -from uuid import UUID - -import structlog -from annotated_types import BaseMetadata -from orchestrator.db import ( - SubscriptionTable, -) -from orchestrator.services import subscriptions -from pydantic import Field - -from products.product_types.port import Port as ServicePort - -logger = structlog.get_logger(__name__) - -GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] - -PORT_SPEED = "port_speed" -MAX_SPEED_POSSIBLE = 400_000 - - -class PortMode(StrEnum): - tagged = auto() - untagged = auto() - link_member = auto() - - -class PortTag(StrEnum): - SP = "SP" - SPNL = "SPNL" - AGGSP = "AGGSP" - AGGSPNL = "AGGSPNL" - MSC = "MSC" - MSCNL = "MSCNL" - IRBSP = "IRBSP" - - -def _get_port_mode(subscription: SubscriptionTable) -> PortMode: - if subscription.product.tag in [PortTag.AGGSP + PortTag.SP]: - return subscription.port_mode - return PortMode.tagged - - -def get_port_speed_for_port_subscription( - subscription_id: UUID, get_subscription: GetSubscriptionByIdFunc | None = None -) -> int: - print("HELLO from get_port_speed_for_port_subscription") - if get_subscription: - subscription = get_subscription(subscription_id) - print("subscription from get_subscription", subscription) - else: - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) - - if subscription.tag in [PortTag.MSC + PortTag.AGGSP]: - port_speed = ServicePort.from_subscription( - subscription.subscription_id - ).get_port_speed() - elif subscription.tag in PortTag.IRBSP: - port_speed = MAX_SPEED_POSSIBLE - else: - port_speed = int(subscription.product.fixed_input_value(PORT_SPEED)) - - logger.info( - "Validation determined speed for port", - product_tag=subscription.tag, - port_speed=port_speed, - ) - return port_speed - - -def merge_uniforms(schema: dict[str, Any], *, to_merge: dict[str, Any]) -> None: - schema["uniforms"] = schema.get("uniforms", {}) | to_merge - - -def uniforms_field(to_merge: dict[str, Any]) -> BaseMetadata: - return Field(json_schema_extra=partial(merge_uniforms, to_merge=to_merge)) diff --git a/forms/validator/subscription_bandwidth.py b/forms/validator/subscription_bandwidth.py deleted file mode 100644 index 8fadc6d..0000000 --- a/forms/validator/subscription_bandwidth.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.validator.shared import ( - GetSubscriptionByIdFunc, - get_port_speed_for_port_subscription, - uniforms_field, -) -from utils.exceptions import PortsValueError - - -def validate_service_port_bandwidth( - v: UUID, *, bandwidth: int, get_subscription: GetSubscriptionByIdFunc -) -> UUID: - port_speed = get_port_speed_for_port_subscription( - v, get_subscription=get_subscription - ) - - if bandwidth > port_speed: - raise PortsValueError( - f"The port speed is lower than the desired speed {bandwidth}" - ) - - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionBandwidthValidator(GroupedMetadata): - bandwidth: int - get_subscription: GetSubscriptionByIdFunc - bandwidth_key: str | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_service_port_bandwidth, - bandwidth=self.bandwidth, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field( - {"bandwidth": self.bandwidth, "bandwidthKey": self.bandwidth_key} - ) diff --git a/forms/validator/subscription_customer.py b/forms/validator/subscription_customer.py deleted file mode 100644 index 63a7c55..0000000 --- a/forms/validator/subscription_customer.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator -from pydantic_forms.types import UUIDstr - -from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field -from utils.exceptions import CustomerValueError - - -def validate_customer( - v: UUID, *, customer_id: UUIDstr, get_subscription: GetSubscriptionByIdFunc -) -> UUID: - subscription = get_subscription(v) - - if subscription.customer_id != str(customer_id): - raise CustomerValueError(f"Subscription is not of customer {customer_id}") - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionCustomerValidator(GroupedMetadata): - customer_id: UUIDstr - get_subscription: GetSubscriptionByIdFunc - customer_key: str | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_customer, - customer_id=self.customer_id, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field( - {"customerId": self.customer_id, "customerKey": self.customer_key} - ) diff --git a/forms/validator/subscription_exclude_subscriptions.py b/forms/validator/subscription_exclude_subscriptions.py deleted file mode 100644 index 8636f3f..0000000 --- a/forms/validator/subscription_exclude_subscriptions.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.validator.shared import uniforms_field -from utils.exceptions import ProductValueError - - -def validate_excluded_subscriptions( - v: UUID, *, excluded_subscriptions: list[UUID] -) -> UUID: - if excluded_subscriptions and v in excluded_subscriptions: - raise ProductValueError( - "Subscription is in the excluded list and cannot be chosen" - ) - - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionExcludeSubscriptionsValidator(GroupedMetadata): - excluded_subscriptions: list[UUID] - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_excluded_subscriptions, - excluded_subscriptions=self.excluded_subscriptions, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field( - {"excludedSubscriptionIds": list(map(str, self.excluded_subscriptions))} - ) diff --git a/forms/validator/subscription_id.py b/forms/validator/subscription_id.py deleted file mode 100644 index b0a966d..0000000 --- a/forms/validator/subscription_id.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import random -import string -from collections.abc import Generator -from types import new_class -from typing import Annotated, Any -from uuid import UUID - -import structlog -from orchestrator.db import ( - SubscriptionTable, -) -from orchestrator.services import subscriptions -from orchestrator.types import SubscriptionLifecycle -from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler -from pydantic.json_schema import JsonSchemaValue -from pydantic_core import CoreSchema -from pydantic_forms.types import UUIDstr - -from forms.types import Tags, VisiblePortMode # move to other location -from forms.validator.shared import GetSubscriptionByIdFunc -from forms.validator.subscription_bandwidth import SubscriptionBandwidthValidator -from forms.validator.subscription_customer import SubscriptionCustomerValidator -from forms.validator.subscription_exclude_subscriptions import ( - SubscriptionExcludeSubscriptionsValidator, -) -from forms.validator.subscription_in_sync import SubscriptionInSyncValidator -from forms.validator.subscription_is_port import SubscriptionIsPortValidator -from forms.validator.subscription_port_mode import SubscriptionPortModeValidator -from forms.validator.subscription_product_id import SubscriptionProductIdValidator -from forms.validator.subscription_status import SubscriptionStatusValidator -from forms.validator.subscription_tag import SubscriptionTagValidator - -logger = structlog.get_logger(__name__) - - -class SubscriptionId: # TODO #1983 change to Annotated Type - @classmethod - def __get_pydantic_json_schema__( - cls, schema: CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - json_schema_extra = {"type": "string", "format": "subscriptionId"} - json_schema = handler(schema) - json_schema.update(json_schema_extra) - return json_schema - - @classmethod - def __get_pydantic_core_schema__( - cls, _source_type: Any, handler: GetCoreSchemaHandler - ) -> CoreSchema: - return handler(UUID) - - -def default_get_subscription(v: UUID) -> SubscriptionTable: - print("uuid v", v) - print("subscription", subscriptions.get_subscription(v, model=SubscriptionTable)) - - return subscriptions.get_subscription(v, model=SubscriptionTable) - - -def subscription_id( - product_ids: list[UUID] | None = None, - visible_port_mode: VisiblePortMode | None = None, - customer_id: UUIDstr | None = None, - customer_key: str | None = None, - bandwidth: int | None = None, - bandwidth_key: str | None = None, - allowed_tags: list[Tags] | None = None, - excluded_subscriptions: list[UUID] | None = None, - allowed_statuses: list[SubscriptionLifecycle] | None = None, - allow_out_of_sync: bool = False, - get_subscription: GetSubscriptionByIdFunc | None = None, -) -> type: - # Create type name - org_string = f"O{str(customer_id)[0:8]}" if customer_id else "" - bandwidth_str = f"B{bandwidth}" if bandwidth else "" - random_str = "".join( - random.choice(string.ascii_letters + string.digits) for _ in range(8) - ) # noqa: S311 - class_name = ( - f"SubscriptionId{visible_port_mode}{org_string}{bandwidth_str}{random_str}Value" - ) - - # TODO: caching is disabled here since it was in the wrong place. First #1983 has to be fixed - get_subscription = ( - get_subscription if get_subscription else default_get_subscription - ) - - def get_validators() -> Generator[Any, Any, None]: - yield SubscriptionStatusValidator( - allowed_statuses=allowed_statuses, get_subscription=get_subscription - ) - if product_ids: - yield SubscriptionProductIdValidator( - allowed_product_ids=product_ids, get_subscription=get_subscription - ) - if allowed_tags: - yield SubscriptionTagValidator( - allowed_product_tags=allowed_tags, get_subscription=get_subscription - ) - if visible_port_mode: - yield SubscriptionIsPortValidator(get_subscription=get_subscription) - yield SubscriptionPortModeValidator( - visible_port_mode=visible_port_mode, get_subscription=get_subscription - ) - if excluded_subscriptions: - yield SubscriptionExcludeSubscriptionsValidator( - excluded_subscriptions=excluded_subscriptions - ) - if customer_id: - print("customer_id in get_validators", customer_id) - yield SubscriptionCustomerValidator( - customer_id=customer_id, - customer_key=customer_key, - get_subscription=get_subscription, - ) - if not allow_out_of_sync: - yield SubscriptionInSyncValidator(get_subscription=get_subscription) - if bandwidth is not None: - yield SubscriptionBandwidthValidator( - bandwidth=bandwidth, - bandwidth_key=bandwidth_key, - get_subscription=get_subscription, - ) - - validator = new_class(class_name, (SubscriptionId,)) - return Annotated[validator, *get_validators()] # type: ignore diff --git a/forms/validator/subscription_in_sync.py b/forms/validator/subscription_in_sync.py deleted file mode 100644 index eb7fa19..0000000 --- a/forms/validator/subscription_in_sync.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from orchestrator.db.models import SubscriptionTable -from pydantic import AfterValidator - -from utils.exceptions import InSyncValueError - -GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] - - -def validate_in_sync(v: UUID, *, get_subscription: GetSubscriptionByIdFunc) -> UUID: - subscription = get_subscription(v) - if not subscription.insync: - raise InSyncValueError("Subscription is not in sync") - - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionInSyncValidator(GroupedMetadata): - get_subscription: GetSubscriptionByIdFunc - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial(validate_in_sync, get_subscription=self.get_subscription) - yield cast(BaseMetadata, AfterValidator(validator)) diff --git a/forms/validator/subscription_is_port.py b/forms/validator/subscription_is_port.py deleted file mode 100644 index a28ed58..0000000 --- a/forms/validator/subscription_is_port.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.validator.service_port_tags import PORT_TAGS_ALL -from forms.validator.shared import GetSubscriptionByIdFunc -from utils.exceptions import PortsValueError - - -def validate_subscription_is_port( - v: UUID, *, get_subscription: GetSubscriptionByIdFunc -) -> UUID: - subscription = get_subscription(v) - if subscription.product.tag not in PORT_TAGS_ALL: - raise PortsValueError("Not a service port subscription") - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionIsPortValidator(GroupedMetadata): - get_subscription: GetSubscriptionByIdFunc - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_subscription_is_port, get_subscription=self.get_subscription - ) - yield cast(BaseMetadata, AfterValidator(validator)) diff --git a/forms/validator/subscription_port_mode.py b/forms/validator/subscription_port_mode.py deleted file mode 100644 index 9f6f438..0000000 --- a/forms/validator/subscription_port_mode.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.types import VisiblePortMode -from forms.validator.shared import ( - GetSubscriptionByIdFunc, - PortMode, - _get_port_mode, - uniforms_field, -) -from utils.exceptions import PortsModeValueError - - -def validate_port_mode( - v: UUID, - *, - visible_port_mode: VisiblePortMode, - get_subscription: GetSubscriptionByIdFunc, -) -> UUID: - subscription = get_subscription(v) - port_mode = _get_port_mode(subscription) - - if visible_port_mode == "normal" and port_mode == PortMode.link_member: - raise PortsModeValueError("normal", "Port mode should be 'untagged' or 'tagged") - elif visible_port_mode == "link_member" and port_mode != PortMode.link_member: # noqa: RET506 - raise PortsModeValueError( - PortMode.link_member, "Port mode should be 'link_member'" - ) - elif visible_port_mode == "untagged" and port_mode != PortMode.untagged: - raise PortsModeValueError(PortMode.untagged, "Port mode should be 'untagged'") - elif visible_port_mode == "tagged" and port_mode != PortMode.tagged: - raise PortsModeValueError(PortMode.tagged, "Port mode should be 'tagged'") - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionPortModeValidator(GroupedMetadata): - visible_port_mode: VisiblePortMode - get_subscription: GetSubscriptionByIdFunc - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_port_mode, - visible_port_mode=self.visible_port_mode, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field({"visiblePortMode": self.visible_port_mode}) diff --git a/forms/validator/subscription_product_id.py b/forms/validator/subscription_product_id.py deleted file mode 100644 index 2287161..0000000 --- a/forms/validator/subscription_product_id.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field -from utils.exceptions import ProductValueError - - -def validate_product_id( - v: UUID, - *, - allowed_product_ids: list[UUID], - get_subscription: GetSubscriptionByIdFunc, -) -> UUID: - subscription = get_subscription(v) - if ( - allowed_product_ids - and subscription.product.product_id not in allowed_product_ids - ): - raise ProductValueError( - f"Subscription is not of products: {allowed_product_ids}" - ) - - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionProductIdValidator(GroupedMetadata): - allowed_product_ids: list[UUID] - get_subscription: GetSubscriptionByIdFunc - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_product_id, - allowed_product_ids=self.allowed_product_ids, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field({"productIds": list(map(str, self.allowed_product_ids))}) diff --git a/forms/validator/subscription_status.py b/forms/validator/subscription_status.py deleted file mode 100644 index 8fe17fd..0000000 --- a/forms/validator/subscription_status.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from orchestrator.types import SubscriptionLifecycle -from pydantic import AfterValidator - -from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field -from utils.exceptions import AllowedStatusValueError - - -def validate_allowed_statuses( - v: UUID, - *, - allowed_statuses: list[SubscriptionLifecycle] | None = None, - get_subscription: GetSubscriptionByIdFunc, -) -> UUID: - allowed_statuses = allowed_statuses or [SubscriptionLifecycle.ACTIVE] - subscription = get_subscription(v) - if subscription.status not in allowed_statuses: - raise AllowedStatusValueError( - f"Subscription has status {subscription.status}. Allowed statuses: {list(map(str, allowed_statuses))}" - ) - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionStatusValidator(GroupedMetadata): - get_subscription: GetSubscriptionByIdFunc - allowed_statuses: list[SubscriptionLifecycle] | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_allowed_statuses, - allowed_statuses=self.allowed_statuses, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - allowed_statuses = self.allowed_statuses or [SubscriptionLifecycle.ACTIVE] - yield uniforms_field({"statuses": list(map(str, allowed_statuses))}) diff --git a/forms/validator/subscription_tag.py b/forms/validator/subscription_tag.py deleted file mode 100644 index 610a125..0000000 --- a/forms/validator/subscription_tag.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Iterator -from dataclasses import dataclass -from functools import partial -from typing import cast -from uuid import UUID - -from annotated_types import SLOTS, BaseMetadata, GroupedMetadata -from pydantic import AfterValidator - -from forms.types import Tags -from forms.validator.shared import GetSubscriptionByIdFunc, uniforms_field -from utils.exceptions import ProductValueError - - -def validate_product_tags( - v: UUID, - *, - allowed_product_tags: list[Tags], - get_subscription: GetSubscriptionByIdFunc, -) -> UUID: - subscription = get_subscription(v) - if subscription.product.tag not in allowed_product_tags: - raise ProductValueError( - f"Subscription is of the wrong product. Allowed product tags: {allowed_product_tags}" - ) - return v - - -@dataclass(frozen=True, **SLOTS) -class SubscriptionTagValidator(GroupedMetadata): - allowed_product_tags: list[Tags] - get_subscription: GetSubscriptionByIdFunc - - def __iter__(self) -> Iterator[BaseMetadata]: - validator = partial( - validate_product_tags, - allowed_product_tags=self.allowed_product_tags, - get_subscription=self.get_subscription, - ) - yield cast(BaseMetadata, AfterValidator(validator)) - yield uniforms_field({"tags": list(map(str, self.allowed_product_tags))}) diff --git a/forms/validator/vlan_ranges.py b/forms/validator/vlan_ranges.py deleted file mode 100644 index b7abda0..0000000 --- a/forms/validator/vlan_ranges.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Annotated - -from nwastdlib.vlans import VlanRanges -from pydantic import AfterValidator, Field, TypeAdapter - -from utils.exceptions import VlanValueError - - -def validate_vlan_range(vlan_ranges: VlanRanges) -> VlanRanges: - for vlan in vlan_ranges: - if vlan == 0 or (2 <= vlan <= 4094): - continue - raise VlanValueError("VLAN range must be between 2 and 4094") - return vlan_ranges - - -_vlan_ranges_schema = TypeAdapter(VlanRanges).json_schema() - -NsiVlanRanges = Annotated[ - VlanRanges, - Field(json_schema_extra=_vlan_ranges_schema | {"uniforms": {"nsiVlansOnly": True}}), - AfterValidator(validate_vlan_range), -] diff --git a/products/product_blocks/sap.py b/products/product_blocks/sap.py index d0d4518..9ef9fc6 100644 --- a/products/product_blocks/sap.py +++ b/products/product_blocks/sap.py @@ -16,18 +16,27 @@ from orchestrator.types import SubscriptionLifecycle from pydantic import computed_field -from products.product_blocks.port import PortBlock, PortBlockInactive, PortBlockProvisioning +from products.product_blocks.port import ( + PortBlock, + PortBlockInactive, + PortBlockProvisioning, +) +from workflows.nsistp.shared.shared import CustomVlanRanges class SAPBlockInactive(ProductBlockModel, product_block_name="SAP"): port: PortBlockInactive | None = None vlan: int | None = None + # vlan: CustomVlanRanges # TODO: check where to use int or CustomVlanRanges ims_id: int | None = None -class SAPBlockProvisioning(SAPBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): +class SAPBlockProvisioning( + SAPBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] +): port: PortBlockProvisioning vlan: int + # vlan: CustomVlanRanges # TODO: check where to use int or CustomVlanRanges ims_id: int | None = None @computed_field # type: ignore[misc] @@ -38,5 +47,6 @@ def title(self) -> str: class SAPBlock(SAPBlockProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): port: PortBlock - vlan: int + # vlan: int + vlan: CustomVlanRanges | int # TODO: check where to use int or CustomVlanRanges ims_id: int diff --git a/products/product_types/nsistp.py b/products/product_types/nsistp.py index a1ea2c4..426e55a 100644 --- a/products/product_types/nsistp.py +++ b/products/product_types/nsistp.py @@ -7,6 +7,7 @@ NsistpBlockInactive, NsistpBlockProvisioning, ) +from workflows.nsistp.shared.shared import CustomVlanRanges class NsistpInactive(SubscriptionModel, is_base=True): @@ -21,3 +22,7 @@ class NsistpProvisioning( class Nsistp(NsistpProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): nsistp: NsistpBlock + + @property + def vlan_range(self) -> CustomVlanRanges: + return self.nsistp.sap.vlan diff --git a/forms/types.py b/utils/types.py similarity index 60% rename from forms/types.py rename to utils/types.py index 6d2aa05..e4ae04b 100644 --- a/forms/types.py +++ b/utils/types.py @@ -11,9 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network from typing import Literal +# TODO: remove entirely or irrelevant tags Tags = Literal[ "SP", "SPNL", @@ -42,25 +42,3 @@ "Wireless", "OS", ] - -# IMSStatus = Literal["RFS", "PL", "IS", "MI", "RFC", "OOS"] -TransitionType = Literal["speed", "upgrade", "downgrade", "replace"] -VisiblePortMode = Literal["all", "normal", "tagged", "untagged", "link_member"] - - -# class MailAddress(TypedDict): -# email: EmailStr -# name: str - - -# class ConfirmationMail(TypedDict): -# message: str -# subject: str -# language: str -# to: list[MailAddress] -# cc: list[MailAddress] -# bcc: list[MailAddress] - - -IPAddress = IPv4Address | IPv6Address -IPNetwork = IPv4Network | IPv6Network diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 8b891ae..68e839d 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -2,6 +2,8 @@ # from typing import TypeAlias, cast +from typing import Annotated + import structlog from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage @@ -11,7 +13,7 @@ from orchestrator.workflow import StepList, begin, step from orchestrator.workflows.steps import store_process_subscription from orchestrator.workflows.utils import create_workflow -from pydantic import ConfigDict +from pydantic import AfterValidator, ConfigDict, model_validator from pydantic_forms.types import FormGenerator, State, UUIDstr from products.product_types.nsistp import NsistpInactive, NsistpProvisioning @@ -24,8 +26,14 @@ StpId, Topology, nsistp_fill_sap, + ports_selector, + validate_both_aliases_empty_or_not, +) +from workflows.nsistp.shared.vlan import ( + CustomVlanRanges, + validate_vlan, + validate_vlan_not_in_use, ) -from workflows.nsistp.shared.helpers import FormNsistpPort from workflows.shared import create_summary_form @@ -44,7 +52,7 @@ def subscription_description(subscription: SubscriptionModel) -> str: def initial_input_form_generator(product_name: str) -> FormGenerator: # TODO add additional fields to form if needed - class CreateNsistpForm(FormPage): + class CreateNsiStpForm(FormPage): model_config = ConfigDict(title=product_name) # TODO: check whether this should be removed @@ -54,7 +62,15 @@ class CreateNsistpForm(FormPage): divider_1: Divider # TODO: could this be multiple service ports?? - service_port: FormNsistpPort + subscription_id: Annotated[ + UUIDstr, + ports_selector(), + ] + vlan: Annotated[ + CustomVlanRanges, + AfterValidator(validate_vlan), + AfterValidator(validate_vlan_not_in_use), + ] topology: Topology stp_id: StpId @@ -64,12 +80,18 @@ class CreateNsistpForm(FormPage): expose_in_topology: bool = True bandwidth: ServiceSpeed - user_input = yield CreateNsistpForm + @model_validator(mode="after") + def validate_is_alias_in_out(self) -> "CreateNsiStpForm": + validate_both_aliases_empty_or_not(self.is_alias_in, self.is_alias_out) + return self + + user_input = yield CreateNsiStpForm user_input_dict = user_input.dict() summary_fields = [ + "subscription_id", + "vlan", "topology", - # "vlan", "stp_id", "stp_description", "is_alias_in", @@ -86,7 +108,8 @@ class CreateNsistpForm(FormPage): def construct_nsistp_model( product: UUIDstr, customer_id: UUIDstr, - service_port: list[dict], + subscription_id, + vlan, topology: str, stp_id: str, stp_description: str | None, @@ -95,7 +118,6 @@ def construct_nsistp_model( expose_in_topology: bool | None, bandwidth: int | None, ) -> State: - print("service Port in construct", service_port) nsistp = NsistpInactive.from_product_id( product_id=product, customer_id=customer_id, @@ -109,7 +131,10 @@ def construct_nsistp_model( nsistp.nsistp.expose_in_topology = expose_in_topology nsistp.nsistp.bandwidth = bandwidth - nsistp_fill_sap(nsistp, service_port) + # TODO: change to support CustomVlanRanges + vlan_int = int(vlan) if not isinstance(vlan, int) else vlan + + nsistp.nsistp.sap = nsistp_fill_sap(subscription_id, vlan_int) nsistp = NsistpProvisioning.from_other_lifecycle( nsistp, SubscriptionLifecycle.PROVISIONING diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index ec0c952..a6134f8 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -14,34 +14,33 @@ from collections.abc import Iterator from datetime import datetime from functools import partial -from typing import Annotated +from typing import Annotated, Any from uuid import UUID -from annotated_types import Ge, Le -from more_itertools import one +from annotated_types import BaseMetadata, Ge, Le from orchestrator.db import ProductTable, db from orchestrator.db.models import ( SubscriptionTable, ) from orchestrator.domain.base import SubscriptionModel from orchestrator.types import SubscriptionLifecycle -from pydantic import AfterValidator, ValidationInfo -from pydantic_forms.types import State -from pydantic_forms.validators import Choice, choice_list +from pydantic import AfterValidator, Field, ValidationInfo +from pydantic_forms.types import UUIDstr +from pydantic_forms.validators import Choice from sqlalchemy import select from typing_extensions import Doc -from forms.validator.service_port import service_port -from forms.validator.service_port_tags import ( - PORT_TAG_GENERAL, -) -from forms.validator.shared import MAX_SPEED_POSSIBLE from products.product_blocks.port import PortMode -from products.product_types.nsistp import Nsistp, NsistpInactive -from utils.exceptions import DuplicateValueError, FieldValueError -from workflows.shared import ( - subscriptions_by_product_type_and_instance_value, +from products.product_blocks.sap import SAPBlockInactive +from products.product_types.nsistp import Nsistp +from utils.exceptions import ( + DuplicateValueError, + FieldValueError, +) +from workflows.nsistp.shared.shared import ( + MAX_SPEED_POSSIBLE, ) +from workflows.shared import subscriptions_by_product_type_and_instance_value TOPOLOGY_REGEX = r"^[-a-z0-9+,.;=_]+$" STP_ID_REGEX = r"^[-a-z0-9+,.;=_:]+$" @@ -49,13 +48,20 @@ FQDN_REQEX = r"^(?!.{255}|.{253}[^.])([a-z0-9](?:[-a-z-0-9]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?[.]?$" -def nsistp_service_port(current: list[State] | None = None) -> type: - print("current", current) - return service_port( - visible_port_mode="tagged", - allowed_tags=PORT_TAG_GENERAL, - current=current, +def ports_selector() -> type[list[Choice]]: + port_subscriptions = subscriptions_by_product_type_and_instance_value( + "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] ) + ports = { + str(subscription.subscription_id): subscription.description + for subscription in sorted( + port_subscriptions, key=lambda port: port.description + ) + } + + print("Port choices found:", ports) # --- IGNORE --- + + return Choice("ServicePort", zip(ports.keys(), ports.items())) def is_fqdn(hostname: str) -> bool: @@ -159,13 +165,6 @@ def is_not_unique(nsistp: Nsistp) -> bool: return stp_id -StpId = Annotated[ - str, - AfterValidator(partial(validate_regex, STP_ID_REGEX, "STP identifier")), - Doc("must be unique along the set of NSISTP's in the same TOPOLOGY"), -] - - def validate_both_aliases_empty_or_not( is_alias_in: str | None, is_alias_out: str | None ) -> None: @@ -184,20 +183,41 @@ def validate_nurn(nurn: str | None) -> str | None: return nurn -def nsistp_fill_sap(subscription: NsistpInactive, service_ports: list[dict]) -> None: - print("nsi_fill_sap", service_ports) - sp = one(service_ports) - subscription.nsistp.sap.vlan = sp["vlan"] - # SubscriptionModel can be any type of ServicePort - subscription.nsistp.sap.port = SubscriptionModel.from_subscription( - sp["port_id"] - ).port # type: ignore +def nsistp_fill_sap(subscription_id: UUIDstr, vlan: int) -> SAPBlockInactive: + sap = SAPBlockInactive.new(subscription_id=subscription_id) + sap.port = SubscriptionModel.from_subscription(subscription_id).port # type: ignore + sap.vlan = vlan + return sap -IsAlias = Annotated[ +# def nsistp_fill_sap( +# subscription: NsistpInactive, subscription_id: UUIDstr, vlan: CustomVlanRanges +# ) -> None: +# subscription.nsistp.sap.vlan = vlan +# # SubscriptionModel can be any type of ServicePort +# subscription.nsistp.sap.port = SubscriptionModel.from_subscription( +# subscription_id +# ).port # type: ignore + + +def merge_uniforms(schema: dict[str, Any], *, to_merge: dict[str, Any]) -> None: + schema["uniforms"] = schema.get("uniforms", {}) | to_merge + + +def uniforms_field(to_merge: dict[str, Any]) -> BaseMetadata: + return Field(json_schema_extra=partial(merge_uniforms, to_merge=to_merge)) + + +Topology = Annotated[ str, - AfterValidator(validate_nurn), - Doc("ISALIAS conform https://www.ogf.org/documents/GFD.202.pdf"), + AfterValidator(partial(validate_regex, TOPOLOGY_REGEX, "Topology")), + Doc("topology string may only consist of characters from the set [-a-z+,.;=_]"), +] + +StpId = Annotated[ + str, + AfterValidator(partial(validate_regex, STP_ID_REGEX, "STP identifier")), + Doc("must be unique along the set of NSISTP's in the same TOPOLOGY"), ] StpDescription = Annotated[ @@ -206,13 +226,12 @@ def nsistp_fill_sap(subscription: NsistpInactive, service_ports: list[dict]) -> Doc("STP description may not contain characters from the set [<>&]"), ] -Topology = Annotated[ +IsAlias = Annotated[ str, - AfterValidator(partial(validate_regex, TOPOLOGY_REGEX, "Topology")), - Doc("topology string may only consist of characters from the set [-a-z+,.;=_]"), + AfterValidator(validate_nurn), + Doc("ISALIAS conform https://www.ogf.org/documents/GFD.202.pdf"), ] - Bandwidth = Annotated[ int, Ge(1), @@ -221,22 +240,3 @@ def nsistp_fill_sap(subscription: NsistpInactive, service_ports: list[dict]) -> ] ServiceSpeed = Bandwidth - - -# NOTE: currently in helpers.py -def ports_selector() -> type[list[Choice]]: - port_subscriptions = subscriptions_by_product_type_and_instance_value( - "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] - ) - ports = { - str(subscription.subscription_id): subscription.description - for subscription in sorted( - port_subscriptions, key=lambda port: port.description - ) - } - return choice_list( - Choice("PortsEnum", zip(ports.keys(), ports.items())), # type: ignore - unique_items=True, - min_items=1, - max_items=1, - ) diff --git a/workflows/nsistp/shared/helpers.py b/workflows/nsistp/shared/helpers.py deleted file mode 100644 index 714dd6e..0000000 --- a/workflows/nsistp/shared/helpers.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import structlog -from nwastdlib.vlans import VlanRanges -from orchestrator.db import ( - SubscriptionTable, -) -from orchestrator.services import subscriptions -from orchestrator.types import SubscriptionLifecycle -from pydantic import BaseModel, GetJsonSchemaHandler, TypeAdapter, field_validator -from pydantic.json_schema import JsonSchemaValue -from pydantic_core import CoreSchema -from pydantic_core.core_schema import ValidationInfo -from pydantic_forms.validators import Choice - -from forms.validator.shared import ( - _get_port_mode, -) -from products.product_blocks.port import PortMode -from utils.exceptions import VlanValueError -from workflows.shared import subscriptions_by_product_type_and_instance_value - -logger = structlog.get_logger(__name__) - - -# Custom VlanRanges needed to avoid matching conflict with SURF orchestrator-ui components -class CustomVlanRanges(VlanRanges): - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - parent_schema = super().__get_pydantic_json_schema__(core_schema_, handler) - parent_schema["format"] = "custom-vlan" - - return parent_schema - - -# Add this after your CustomVlanRanges class definition -adapter = TypeAdapter(CustomVlanRanges) -print("CustomVlanRanges schema:", adapter.json_schema()) - - -def ports_selector() -> type[list[Choice]]: - port_subscriptions = subscriptions_by_product_type_and_instance_value( - "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] - ) - ports = { - str(subscription.subscription_id): subscription.description - for subscription in sorted( - port_subscriptions, key=lambda port: port.description - ) - } - - return Choice("ServicePort", zip(ports.keys(), ports.items())) - - -class FormNsistpPort(BaseModel): - # NOTE: subscription_id and vlan are pydantic_forms fields which render ui components - port_id: ports_selector() # type: ignore # noqa: F821 - vlan: CustomVlanRanges - - def __repr__(self) -> str: - # Help distinguish this from the ServicePort product type.. - return f"FormServicePort({self.port_id=}, {self.vlan=})" - - @field_validator("vlan") - @classmethod - def check_vlan( - cls, vlan: CustomVlanRanges, info: ValidationInfo - ) -> CustomVlanRanges: - print("hallo_vlan", vlan) - # We assume an empty string is untagged and thus 0 - if not vlan: - vlan = CustomVlanRanges(0) - - subscription_id = info.data.get("port_id") - print("port:", subscription_id) - if not subscription_id: - return vlan - - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) - - port_mode = _get_port_mode(subscription) - print("port_mode:", port_mode) - - if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): - raise VlanValueError( - f"{port_mode} {subscription.product.tag} must have a vlan" - ) - elif port_mode == PortMode.UNTAGGED and vlan != CustomVlanRanges(0): # noqa: RET506 - raise VlanValueError( - f"{port_mode} {subscription.product.tag} can not have a vlan" - ) - - return vlan diff --git a/workflows/nsistp/shared/nsistp.py b/workflows/nsistp/shared/nsistp_services.py similarity index 70% rename from workflows/nsistp/shared/nsistp.py rename to workflows/nsistp/shared/nsistp_services.py index c0a50a2..da1f9b7 100644 --- a/workflows/nsistp/shared/nsistp.py +++ b/workflows/nsistp/shared/nsistp_services.py @@ -14,7 +14,6 @@ from uuid import UUID from more_itertools import flatten -from nwastdlib.vlans import VlanRanges from orchestrator.db import db from orchestrator.db.models import ( ProductTable, @@ -26,8 +25,8 @@ from sqlalchemy import select from sqlalchemy.orm import aliased -# from surf.products.product_types.nsi_lp import NsiLightPath from products.product_types.nsistp import Nsistp +from workflows.nsistp.shared.shared import CustomVlanRanges def _get_subscriptions_inuseby_port_id( @@ -63,42 +62,20 @@ def nsistp_get_by_port_id(port_id: UUID) -> list[Nsistp]: SubscriptionLifecycle.PROVISIONING, SubscriptionLifecycle.MIGRATING, ] - result = _get_subscriptions_inuseby_port_id(port_id, "NSISTP", statuses) + result = _get_subscriptions_inuseby_port_id(port_id, "Nsistp", statuses) return [Nsistp.from_subscription(id) for id in list(set(result))] -# def nsi_lp_get_by_port_id(port_id: UUID) -> list[NsiLightPath]: -# """Get NsiLightPaths by service port id. - -# Args: -# port_id: ID of the service port for which you want all NsiLightPaths of. -# """ -# statuses = [SubscriptionLifecycle.ACTIVE] -# result = _get_subscriptions_inuseby_port_id(port_id, "NSILP", statuses) - -# return [NsiLightPath.from_subscription(id) for id in list(set(result))] - - -def get_available_vlans_by_port_id(port_id: UUID) -> VlanRanges: +def get_available_vlans_by_port_id(port_id: UUID) -> CustomVlanRanges: """Get available vlans by service port id. This will get all NSISTPs and adds their vlan ranges to a single VlanRanges to get the available vlans by nsistps. - Then filters out the vlans that are already in use by NSI light paths and returns the available vlans. - Args: port_id: ID of the service port to find available vlans. """ nsistps = nsistp_get_by_port_id(port_id) - available_vlans = VlanRanges(flatten(nsistp.vlan_range for nsistp in nsistps)) - - # NOTE: Lightpad can be ommited? - # nsi_lps = nsi_lp_get_by_port_id(port_id) - # used_vlans = VlanRanges( - # flatten(sap.vlanrange for nsi_lp in nsi_lps for sap in nsi_lp.vc.saps) - # ) - - # return available_vlans - used_vlans + available_vlans = CustomVlanRanges(flatten(nsistp.vlan_range for nsistp in nsistps)) return available_vlans diff --git a/workflows/nsistp/shared/shared.py b/workflows/nsistp/shared/shared.py new file mode 100644 index 0000000..592d927 --- /dev/null +++ b/workflows/nsistp/shared/shared.py @@ -0,0 +1,67 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Callable +from enum import StrEnum +from uuid import UUID + +import structlog +from nwastdlib.vlans import VlanRanges +from orchestrator.db import ( + SubscriptionTable, +) +from pydantic import GetJsonSchemaHandler +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema + +from utils.types import Tags + +logger = structlog.get_logger(__name__) + +GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] + +PORT_SPEED = "port_speed" +MAX_SPEED_POSSIBLE = 400_000 + + +class PortTag(StrEnum): + SP = "SP" + SPNL = "SPNL" + AGGSP = "AGGSP" + AGGSPNL = "AGGSPNL" + MSC = "MSC" + MSCNL = "MSCNL" + IRBSP = "IRBSP" + + +PORT_TAG_GENERAL: list[Tags] = ["PORT"] + +# TODO: these tags can probably be removed +PORT_TAGS_AGGSP: list[Tags] = ["AGGSP", "AGGSPNL"] +PORT_TAGS_IRBSP: list[Tags] = ["IRBSP"] +PORT_TAGS_MSC: list[Tags] = ["MSC", "MSCNL"] +PORT_TAGS_SP: list[Tags] = ["SP"] +PORT_TAGS_ALL: list[Tags] = ( + PORT_TAGS_SP + PORT_TAGS_AGGSP + PORT_TAGS_MSC + PORT_TAGS_IRBSP + PORT_TAG_GENERAL +) + + +# Custom VlanRanges needed to avoid matching conflict with SURF orchestrator-ui components +class CustomVlanRanges(VlanRanges): + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + parent_schema = super().__get_pydantic_json_schema__(core_schema_, handler) + parent_schema["format"] = "custom-vlan" + + return parent_schema diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py new file mode 100644 index 0000000..17e622c --- /dev/null +++ b/workflows/nsistp/shared/vlan.py @@ -0,0 +1,232 @@ +# Copyright 2019-2024 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +from uuid import UUID + +import structlog +from more_itertools import first, flatten +from orchestrator.db import ( + SubscriptionTable, +) +from orchestrator.services import subscriptions +from pydantic_core.core_schema import ValidationInfo +from pydantic_forms.types import State + +from products.product_blocks.port import PortMode +from utils.exceptions import PortsValueError, VlanValueError +from workflows.nsistp.shared.nsistp_services import get_available_vlans_by_port_id +from workflows.nsistp.shared.shared import CustomVlanRanges, PortTag + +logger = structlog.get_logger(__name__) + +# def get_vlans_by_ims_circuit_id(ims_circuit_id: int) -> list[VlanRange]: +# return VlansApi(ims_api_client).vlans_by_msp(ims_circuit_id) + + +# def _clean_vlan_ranges(vlans: list[VlanRange]) -> list[list[int]]: +# """Return a list of VlanRange objects that each has a start and end integer property. + +# It flattens this list to a list of numbers to dedupe and potentially apply a filter (default to True/all). It +# groups that list by adjacency creating a new list of range start/ends - e.g. [[2,4],[5,19]] the (monadic/Right) +# return value of this function. + +# Args: +# vlans: a list of VlanRange objects + +# >>> import collections +# >>> VlanRange = collections.namedtuple("VlanRange", ["start", "end"]) +# >>> _clean_vlan_ranges([VlanRange(1,4), VlanRange(3,3), VlanRange(5, 5), VlanRange(7, 10), VlanRange(12, 12)]) +# [[1, 5], [7, 10], [12]] + +# """ +# numbers: list[int] = expand_ranges( +# [(vlan.start, vlan.end) for vlan in vlans], inclusive=True +# ) +# sorted_numbers: Iterable[int] = sorted(set(numbers)) +# grouped_by = ( +# list(x) for _, x in groupby(sorted_numbers, lambda x, c=count(): next(c) - x) +# ) # type: ignore # noqa: B008 +# return [ +# [element[0], element[-1]] if element[0] != element[-1] else [element[0]] +# for element in grouped_by +# ] + + +def get_port_mode(subscription: SubscriptionTable) -> PortMode: + if subscription.product.tag in [PortTag.AGGSP + PortTag.SP]: + return subscription.port_mode + return PortMode.TAGGED + + +# TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) +def get_vlans_by_subscription_id(subscription_id: UUID) -> list[list[int]]: + values = subscriptions.find_values_for_resource_types( + subscription_id, ["vlan"], strict=False + ) + print("Values from vlan resource type:", values) + vlan = first(values["vlan"], default=[]) # Provide empty list as default + print("First vlan value:", vlan) + return vlan + # return _clean_vlan_ranges(get_vlans_by_ims_circuit_id(int(ims_circuit_id))) + + +def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRanges: + # We assume an empty string is untagged and thus 0 + if not vlan: + vlan = CustomVlanRanges(0) + + subscription_id = info.data.get("port_id") + if not subscription_id: + return vlan + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + port_mode = get_port_mode(subscription) + + if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): + raise VlanValueError(f"{port_mode} {subscription.product.tag} must have a vlan") + elif port_mode == PortMode.UNTAGGED and vlan != CustomVlanRanges(0): # noqa: RET506 + raise VlanValueError( + f"{port_mode} {subscription.product.tag} can not have a vlan" + ) + + return vlan + + +def check_vlan_in_use( + current: list[State], v: CustomVlanRanges, info: ValidationInfo +) -> CustomVlanRanges: + """Check if vlan value is already in use by service port. + + Args: + current: List of current form states, used to filter out self from used vlans. + v: Vlan range of the form input. + info: validation info, contains other fields in info.data + + 1. Get all used vlans in a service port. + 2. Get all nsi reserved vlans and add them to the used_vlans. + 3. Filter out vlans used in current subscription from used_vlans. + 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. + 5. checks if input value uses already used vlans. errors if true. + 6. return input value. + + + """ + if not (subscription_id := info.data.get("subscription_id")): + return v + + # TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) + vlans = get_vlans_by_subscription_id(subscription_id) + used_vlans = [vlan.vid for vlan in vlans if vlan is not None] + nsistp_reserved_vlans = get_available_vlans_by_port_id(subscription_id) + used_vlans = CustomVlanRanges( + flatten([list(used_vlans), list(nsistp_reserved_vlans)]) + ) + + # Remove currently chosen vlans for this port to prevent tripping on in used by itself + current_selected_vlan_ranges: list[str] = [] + if current: + current_selected_service_port = filter( + lambda c: str(c["subscription_id"]) == str(subscription_id), current + ) + current_selected_vlans = list( + map(operator.itemgetter("vlan"), current_selected_service_port) + ) + for current_selected_vlan in current_selected_vlans: + # We assume an empty string is untagged and thus 0 + if not current_selected_vlan: + current_selected_vlan = "0" + + current_selected_vlan_range = CustomVlanRanges(current_selected_vlan) + used_vlans -= current_selected_vlan_range + current_selected_vlan_ranges = [ + *current_selected_vlan_ranges, + *list(current_selected_vlan_range), + ] + + # TODO (#1842): probably better to have a separate type/validator for this + # if nsi_vlans_only: + # vlan_list = list(v) + # invalid_ranges = [ + # vlan + # for vlan in vlan_list + # if vlan not in list(nsistp_reserved_vlans) + # and vlan not in current_selected_vlan_ranges + # ] + # used_vlans -= nsistp_reserved_vlans + + # if invalid_ranges: + # raise VlanValueError( + # f"Vlan(s) {CustomVlanRanges(invalid_ranges)} not valid nsi vlan range" + # ) + + # logger.info( + # "Validation info for current chosen vlans vs vlan already in use", + # current=current, + # used_vlans=used_vlans, + # subscription_id=subscription_id, + # ) + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + if v & used_vlans: + port_mode = get_port_mode(subscription) + + # for tagged only; for link_member/untagged say "SP already in use" + if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: + raise PortsValueError("Port already in use") + raise VlanValueError(f"Vlan(s) {used_vlans} already in use") + + return v + + +def validate_vlan_not_in_use( + v: CustomVlanRanges, info: ValidationInfo +) -> CustomVlanRanges: + """Wrapper for check_vlan_in_use to work with AfterValidator.""" + # For single form validation, we don't have a 'current' list, so pass empty list + current: list[State] = [] + return check_vlan_in_use(current, v, info) + + +def parse_vlan_ranges_to_list(vlan_string: str) -> list[int]: + """Convert VLAN range string to list of integers. + + Args: + vlan_string: String like "3,5-6,10-12" or "3,5,6" + + Returns: + List of integers: [3, 5, 6, 10, 11, 12] + """ + if not vlan_string or vlan_string == "0": + return [0] + + vlans = [] + parts = vlan_string.split(",") + + for part in parts: + part = part.strip() + if "-" in part: + # Handle range like "5-6" + start, end = map(int, part.split("-")) + vlans.extend(range(start, end + 1)) + else: + # Handle single VLAN + vlans.append(int(part)) + + return sorted(set(vlans)) From 780b5b5de4d8d578988017bb3b6bf4cf45a1005e Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 13:29:25 +0200 Subject: [PATCH 04/23] Selecting of vlans works --- workflows/nsistp/shared/vlan.py | 178 +++++++++++++++++++++++++++++++- workflows/shared.py | 14 +++ 2 files changed, 187 insertions(+), 5 deletions(-) diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index 17e622c..596a54a 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -12,16 +12,23 @@ # limitations under the License. import operator +from collections.abc import Sequence from uuid import UUID import structlog from more_itertools import first, flatten from orchestrator.db import ( + ResourceTypeTable, + SubscriptionInstanceRelationTable, + SubscriptionInstanceTable, + SubscriptionInstanceValueTable, SubscriptionTable, + db, ) from orchestrator.services import subscriptions from pydantic_core.core_schema import ValidationInfo -from pydantic_forms.types import State +from pydantic_forms.types import State, UUIDstr +from sqlalchemy import select from products.product_blocks.port import PortMode from utils.exceptions import PortsValueError, VlanValueError @@ -72,12 +79,22 @@ def get_port_mode(subscription: SubscriptionTable) -> PortMode: # TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) def get_vlans_by_subscription_id(subscription_id: UUID) -> list[list[int]]: values = subscriptions.find_values_for_resource_types( - subscription_id, ["vlan"], strict=False + subscription_id, ["vlan"], strict=True ) - print("Values from vlan resource type:", values) + # print("Values from vlan resource type:", values) vlan = first(values["vlan"], default=[]) # Provide empty list as default - print("First vlan value:", vlan) + # print("First vlan value:", vlan) + return vlan + + # values = subscriptions.find_values_for_resource_types( + # subscription_id, ["sap"], strict=True + # ) + # print("Values from sap resource type:", values) + # sap = first(values["vlan", "sap"]) # Provide empty list as default + # print("First sap value:", sap) + + # return sap # return _clean_vlan_ranges(get_vlans_by_ims_circuit_id(int(ims_circuit_id))) @@ -201,7 +218,8 @@ def validate_vlan_not_in_use( """Wrapper for check_vlan_in_use to work with AfterValidator.""" # For single form validation, we don't have a 'current' list, so pass empty list current: list[State] = [] - return check_vlan_in_use(current, v, info) + return check_vlan_not_used(current, v, info) + # return check_vlan_in_use(current, v, info) def parse_vlan_ranges_to_list(vlan_string: str) -> list[int]: @@ -230,3 +248,153 @@ def parse_vlan_ranges_to_list(vlan_string: str) -> list[int]: vlans.append(int(part)) return sorted(set(vlans)) + + +def check_vlan_not_used( + current: list[State], v: CustomVlanRanges, info: ValidationInfo +) -> CustomVlanRanges: + """Check if vlan value is already in use by service port. + + Args: + current: List of current form states, used to filter out self from used vlans. + v: Vlan range of the form input. + info: validation info, contains other fields in info.data + + 1. Get all used vlans in a service port. + 2. Get all nsi reserved vlans and add them to the used_vlans. + 3. Filter out vlans used in current subscription from used_vlans. + 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. + 5. checks if input value uses already used vlans. errors if true. + 6. return input value. + + + """ + if not (subscription_id := info.data.get("subscription_id")): + return v + + # TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) + # vlans = get_vlans_by_subscription_id(subscription_id) + used_vlans = find_allocated_vlans(subscription_id, ["vlan"], strict=False) + + # Remove currently chosen vlans for this port to prevent tripping on in used by itself + current_selected_vlan_ranges: list[str] = [] + if current: + current_selected_service_port = filter( + lambda c: str(c["subscription_id"]) == str(subscription_id), current + ) + current_selected_vlans = list( + map(operator.itemgetter("vlan"), current_selected_service_port) + ) + for current_selected_vlan in current_selected_vlans: + # We assume an empty string is untagged and thus 0 + if not current_selected_vlan: + current_selected_vlan = "0" + + current_selected_vlan_range = CustomVlanRanges(current_selected_vlan) + used_vlans -= current_selected_vlan_range + current_selected_vlan_ranges = [ + *current_selected_vlan_ranges, + *list(current_selected_vlan_range), + ] + + subscription = subscriptions.get_subscription( + subscription_id, model=SubscriptionTable + ) + + print("validating vlans:", v) + print("used vlans:", used_vlans) + + if v & used_vlans: + port_mode = get_port_mode(subscription) + + # for tagged only; for link_member/untagged say "SP already in use" + if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: + raise PortsValueError("Port already in use") + raise VlanValueError(f"Vlan(s) {used_vlans} already in use") + + return v + + +def find_allocated_vlans( + subscription_id: UUID | UUIDstr, resource_types: Sequence[str], strict: bool = False +) -> dict[str, list[str]]: + """Find all vlans already allocated to a SAP for a given port.""" + # the `order_by` on `subscription_instance_id` is there to guarantee the matched ordering across resource_types + # (see also docstring) + print("subscription_id in allocated vlans:", subscription_id) + print("resource_types in allocated vlans:", resource_types) + + subscription_instance_id = db.session.execute( + select(SubscriptionInstanceTable.subscription_instance_id).filter( + SubscriptionInstanceTable.subscription_id == subscription_id + ) + ).scalar_one_or_none() + + print("subscription_instance_id:", subscription_instance_id) + + vlan_resource_type = db.session.execute( + select(ResourceTypeTable).filter( + ResourceTypeTable.resource_type == "vlan", + ) + ).scalar_one_or_none() + + vlan_ids = ( + db.session.execute( + select(SubscriptionInstanceValueTable.subscription_instance_id).filter( + SubscriptionInstanceValueTable.resource_type_id + == vlan_resource_type.resource_type_id, + ) + ) + .scalars() + .all() + ) + + # vlan_ids = [vlan.subscription_instance_id for vlan in vlans] + + print("Found vlan IDs:", vlan_ids) + print("Number of vlan IDs:", len(vlan_ids)) + + # Check for VLAN IDs that are in use by this subscription instance + vlan_relations_query = ( + select(SubscriptionInstanceRelationTable.in_use_by_id) + .join( + SubscriptionInstanceTable, + SubscriptionInstanceRelationTable.depends_on_id + == SubscriptionInstanceTable.subscription_instance_id, + ) + .filter( + SubscriptionInstanceTable.subscription_instance_id + == subscription_instance_id + ) + ) + + vlan_ids_in_use = db.session.execute(vlan_relations_query).scalars().all() + print("VLAN IDs in use by this subscription:", vlan_ids_in_use) + print("Number of VLAN IDs in use:", len(vlan_ids_in_use)) + + # Get VLAN values for the subscription instances that are in use + if vlan_ids_in_use: + vlan_values_query = ( + select(SubscriptionInstanceValueTable.value) + .join( + ResourceTypeTable, + SubscriptionInstanceValueTable.resource_type_id + == ResourceTypeTable.resource_type_id, + ) + .filter( + SubscriptionInstanceValueTable.subscription_instance_id.in_( + vlan_ids_in_use + ), + ResourceTypeTable.resource_type == "vlan", + ) + ) + + used_vlan_values = db.session.execute(vlan_values_query).scalars().all() + print("Used VLAN values:", used_vlan_values) + used_vlan_values_int = {int(vlan) for vlan in used_vlan_values} + return used_vlan_values_int + + else: + used_vlan_values = [] + print("No VLAN values in use found.") + return used_vlan_values diff --git a/workflows/shared.py b/workflows/shared.py index 5c8dff4..2731e4e 100644 --- a/workflows/shared.py +++ b/workflows/shared.py @@ -92,6 +92,20 @@ def subscriptions_by_product_type_and_instance_value( Returns: Subscription or None """ + + query = ( + SubscriptionTable.query.join(ProductTable) + .join(SubscriptionInstanceTable) + .join(SubscriptionInstanceValueTable) + .join(ResourceTypeTable) + .filter(ProductTable.product_type == product_type) + .filter(SubscriptionInstanceValueTable.value == value) + .filter(ResourceTypeTable.resource_type == resource_type) + .filter(SubscriptionTable.status.in_(status)) + ) + + print("subscription HELLO", query.all()) + return ( SubscriptionTable.query.join(ProductTable) .join(SubscriptionInstanceTable) From ff2a9b5e3194c0ddaa79fff1921426e19cf585c8 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 14:05:50 +0200 Subject: [PATCH 05/23] Using CustomRanges --- workflows/nsistp/create_nsistp.py | 2 +- workflows/nsistp/shared/forms.py | 26 +-- workflows/nsistp/shared/shared.py | 8 + workflows/nsistp/shared/vlan.py | 297 ++++-------------------------- 4 files changed, 54 insertions(+), 279 deletions(-) diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 68e839d..63d44dd 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -134,7 +134,7 @@ def construct_nsistp_model( # TODO: change to support CustomVlanRanges vlan_int = int(vlan) if not isinstance(vlan, int) else vlan - nsistp.nsistp.sap = nsistp_fill_sap(subscription_id, vlan_int) + nsistp_fill_sap(nsistp, subscription_id, vlan_int) nsistp = NsistpProvisioning.from_other_lifecycle( nsistp, SubscriptionLifecycle.PROVISIONING diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index a6134f8..fb0cd69 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -31,14 +31,14 @@ from typing_extensions import Doc from products.product_blocks.port import PortMode -from products.product_blocks.sap import SAPBlockInactive -from products.product_types.nsistp import Nsistp +from products.product_types.nsistp import Nsistp, NsistpInactive from utils.exceptions import ( DuplicateValueError, FieldValueError, ) from workflows.nsistp.shared.shared import ( MAX_SPEED_POSSIBLE, + CustomVlanRanges, ) from workflows.shared import subscriptions_by_product_type_and_instance_value @@ -183,21 +183,13 @@ def validate_nurn(nurn: str | None) -> str | None: return nurn -def nsistp_fill_sap(subscription_id: UUIDstr, vlan: int) -> SAPBlockInactive: - sap = SAPBlockInactive.new(subscription_id=subscription_id) - sap.port = SubscriptionModel.from_subscription(subscription_id).port # type: ignore - sap.vlan = vlan - return sap - - -# def nsistp_fill_sap( -# subscription: NsistpInactive, subscription_id: UUIDstr, vlan: CustomVlanRanges -# ) -> None: -# subscription.nsistp.sap.vlan = vlan -# # SubscriptionModel can be any type of ServicePort -# subscription.nsistp.sap.port = SubscriptionModel.from_subscription( -# subscription_id -# ).port # type: ignore +def nsistp_fill_sap( + subscription: NsistpInactive, subscription_id: UUIDstr, vlan: CustomVlanRanges | int +) -> None: + subscription.nsistp.sap.vlan = vlan + subscription.nsistp.sap.port = SubscriptionModel.from_subscription( + subscription_id + ).port # type: ignore def merge_uniforms(schema: dict[str, Any], *, to_merge: dict[str, Any]) -> None: diff --git a/workflows/nsistp/shared/shared.py b/workflows/nsistp/shared/shared.py index 592d927..0d92dd1 100644 --- a/workflows/nsistp/shared/shared.py +++ b/workflows/nsistp/shared/shared.py @@ -57,6 +57,14 @@ class PortTag(StrEnum): # Custom VlanRanges needed to avoid matching conflict with SURF orchestrator-ui components class CustomVlanRanges(VlanRanges): + def __str__(self) -> str: + # `range` objects have an exclusive `stop`. VlanRanges is expressed using terms that use an inclusive stop, + # which is one less then the exclusive one we use for the internal representation. Hence the `-1` + return ", ".join( + str(vr.start) if len(vr) == 1 else f"{vr.start}-{vr.stop - 1}" + for vr in self._vlan_ranges + ) + @classmethod def __get_pydantic_json_schema__( cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index 596a54a..62ba941 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -16,7 +16,6 @@ from uuid import UUID import structlog -from more_itertools import first, flatten from orchestrator.db import ( ResourceTypeTable, SubscriptionInstanceRelationTable, @@ -32,72 +31,17 @@ from products.product_blocks.port import PortMode from utils.exceptions import PortsValueError, VlanValueError -from workflows.nsistp.shared.nsistp_services import get_available_vlans_by_port_id from workflows.nsistp.shared.shared import CustomVlanRanges, PortTag logger = structlog.get_logger(__name__) -# def get_vlans_by_ims_circuit_id(ims_circuit_id: int) -> list[VlanRange]: -# return VlansApi(ims_api_client).vlans_by_msp(ims_circuit_id) - -# def _clean_vlan_ranges(vlans: list[VlanRange]) -> list[list[int]]: -# """Return a list of VlanRange objects that each has a start and end integer property. - -# It flattens this list to a list of numbers to dedupe and potentially apply a filter (default to True/all). It -# groups that list by adjacency creating a new list of range start/ends - e.g. [[2,4],[5,19]] the (monadic/Right) -# return value of this function. - -# Args: -# vlans: a list of VlanRange objects - -# >>> import collections -# >>> VlanRange = collections.namedtuple("VlanRange", ["start", "end"]) -# >>> _clean_vlan_ranges([VlanRange(1,4), VlanRange(3,3), VlanRange(5, 5), VlanRange(7, 10), VlanRange(12, 12)]) -# [[1, 5], [7, 10], [12]] - -# """ -# numbers: list[int] = expand_ranges( -# [(vlan.start, vlan.end) for vlan in vlans], inclusive=True -# ) -# sorted_numbers: Iterable[int] = sorted(set(numbers)) -# grouped_by = ( -# list(x) for _, x in groupby(sorted_numbers, lambda x, c=count(): next(c) - x) -# ) # type: ignore # noqa: B008 -# return [ -# [element[0], element[-1]] if element[0] != element[-1] else [element[0]] -# for element in grouped_by -# ] - - -def get_port_mode(subscription: SubscriptionTable) -> PortMode: +def _get_port_mode(subscription: SubscriptionTable) -> PortMode: if subscription.product.tag in [PortTag.AGGSP + PortTag.SP]: return subscription.port_mode return PortMode.TAGGED -# TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) -def get_vlans_by_subscription_id(subscription_id: UUID) -> list[list[int]]: - values = subscriptions.find_values_for_resource_types( - subscription_id, ["vlan"], strict=True - ) - # print("Values from vlan resource type:", values) - vlan = first(values["vlan"], default=[]) # Provide empty list as default - # print("First vlan value:", vlan) - - return vlan - - # values = subscriptions.find_values_for_resource_types( - # subscription_id, ["sap"], strict=True - # ) - # print("Values from sap resource type:", values) - # sap = first(values["vlan", "sap"]) # Provide empty list as default - # print("First sap value:", sap) - - # return sap - # return _clean_vlan_ranges(get_vlans_by_ims_circuit_id(int(ims_circuit_id))) - - def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRanges: # We assume an empty string is untagged and thus 0 if not vlan: @@ -111,7 +55,7 @@ def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRan subscription_id, model=SubscriptionTable ) - port_mode = get_port_mode(subscription) + port_mode = _get_port_mode(subscription) if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): raise VlanValueError(f"{port_mode} {subscription.product.tag} must have a vlan") @@ -123,158 +67,31 @@ def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRan return vlan -def check_vlan_in_use( - current: list[State], v: CustomVlanRanges, info: ValidationInfo -) -> CustomVlanRanges: - """Check if vlan value is already in use by service port. - - Args: - current: List of current form states, used to filter out self from used vlans. - v: Vlan range of the form input. - info: validation info, contains other fields in info.data - - 1. Get all used vlans in a service port. - 2. Get all nsi reserved vlans and add them to the used_vlans. - 3. Filter out vlans used in current subscription from used_vlans. - 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. - 5. checks if input value uses already used vlans. errors if true. - 6. return input value. - - - """ - if not (subscription_id := info.data.get("subscription_id")): - return v - - # TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) - vlans = get_vlans_by_subscription_id(subscription_id) - used_vlans = [vlan.vid for vlan in vlans if vlan is not None] - nsistp_reserved_vlans = get_available_vlans_by_port_id(subscription_id) - used_vlans = CustomVlanRanges( - flatten([list(used_vlans), list(nsistp_reserved_vlans)]) - ) - - # Remove currently chosen vlans for this port to prevent tripping on in used by itself - current_selected_vlan_ranges: list[str] = [] - if current: - current_selected_service_port = filter( - lambda c: str(c["subscription_id"]) == str(subscription_id), current - ) - current_selected_vlans = list( - map(operator.itemgetter("vlan"), current_selected_service_port) - ) - for current_selected_vlan in current_selected_vlans: - # We assume an empty string is untagged and thus 0 - if not current_selected_vlan: - current_selected_vlan = "0" - - current_selected_vlan_range = CustomVlanRanges(current_selected_vlan) - used_vlans -= current_selected_vlan_range - current_selected_vlan_ranges = [ - *current_selected_vlan_ranges, - *list(current_selected_vlan_range), - ] - - # TODO (#1842): probably better to have a separate type/validator for this - # if nsi_vlans_only: - # vlan_list = list(v) - # invalid_ranges = [ - # vlan - # for vlan in vlan_list - # if vlan not in list(nsistp_reserved_vlans) - # and vlan not in current_selected_vlan_ranges - # ] - # used_vlans -= nsistp_reserved_vlans - - # if invalid_ranges: - # raise VlanValueError( - # f"Vlan(s) {CustomVlanRanges(invalid_ranges)} not valid nsi vlan range" - # ) - - # logger.info( - # "Validation info for current chosen vlans vs vlan already in use", - # current=current, - # used_vlans=used_vlans, - # subscription_id=subscription_id, - # ) - - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) - - if v & used_vlans: - port_mode = get_port_mode(subscription) - - # for tagged only; for link_member/untagged say "SP already in use" - if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: - raise PortsValueError("Port already in use") - raise VlanValueError(f"Vlan(s) {used_vlans} already in use") - - return v - - def validate_vlan_not_in_use( v: CustomVlanRanges, info: ValidationInfo ) -> CustomVlanRanges: """Wrapper for check_vlan_in_use to work with AfterValidator.""" # For single form validation, we don't have a 'current' list, so pass empty list current: list[State] = [] - return check_vlan_not_used(current, v, info) - # return check_vlan_in_use(current, v, info) - + return check_vlan_already_used(current, v, info) -def parse_vlan_ranges_to_list(vlan_string: str) -> list[int]: - """Convert VLAN range string to list of integers. - - Args: - vlan_string: String like "3,5-6,10-12" or "3,5,6" - Returns: - List of integers: [3, 5, 6, 10, 11, 12] - """ - if not vlan_string or vlan_string == "0": - return [0] - - vlans = [] - parts = vlan_string.split(",") - - for part in parts: - part = part.strip() - if "-" in part: - # Handle range like "5-6" - start, end = map(int, part.split("-")) - vlans.extend(range(start, end + 1)) - else: - # Handle single VLAN - vlans.append(int(part)) - - return sorted(set(vlans)) - - -def check_vlan_not_used( +def check_vlan_already_used( current: list[State], v: CustomVlanRanges, info: ValidationInfo ) -> CustomVlanRanges: - """Check if vlan value is already in use by service port. + """Check if vlan value is already in use by a subscription. Args: current: List of current form states, used to filter out self from used vlans. v: Vlan range of the form input. info: validation info, contains other fields in info.data - 1. Get all used vlans in a service port. - 2. Get all nsi reserved vlans and add them to the used_vlans. - 3. Filter out vlans used in current subscription from used_vlans. - 4. if nsi_vlans_only is true, it will also check if the input value is in the range of nsistp vlan ranges. - 5. checks if input value uses already used vlans. errors if true. - 6. return input value. - - + Returns: input value if no errors """ if not (subscription_id := info.data.get("subscription_id")): return v - # TODO: check why this cannot find vlans by subsriptions_id (it seems that there is no subscription_instance_id created) - # vlans = get_vlans_by_subscription_id(subscription_id) - used_vlans = find_allocated_vlans(subscription_id, ["vlan"], strict=False) + used_vlans = find_allocated_vlans(subscription_id, ["vlan"]) # Remove currently chosen vlans for this port to prevent tripping on in used by itself current_selected_vlan_ranges: list[str] = [] @@ -301,11 +118,8 @@ def check_vlan_not_used( subscription_id, model=SubscriptionTable ) - print("validating vlans:", v) - print("used vlans:", used_vlans) - if v & used_vlans: - port_mode = get_port_mode(subscription) + port_mode = _get_port_mode(subscription) # for tagged only; for link_member/untagged say "SP already in use" if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: @@ -316,85 +130,46 @@ def check_vlan_not_used( def find_allocated_vlans( - subscription_id: UUID | UUIDstr, resource_types: Sequence[str], strict: bool = False -) -> dict[str, list[str]]: + subscription_id: UUID | UUIDstr, + resource_types: Sequence[str], +) -> CustomVlanRanges: """Find all vlans already allocated to a SAP for a given port.""" - # the `order_by` on `subscription_instance_id` is there to guarantee the matched ordering across resource_types - # (see also docstring) - print("subscription_id in allocated vlans:", subscription_id) - print("resource_types in allocated vlans:", resource_types) - - subscription_instance_id = db.session.execute( - select(SubscriptionInstanceTable.subscription_instance_id).filter( - SubscriptionInstanceTable.subscription_id == subscription_id - ) - ).scalar_one_or_none() - - print("subscription_instance_id:", subscription_instance_id) + logger.debug( + "Finding allocated VLANs", + subscription_id=subscription_id, + resource_types=resource_types, + ) - vlan_resource_type = db.session.execute( - select(ResourceTypeTable).filter( - ResourceTypeTable.resource_type == "vlan", + # Get all VLAN values used by the subscription + query = ( + select(SubscriptionInstanceValueTable.value) + .join( + ResourceTypeTable, + SubscriptionInstanceValueTable.resource_type_id + == ResourceTypeTable.resource_type_id, ) - ).scalar_one_or_none() - - vlan_ids = ( - db.session.execute( - select(SubscriptionInstanceValueTable.subscription_instance_id).filter( - SubscriptionInstanceValueTable.resource_type_id - == vlan_resource_type.resource_type_id, - ) + .join( + SubscriptionInstanceRelationTable, + SubscriptionInstanceValueTable.subscription_instance_id + == SubscriptionInstanceRelationTable.in_use_by_id, ) - .scalars() - .all() - ) - - # vlan_ids = [vlan.subscription_instance_id for vlan in vlans] - - print("Found vlan IDs:", vlan_ids) - print("Number of vlan IDs:", len(vlan_ids)) - - # Check for VLAN IDs that are in use by this subscription instance - vlan_relations_query = ( - select(SubscriptionInstanceRelationTable.in_use_by_id) .join( SubscriptionInstanceTable, SubscriptionInstanceRelationTable.depends_on_id == SubscriptionInstanceTable.subscription_instance_id, ) .filter( - SubscriptionInstanceTable.subscription_instance_id - == subscription_instance_id + SubscriptionInstanceTable.subscription_id == subscription_id, + ResourceTypeTable.resource_type == "vlan", ) ) - vlan_ids_in_use = db.session.execute(vlan_relations_query).scalars().all() - print("VLAN IDs in use by this subscription:", vlan_ids_in_use) - print("Number of VLAN IDs in use:", len(vlan_ids_in_use)) - - # Get VLAN values for the subscription instances that are in use - if vlan_ids_in_use: - vlan_values_query = ( - select(SubscriptionInstanceValueTable.value) - .join( - ResourceTypeTable, - SubscriptionInstanceValueTable.resource_type_id - == ResourceTypeTable.resource_type_id, - ) - .filter( - SubscriptionInstanceValueTable.subscription_instance_id.in_( - vlan_ids_in_use - ), - ResourceTypeTable.resource_type == "vlan", - ) - ) + used_vlan_values = db.session.execute(query).scalars().all() - used_vlan_values = db.session.execute(vlan_values_query).scalars().all() - print("Used VLAN values:", used_vlan_values) - used_vlan_values_int = {int(vlan) for vlan in used_vlan_values} - return used_vlan_values_int + if not used_vlan_values: + logger.debug("No VLAN values in use found") + return CustomVlanRanges([]) - else: - used_vlan_values = [] - print("No VLAN values in use found.") - return used_vlan_values + logger.debug("Found used VLAN values", values=used_vlan_values) + used_vlan_values_int = {int(vlan) for vlan in used_vlan_values} + return CustomVlanRanges(used_vlan_values_int) From 923543df38abe60efb0475abf18540de97d9e082 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 16:02:20 +0200 Subject: [PATCH 06/23] Nsistp products types and workflow now works --- products/product_blocks/sap.py | 8 +-- workflows/node/create_node.py | 31 ++++++--- workflows/nsistp/create_nsistp.py | 21 +++--- workflows/nsistp/modify_nsistp.py | 10 +-- workflows/nsistp/shared/forms.py | 5 +- workflows/nsistp/shared/nsistp_services.py | 81 ---------------------- workflows/nsistp/shared/vlan.py | 29 +++++--- workflows/shared.py | 14 ---- 8 files changed, 55 insertions(+), 144 deletions(-) delete mode 100644 workflows/nsistp/shared/nsistp_services.py diff --git a/products/product_blocks/sap.py b/products/product_blocks/sap.py index 9ef9fc6..bccc41e 100644 --- a/products/product_blocks/sap.py +++ b/products/product_blocks/sap.py @@ -21,13 +21,11 @@ PortBlockInactive, PortBlockProvisioning, ) -from workflows.nsistp.shared.shared import CustomVlanRanges class SAPBlockInactive(ProductBlockModel, product_block_name="SAP"): port: PortBlockInactive | None = None vlan: int | None = None - # vlan: CustomVlanRanges # TODO: check where to use int or CustomVlanRanges ims_id: int | None = None @@ -35,8 +33,7 @@ class SAPBlockProvisioning( SAPBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] ): port: PortBlockProvisioning - vlan: int - # vlan: CustomVlanRanges # TODO: check where to use int or CustomVlanRanges + vlan: int # TODO: refactor to CustomVlanRanges together with L2VPN product and workflow ims_id: int | None = None @computed_field # type: ignore[misc] @@ -47,6 +44,5 @@ def title(self) -> str: class SAPBlock(SAPBlockProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): port: PortBlock - # vlan: int - vlan: CustomVlanRanges | int # TODO: check where to use int or CustomVlanRanges + vlan: int # TODO: refactor to CustomVlanRanges together with L2VPN product and workflow ims_id: int diff --git a/workflows/node/create_node.py b/workflows/node/create_node.py index 4581f70..c95c33d 100644 --- a/workflows/node/create_node.py +++ b/workflows/node/create_node.py @@ -12,22 +12,21 @@ # limitations under the License. -from pydantic_forms.types import UUIDstr -import uuid import json +import uuid from random import randrange from typing import TypeAlias, cast from orchestrator.services.products import get_product_by_id from orchestrator.targets import Target from orchestrator.types import SubscriptionLifecycle +from orchestrator.utils.json import json_dumps from orchestrator.workflow import StepList, begin, step from orchestrator.workflows.steps import store_process_subscription from orchestrator.workflows.utils import create_workflow -from orchestrator.utils.json import json_dumps from pydantic import ConfigDict from pydantic_forms.core import FormPage -from pydantic_forms.types import FormGenerator, State +from pydantic_forms.types import FormGenerator, State, UUIDstr from pydantic_forms.validators import Choice, Label from products.product_blocks.shared.types import NodeStatus @@ -36,7 +35,12 @@ from products.services.netbox.netbox import build_payload from services import netbox from services.lso_client import execute_playbook, lso_interaction -from workflows.node.shared.forms import NodeStatusChoice, node_role_selector, node_type_selector, site_selector +from workflows.node.shared.forms import ( + NodeStatusChoice, + node_role_selector, + node_type_selector, + site_selector, +) from workflows.node.shared.steps import update_node_in_ims from workflows.shared import create_summary_form @@ -64,7 +68,14 @@ class CreateNodeForm(FormPage): user_input = yield CreateNodeForm user_input_dict = user_input.model_dump() - summary_fields = ["role_id", "type_id", "site_id", "node_status", "node_name", "node_description"] + summary_fields = [ + "role_id", + "type_id", + "site_id", + "node_status", + "node_name", + "node_description", + ] yield from create_summary_form(user_input_dict, product_name, summary_fields) return user_input_dict @@ -95,7 +106,9 @@ def construct_node_model( subscription.node.node_name = node_name subscription.node.node_description = node_description - subscription = NodeProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) + subscription = NodeProvisioning.from_other_lifecycle( + subscription, SubscriptionLifecycle.PROVISIONING + ) subscription.description = description(subscription) return { @@ -114,8 +127,8 @@ def create_node_in_ims(subscription: NodeProvisioning) -> State: @step("Reserve loopback addresses") def reserve_loopback_addresses(subscription: NodeProvisioning) -> State: - subscription.node.ipv4_ipam_id, subscription.node.ipv6_ipam_id = netbox.reserve_loopback_addresses( - subscription.node.ims_id + subscription.node.ipv4_ipam_id, subscription.node.ipv6_ipam_id = ( + netbox.reserve_loopback_addresses(subscription.node.ims_id) ) return {"subscription": subscription} diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 63d44dd..69332dc 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -2,12 +2,13 @@ # from typing import TypeAlias, cast +import uuid from typing import Annotated import structlog from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage -from orchestrator.forms.validators import CustomerId, Divider, Label +from orchestrator.forms.validators import Divider, Label from orchestrator.targets import Target from orchestrator.types import SubscriptionLifecycle from orchestrator.workflow import StepList, begin, step @@ -30,7 +31,6 @@ validate_both_aliases_empty_or_not, ) from workflows.nsistp.shared.vlan import ( - CustomVlanRanges, validate_vlan, validate_vlan_not_in_use, ) @@ -50,28 +50,25 @@ def subscription_description(subscription: SubscriptionModel) -> str: def initial_input_form_generator(product_name: str) -> FormGenerator: - # TODO add additional fields to form if needed - class CreateNsiStpForm(FormPage): model_config = ConfigDict(title=product_name) - # TODO: check whether this should be removed - customer_id: CustomerId + # customer_id: CustomerId nsistp_settings: Label - divider_1: Divider - # TODO: could this be multiple service ports?? subscription_id: Annotated[ UUIDstr, ports_selector(), ] vlan: Annotated[ - CustomVlanRanges, + int, AfterValidator(validate_vlan), AfterValidator(validate_vlan_not_in_use), ] + divider_1: Divider + topology: Topology stp_id: StpId stp_description: StpDescription | None = None @@ -107,9 +104,9 @@ def validate_is_alias_in_out(self) -> "CreateNsiStpForm": @step("Construct Subscription model") def construct_nsistp_model( product: UUIDstr, - customer_id: UUIDstr, + # customer_id: UUIDstr, subscription_id, - vlan, + vlan: int, topology: str, stp_id: str, stp_description: str | None, @@ -120,7 +117,7 @@ def construct_nsistp_model( ) -> State: nsistp = NsistpInactive.from_product_id( product_id=product, - customer_id=customer_id, + customer_id=str(uuid.uuid4()), status=SubscriptionLifecycle.INITIAL, ) nsistp.nsistp.topology = topology diff --git a/workflows/nsistp/modify_nsistp.py b/workflows/nsistp/modify_nsistp.py index fd4af42..a0e1b8f 100644 --- a/workflows/nsistp/modify_nsistp.py +++ b/workflows/nsistp/modify_nsistp.py @@ -1,8 +1,9 @@ # workflows/nsistp/modify_nsistp.py + import structlog from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage -from orchestrator.forms.validators import CustomerId, Divider +from orchestrator.forms.validators import Divider from orchestrator.types import SubscriptionLifecycle from orchestrator.workflow import StepList, begin, step from orchestrator.workflows.steps import set_status @@ -28,14 +29,11 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator: subscription = Nsistp.from_subscription(subscription_id) nsistp = subscription.nsistp - # TODO fill in additional fields if needed - class ModifyNsistpForm(FormPage): - customer_id: CustomerId = subscription.customer_id # type: ignore + stp_id: read_only_field(nsistp.stp_id) divider_1: Divider - stp_id: read_only_field(nsistp.stp_id) topology: str = nsistp.topology stp_description: str | None = nsistp.stp_description is_alias_in: str | None = nsistp.is_alias_in @@ -70,7 +68,6 @@ def update_subscription( expose_in_topology: bool | None, bandwidth: int | None, ) -> State: - # TODO: get all modified fields subscription.nsistp.topology = topology subscription.nsistp.stp_description = stp_description subscription.nsistp.is_alias_in = is_alias_in @@ -101,6 +98,5 @@ def modify_nsistp() -> StepList: >> set_status(SubscriptionLifecycle.PROVISIONING) >> update_subscription >> update_subscription_description - # TODO add additional steps if needed >> set_status(SubscriptionLifecycle.ACTIVE) ) diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index fb0cd69..0c38d50 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -58,10 +58,7 @@ def ports_selector() -> type[list[Choice]]: port_subscriptions, key=lambda port: port.description ) } - - print("Port choices found:", ports) # --- IGNORE --- - - return Choice("ServicePort", zip(ports.keys(), ports.items())) + return Choice("Port", zip(ports.keys(), ports.items())) def is_fqdn(hostname: str) -> bool: diff --git a/workflows/nsistp/shared/nsistp_services.py b/workflows/nsistp/shared/nsistp_services.py deleted file mode 100644 index da1f9b7..0000000 --- a/workflows/nsistp/shared/nsistp_services.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections.abc import Sequence -from uuid import UUID - -from more_itertools import flatten -from orchestrator.db import db -from orchestrator.db.models import ( - ProductTable, - SubscriptionInstanceRelationTable, - SubscriptionInstanceTable, - SubscriptionTable, -) -from orchestrator.types import SubscriptionLifecycle -from sqlalchemy import select -from sqlalchemy.orm import aliased - -from products.product_types.nsistp import Nsistp -from workflows.nsistp.shared.shared import CustomVlanRanges - - -def _get_subscriptions_inuseby_port_id( - port_id: UUID, product_type: str, statuses: list[SubscriptionLifecycle] -) -> Sequence[UUID]: - relations = aliased(SubscriptionInstanceRelationTable) - instances = aliased(SubscriptionInstanceTable) - query = ( - select(SubscriptionTable.subscription_id) - .join(SubscriptionInstanceTable) - .join(ProductTable) - .join( - relations, - relations.in_use_by_id - == SubscriptionInstanceTable.subscription_instance_id, - ) - .join(instances, relations.depends_on_id == instances.subscription_instance_id) - .filter(instances.subscription_id == port_id) - .filter(ProductTable.product_type == product_type) - .filter(SubscriptionTable.status.in_(statuses)) - ) - return db.session.scalars(query).all() - - -def nsistp_get_by_port_id(port_id: UUID) -> list[Nsistp]: - """Get Nsistps by service port id. - - Args: - port_id: ID of the service port for which you want all nsistps of. - """ - statuses = [ - SubscriptionLifecycle.ACTIVE, - SubscriptionLifecycle.PROVISIONING, - SubscriptionLifecycle.MIGRATING, - ] - result = _get_subscriptions_inuseby_port_id(port_id, "Nsistp", statuses) - - return [Nsistp.from_subscription(id) for id in list(set(result))] - - -def get_available_vlans_by_port_id(port_id: UUID) -> CustomVlanRanges: - """Get available vlans by service port id. - - This will get all NSISTPs and adds their vlan ranges to a single VlanRanges to get the available vlans by nsistps. - - Args: - port_id: ID of the service port to find available vlans. - """ - nsistps = nsistp_get_by_port_id(port_id) - available_vlans = CustomVlanRanges(flatten(nsistp.vlan_range for nsistp in nsistps)) - - return available_vlans diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index 62ba941..92a9386 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -67,18 +67,16 @@ def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRan return vlan -def validate_vlan_not_in_use( - v: CustomVlanRanges, info: ValidationInfo -) -> CustomVlanRanges: +def validate_vlan_not_in_use(vlan: int, info: ValidationInfo) -> int | CustomVlanRanges: """Wrapper for check_vlan_in_use to work with AfterValidator.""" # For single form validation, we don't have a 'current' list, so pass empty list current: list[State] = [] - return check_vlan_already_used(current, v, info) + return check_vlan_already_used(current, vlan, info) def check_vlan_already_used( - current: list[State], v: CustomVlanRanges, info: ValidationInfo -) -> CustomVlanRanges: + current: list[State], vlan: int | CustomVlanRanges, info: ValidationInfo +) -> int | CustomVlanRanges: """Check if vlan value is already in use by a subscription. Args: @@ -89,7 +87,7 @@ def check_vlan_already_used( Returns: input value if no errors """ if not (subscription_id := info.data.get("subscription_id")): - return v + return vlan used_vlans = find_allocated_vlans(subscription_id, ["vlan"]) @@ -118,7 +116,14 @@ def check_vlan_already_used( subscription_id, model=SubscriptionTable ) - if v & used_vlans: + # Handle both int and CustomVlanRanges + if isinstance(vlan, int): + vlan_in_use = vlan in used_vlans + else: + # For CustomVlanRanges, check if any of its values are in used_vlans + vlan_in_use = any(v in used_vlans for v in vlan) + + if vlan_in_use: port_mode = _get_port_mode(subscription) # for tagged only; for link_member/untagged say "SP already in use" @@ -126,7 +131,7 @@ def check_vlan_already_used( raise PortsValueError("Port already in use") raise VlanValueError(f"Vlan(s) {used_vlans} already in use") - return v + return vlan def find_allocated_vlans( @@ -168,8 +173,10 @@ def find_allocated_vlans( if not used_vlan_values: logger.debug("No VLAN values in use found") + return [] return CustomVlanRanges([]) logger.debug("Found used VLAN values", values=used_vlan_values) - used_vlan_values_int = {int(vlan) for vlan in used_vlan_values} - return CustomVlanRanges(used_vlan_values_int) + used_vlan_values_int = list({int(vlan) for vlan in used_vlan_values}) + return used_vlan_values_int + # return CustomVlanRanges(used_vlan_values_int) diff --git a/workflows/shared.py b/workflows/shared.py index 2731e4e..5c8dff4 100644 --- a/workflows/shared.py +++ b/workflows/shared.py @@ -92,20 +92,6 @@ def subscriptions_by_product_type_and_instance_value( Returns: Subscription or None """ - - query = ( - SubscriptionTable.query.join(ProductTable) - .join(SubscriptionInstanceTable) - .join(SubscriptionInstanceValueTable) - .join(ResourceTypeTable) - .filter(ProductTable.product_type == product_type) - .filter(SubscriptionInstanceValueTable.value == value) - .filter(ResourceTypeTable.resource_type == resource_type) - .filter(SubscriptionTable.status.in_(status)) - ) - - print("subscription HELLO", query.all()) - return ( SubscriptionTable.query.join(ProductTable) .join(SubscriptionInstanceTable) From 48f2a37199e75cdca6e8aaf46af6112d4ce75c38 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 16:27:00 +0200 Subject: [PATCH 07/23] Removed unused / unnecessary code --- utils/types.py | 44 ------------------------------- workflows/nsistp/shared/shared.py | 17 ------------ workflows/nsistp/shared/vlan.py | 6 +++-- 3 files changed, 4 insertions(+), 63 deletions(-) delete mode 100644 utils/types.py diff --git a/utils/types.py b/utils/types.py deleted file mode 100644 index e4ae04b..0000000 --- a/utils/types.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Literal - -# TODO: remove entirely or irrelevant tags -Tags = Literal[ - "SP", - "SPNL", - "MSC", - "MSCNL", - "AGGSP", - "LightPath", - "LP", - "LR", - "NSILP", - "IPS", - "IPBGP", - "AGGSPNL", - "LRNL", - "LIR_PREFIX", - "SUB_PREFIX", - "Node", - "IRBSP", - "FW", - "Corelink", - "L2VPN", - "L3VPN", - "LPNLNSI", - "IPPG", - "IPPP", - "Wireless", - "OS", -] diff --git a/workflows/nsistp/shared/shared.py b/workflows/nsistp/shared/shared.py index 0d92dd1..95683d9 100644 --- a/workflows/nsistp/shared/shared.py +++ b/workflows/nsistp/shared/shared.py @@ -23,8 +23,6 @@ from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema -from utils.types import Tags - logger = structlog.get_logger(__name__) GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] @@ -35,26 +33,11 @@ class PortTag(StrEnum): SP = "SP" - SPNL = "SPNL" AGGSP = "AGGSP" - AGGSPNL = "AGGSPNL" MSC = "MSC" - MSCNL = "MSCNL" IRBSP = "IRBSP" -PORT_TAG_GENERAL: list[Tags] = ["PORT"] - -# TODO: these tags can probably be removed -PORT_TAGS_AGGSP: list[Tags] = ["AGGSP", "AGGSPNL"] -PORT_TAGS_IRBSP: list[Tags] = ["IRBSP"] -PORT_TAGS_MSC: list[Tags] = ["MSC", "MSCNL"] -PORT_TAGS_SP: list[Tags] = ["SP"] -PORT_TAGS_ALL: list[Tags] = ( - PORT_TAGS_SP + PORT_TAGS_AGGSP + PORT_TAGS_MSC + PORT_TAGS_IRBSP + PORT_TAG_GENERAL -) - - # Custom VlanRanges needed to avoid matching conflict with SURF orchestrator-ui components class CustomVlanRanges(VlanRanges): def __str__(self) -> str: diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index 92a9386..aedae8c 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -129,7 +129,9 @@ def check_vlan_already_used( # for tagged only; for link_member/untagged say "SP already in use" if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: raise PortsValueError("Port already in use") - raise VlanValueError(f"Vlan(s) {used_vlans} already in use") + raise VlanValueError( + f"Vlan(s) {', '.join(map(str, sorted(used_vlans)))} already in use" + ) return vlan @@ -174,7 +176,7 @@ def find_allocated_vlans( if not used_vlan_values: logger.debug("No VLAN values in use found") return [] - return CustomVlanRanges([]) + # return CustomVlanRanges([]) logger.debug("Found used VLAN values", values=used_vlan_values) used_vlan_values_int = list({int(vlan) for vlan in used_vlan_values}) From 597efa756e870b2d86663854e029afc9aa5c0cd0 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 17:03:19 +0200 Subject: [PATCH 08/23] Added single dispatch for subscription description and some minor adjustments --- products/product_blocks/node.py | 8 ++++++-- products/services/description.py | 16 +++++++++++++++- translations/en-GB.json | 1 + workflows/nsistp/create_nsistp.py | 14 ++------------ workflows/nsistp/shared/vlan.py | 8 +++----- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/products/product_blocks/node.py b/products/product_blocks/node.py index 5388ca6..62662c9 100644 --- a/products/product_blocks/node.py +++ b/products/product_blocks/node.py @@ -21,7 +21,9 @@ class NodeBlockInactive(ProductBlockModel, product_block_name="Node"): role_id: int | None = None type_id: int | None = None site_id: int | None = None - node_status: str | None = None # should be NodeStatus, but strEnum is not supported (yet?) + node_status: str | None = ( + None # should be NodeStatus, but strEnum is not supported (yet?) + ) node_name: str | None = None node_description: str | None = None ims_id: int | None = None @@ -30,7 +32,9 @@ class NodeBlockInactive(ProductBlockModel, product_block_name="Node"): ipv6_ipam_id: int | None = None -class NodeBlockProvisioning(NodeBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): +class NodeBlockProvisioning( + NodeBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] +): role_id: int type_id: int site_id: int diff --git a/products/services/description.py b/products/services/description.py index d4b58ef..16ce8f7 100644 --- a/products/services/description.py +++ b/products/services/description.py @@ -21,12 +21,15 @@ from products.product_types.core_link import CoreLinkProvisioning from products.product_types.l2vpn import L2vpnProvisioning from products.product_types.node import NodeProvisioning +from products.product_types.nsistp import NsistpProvisioning from products.product_types.port import PortProvisioning from utils.singledispatch import single_dispatch_base @singledispatch -def description(model: Union[ProductModel, ProductBlockModel, SubscriptionModel]) -> str: +def description( + model: Union[ProductModel, ProductBlockModel, SubscriptionModel], +) -> str: """Build subscription description (generic function). Specific implementations of this generic function will specify the model types they work on. @@ -79,3 +82,14 @@ def _(l2vpn: L2vpnProvisioning) -> str: f"{l2vpn.virtual_circuit.speed} Mbit/s " f"({'-'.join(sorted(list(set([sap.port.node.node_name for sap in l2vpn.virtual_circuit.saps]))))})" ) + + +@description.register +def _(nsistp: NsistpProvisioning) -> str: + product_tag = nsistp.product.tag + stp_id = nsistp.nsistp.stp_id + topology = nsistp.nsistp.topology + + service_speed = nsistp.nsistp.bandwidth + + return f"{product_tag} {stp_id} {topology} {service_speed}" diff --git a/translations/en-GB.json b/translations/en-GB.json index 65636b0..12b887b 100644 --- a/translations/en-GB.json +++ b/translations/en-GB.json @@ -17,6 +17,7 @@ "node_subscription_id_a": "A side node", "node_subscription_id_b": "B side node", "number_of_ports": "Number of ports", + "nsistp_settings": "Network Service Interface Service Termination Point settings", "port_description": "Port description", "port_description_info": "Description of the port", "port_ims_id": "Port", diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 69332dc..571f995 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -6,7 +6,6 @@ from typing import Annotated import structlog -from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage from orchestrator.forms.validators import Divider, Label from orchestrator.targets import Target @@ -18,6 +17,7 @@ from pydantic_forms.types import FormGenerator, State, UUIDstr from products.product_types.nsistp import NsistpInactive, NsistpProvisioning +from products.services.description import description from products.services.netbox.netbox import build_payload from services import netbox from workflows.nsistp.shared.forms import ( @@ -36,16 +36,6 @@ ) from workflows.shared import create_summary_form - -def subscription_description(subscription: SubscriptionModel) -> str: - """Generate subscription description. - - The suggested pattern is to implement a subscription service that generates a subscription specific - description, in case that is not present the description will just be set to the product name. - """ - return f"{subscription.product.name} subscription" - - logger = structlog.get_logger(__name__) @@ -136,7 +126,7 @@ def construct_nsistp_model( nsistp = NsistpProvisioning.from_other_lifecycle( nsistp, SubscriptionLifecycle.PROVISIONING ) - nsistp.description = subscription_description(nsistp) + nsistp.description = description(nsistp) return { "subscription": nsistp, diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index aedae8c..babd43f 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -12,7 +12,6 @@ # limitations under the License. import operator -from collections.abc import Sequence from uuid import UUID import structlog @@ -89,7 +88,7 @@ def check_vlan_already_used( if not (subscription_id := info.data.get("subscription_id")): return vlan - used_vlans = find_allocated_vlans(subscription_id, ["vlan"]) + used_vlans = find_allocated_vlans(subscription_id) # Remove currently chosen vlans for this port to prevent tripping on in used by itself current_selected_vlan_ranges: list[str] = [] @@ -136,15 +135,14 @@ def check_vlan_already_used( return vlan +# TODO: rewrite to support CustomVlanRanges def find_allocated_vlans( subscription_id: UUID | UUIDstr, - resource_types: Sequence[str], -) -> CustomVlanRanges: +) -> list[int]: """Find all vlans already allocated to a SAP for a given port.""" logger.debug( "Finding allocated VLANs", subscription_id=subscription_id, - resource_types=resource_types, ) # Get all VLAN values used by the subscription From 109431d13922356ab0db6be7146ce163a42355c2 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Tue, 7 Oct 2025 17:14:46 +0200 Subject: [PATCH 09/23] Added documentation --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 409e061..6b92137 100644 --- a/README.md +++ b/README.md @@ -486,6 +486,10 @@ should exist, and every port can only be used once in the same L2VPN. This product is only supported on tagged interfaces, and VLAN retagging is not supported. +#### NSISTP + +The Network Service Interface (NSISTP) / Service Termination Point (STP) represents the logical endpoint where a network service connects to a customer port. To create an NSISTP, at least one port subscription must exist. The NSISTP workflow allows you to define service-specific parameters such as VLAN assignment and Service Speed. NSISTP can only be created on tagged ports, as untagged ports are limited to a single service. + ## Products The Orchestrator uses the concept of a Product to describe what can be built to the end user. When From 94c66041043f63bdfd7f159538375302a933da3e Mon Sep 17 00:00:00 2001 From: Thomas <47833122+tvdven@users.noreply.github.com> Date: Wed, 8 Oct 2025 09:01:56 +0200 Subject: [PATCH 10/23] Remove forms volume from docker-compose Removed forms volume from orchestrator service. --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 947dc65..51eb286 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -193,7 +193,6 @@ services: - 5678 #Enable Python debugger volumes: - ./workflows:/home/orchestrator/workflows - - ./forms:/home/orchestrator/forms - ./products:/home/orchestrator/products - ./migrations:/home/orchestrator/migrations - ./docker:/home/orchestrator/etc From a3c4cf76507353e9824cde8e7fa5cd21cef7476f Mon Sep 17 00:00:00 2001 From: Thomas <47833122+tvdven@users.noreply.github.com> Date: Wed, 8 Oct 2025 09:02:33 +0200 Subject: [PATCH 11/23] Netbox Docker image back to v4.4.1 Updated the Netbox base image to version 4.4.1. --- docker/netbox/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/netbox/Dockerfile b/docker/netbox/Dockerfile index 73ca5f6..4f85e11 100644 --- a/docker/netbox/Dockerfile +++ b/docker/netbox/Dockerfile @@ -1,4 +1,4 @@ -FROM netboxcommunity/netbox:v4.0-2.9.1 +FROM netboxcommunity/netbox:v4.4.1 # NOTE: when updating the Netbox version, remember to update the database snapshot. See docker/postgresql/README.md for details From 0a9c1cc0ded1fbd4c6f922631262d688fa5d67c9 Mon Sep 17 00:00:00 2001 From: Thomas van der Ven Date: Wed, 8 Oct 2025 09:07:26 +0200 Subject: [PATCH 12/23] Removed unused exceptions --- utils/exceptions.py | 111 -------------------------------------------- 1 file changed, 111 deletions(-) diff --git a/utils/exceptions.py b/utils/exceptions.py index 6da2ac9..f672132 100644 --- a/utils/exceptions.py +++ b/utils/exceptions.py @@ -14,124 +14,13 @@ """Provides ValueError Exception classes.""" -class AllowedStatusValueError(ValueError): - pass - - -class ASNValueError(ValueError): - pass - - -class BGPPolicyValueError(ValueError): - pass - - -class BlackHoleCommunityValueError(ValueError): - pass - - -class ChoiceValueError(ValueError): - pass - - class DuplicateValueError(ValueError): pass -class EndpointTypeValueError(ValueError): - pass - - class FieldValueError(ValueError): pass -class FreeSpaceValueError(ValueError): - pass - - -class InSyncValueError(ValueError): - pass - - -class IPAddressValueError(ValueError): - pass - - -class IPPrefixValueError(ValueError): - pass - - -class LocationValueError(ValueError): - pass - - -class NodesValueError(ValueError): - pass - - -class CustomerValueError(ValueError): - pass - - -class PeeringValueError(ValueError): - pass - - -class PeerGroupNameError(ValueError): - pass - - -class PeerNameValueError(ValueError): - pass - - -class PeerPortNameValueError(ValueError): - pass - - -class PeerPortValueError(ValueError): - pass - - -class PortsModeValueError(ValueError): - def __init__(self, mode: str = "", message: str = ""): - super().__init__(message) - self.message = message - self.mode = mode - - -class PortsValueError(ValueError): - pass - - -class ProductValueError(ValueError): - pass - - -class ServicesActiveValueError(ValueError): - pass - - -class SubscriptionTypeValueError(ValueError): - pass - - -class UnsupportedSpeedValueError(ValueError): - pass - - -class UnsupportedTypeValueError(ValueError): - pass - - -class VlanRetaggingValueError(ValueError): - pass - - class VlanValueError(ValueError): pass - - -class InUseByAzError(ValueError): - pass From 1367bfeaec511679b5a2956b9ae43e8948af400d Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 10:43:46 +0200 Subject: [PATCH 13/23] reformat with line lengt 120 again --- products/__init__.py | 9 ++---- products/product_blocks/node.py | 8 ++---- products/product_blocks/sap.py | 10 ++----- products/services/description.py | 4 +-- pyproject.toml | 38 +++++++++++++++++++++++++ workflows/l2vpn/create_l2vpn.py | 11 ++----- workflows/l2vpn/shared/forms.py | 13 ++------- workflows/node/create_node.py | 24 ++++------------ workflows/port/create_port.py | 16 ++--------- workflows/shared.py | 49 ++++++++------------------------ 10 files changed, 71 insertions(+), 111 deletions(-) diff --git a/products/__init__.py b/products/__init__.py index 9a0078f..874cf52 100644 --- a/products/__init__.py +++ b/products/__init__.py @@ -17,6 +17,7 @@ from products.product_types.core_link import CoreLink from products.product_types.l2vpn import L2vpn from products.product_types.node import Node +from products.product_types.nsistp import Nsistp from products.product_types.port import Port SUBSCRIPTION_MODEL_REGISTRY.update( @@ -30,12 +31,6 @@ "core link 10G": CoreLink, "core link 100G": CoreLink, "l2vpn": L2vpn, + "nsistp": Nsistp, } ) -from products.product_types.nsistp import Nsistp - -SUBSCRIPTION_MODEL_REGISTRY.update( - { - "nsistp": Nsistp, - }, -) # fmt:skip diff --git a/products/product_blocks/node.py b/products/product_blocks/node.py index 62662c9..5388ca6 100644 --- a/products/product_blocks/node.py +++ b/products/product_blocks/node.py @@ -21,9 +21,7 @@ class NodeBlockInactive(ProductBlockModel, product_block_name="Node"): role_id: int | None = None type_id: int | None = None site_id: int | None = None - node_status: str | None = ( - None # should be NodeStatus, but strEnum is not supported (yet?) - ) + node_status: str | None = None # should be NodeStatus, but strEnum is not supported (yet?) node_name: str | None = None node_description: str | None = None ims_id: int | None = None @@ -32,9 +30,7 @@ class NodeBlockInactive(ProductBlockModel, product_block_name="Node"): ipv6_ipam_id: int | None = None -class NodeBlockProvisioning( - NodeBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] -): +class NodeBlockProvisioning(NodeBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): role_id: int type_id: int site_id: int diff --git a/products/product_blocks/sap.py b/products/product_blocks/sap.py index bccc41e..a465ccb 100644 --- a/products/product_blocks/sap.py +++ b/products/product_blocks/sap.py @@ -16,11 +16,7 @@ from orchestrator.types import SubscriptionLifecycle from pydantic import computed_field -from products.product_blocks.port import ( - PortBlock, - PortBlockInactive, - PortBlockProvisioning, -) +from products.product_blocks.port import PortBlock, PortBlockInactive, PortBlockProvisioning class SAPBlockInactive(ProductBlockModel, product_block_name="SAP"): @@ -29,9 +25,7 @@ class SAPBlockInactive(ProductBlockModel, product_block_name="SAP"): ims_id: int | None = None -class SAPBlockProvisioning( - SAPBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] -): +class SAPBlockProvisioning(SAPBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): port: PortBlockProvisioning vlan: int # TODO: refactor to CustomVlanRanges together with L2VPN product and workflow ims_id: int | None = None diff --git a/products/services/description.py b/products/services/description.py index 16ce8f7..e54ff2d 100644 --- a/products/services/description.py +++ b/products/services/description.py @@ -27,9 +27,7 @@ @singledispatch -def description( - model: Union[ProductModel, ProductBlockModel, SubscriptionModel], -) -> str: +def description(model: Union[ProductModel, ProductBlockModel, SubscriptionModel]) -> str: """Build subscription description (generic function). Specific implementations of this generic function will specify the model types they work on. diff --git a/pyproject.toml b/pyproject.toml index 824d684..38964c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,41 @@ profile = "black" [tool.flake8] max-line-length = 120 + +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.9 +target-version = "py313" diff --git a/workflows/l2vpn/create_l2vpn.py b/workflows/l2vpn/create_l2vpn.py index 517d6e2..4ab1848 100644 --- a/workflows/l2vpn/create_l2vpn.py +++ b/workflows/l2vpn/create_l2vpn.py @@ -47,8 +47,7 @@ class CreateL2vpnForm(FormPage): user_input = yield CreateL2vpnForm user_input_dict = user_input.model_dump() PortsChoiceList: TypeAlias = cast( - type[Choice], - ports_selector(AllowedNumberOfL2vpnPorts(user_input_dict["number_of_ports"])), # noqa: F821 + type[Choice], ports_selector(AllowedNumberOfL2vpnPorts(user_input_dict["number_of_ports"])) # noqa: F821 ) class SelectPortsForm(FormPage): @@ -90,9 +89,7 @@ def to_sap(port: UUIDstr) -> SAPBlockInactive: subscription.virtual_circuit.saps = [to_sap(port) for port in ports] - subscription = L2vpnProvisioning.from_other_lifecycle( - subscription, SubscriptionLifecycle.PROVISIONING - ) + subscription = L2vpnProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) subscription.description = description(subscription) return { @@ -127,9 +124,7 @@ def ims_create_l2vpn_terminations(subscription: L2vpnProvisioning) -> State: l2vpn = netbox.get_l2vpn(id=subscription.virtual_circuit.ims_id) for sap in subscription.virtual_circuit.saps: vlan = netbox.get_vlan(id=sap.ims_id) - payload = netbox.L2vpnTerminationPayload( - l2vpn=l2vpn.id, assigned_object_id=vlan.id - ) + payload = netbox.L2vpnTerminationPayload(l2vpn=l2vpn.id, assigned_object_id=vlan.id) netbox.create(payload) payloads.append(payload) diff --git a/workflows/l2vpn/shared/forms.py b/workflows/l2vpn/shared/forms.py index 173be0e..54645fc 100644 --- a/workflows/l2vpn/shared/forms.py +++ b/workflows/l2vpn/shared/forms.py @@ -16,23 +16,16 @@ from pydantic_forms.validators import Choice, choice_list from products.product_blocks.port import PortMode -from workflows.shared import ( - AllowedNumberOfL2vpnPorts, - subscriptions_by_product_type_and_instance_value, -) +from workflows.shared import AllowedNumberOfL2vpnPorts, subscriptions_by_product_type_and_instance_value -def ports_selector( - number_of_ports: AllowedNumberOfL2vpnPorts, -) -> type[list[Choice]]: +def ports_selector(number_of_ports: AllowedNumberOfL2vpnPorts) -> type[list[Choice]]: port_subscriptions = subscriptions_by_product_type_and_instance_value( "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] ) ports = { str(subscription.subscription_id): subscription.description - for subscription in sorted( - port_subscriptions, key=lambda port: port.description - ) + for subscription in sorted(port_subscriptions, key=lambda port: port.description) } return choice_list( Choice("PortsEnum", zip(ports.keys(), ports.items())), # type: ignore diff --git a/workflows/node/create_node.py b/workflows/node/create_node.py index c95c33d..fb6fe7a 100644 --- a/workflows/node/create_node.py +++ b/workflows/node/create_node.py @@ -35,12 +35,7 @@ from products.services.netbox.netbox import build_payload from services import netbox from services.lso_client import execute_playbook, lso_interaction -from workflows.node.shared.forms import ( - NodeStatusChoice, - node_role_selector, - node_type_selector, - site_selector, -) +from workflows.node.shared.forms import NodeStatusChoice, node_role_selector, node_type_selector, site_selector from workflows.node.shared.steps import update_node_in_ims from workflows.shared import create_summary_form @@ -68,14 +63,7 @@ class CreateNodeForm(FormPage): user_input = yield CreateNodeForm user_input_dict = user_input.model_dump() - summary_fields = [ - "role_id", - "type_id", - "site_id", - "node_status", - "node_name", - "node_description", - ] + summary_fields = ["role_id", "type_id", "site_id", "node_status", "node_name", "node_description"] yield from create_summary_form(user_input_dict, product_name, summary_fields) return user_input_dict @@ -106,9 +94,7 @@ def construct_node_model( subscription.node.node_name = node_name subscription.node.node_description = node_description - subscription = NodeProvisioning.from_other_lifecycle( - subscription, SubscriptionLifecycle.PROVISIONING - ) + subscription = NodeProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) subscription.description = description(subscription) return { @@ -127,8 +113,8 @@ def create_node_in_ims(subscription: NodeProvisioning) -> State: @step("Reserve loopback addresses") def reserve_loopback_addresses(subscription: NodeProvisioning) -> State: - subscription.node.ipv4_ipam_id, subscription.node.ipv6_ipam_id = ( - netbox.reserve_loopback_addresses(subscription.node.ims_id) + subscription.node.ipv4_ipam_id, subscription.node.ipv6_ipam_id = netbox.reserve_loopback_addresses( + subscription.node.ims_id ) return {"subscription": subscription} diff --git a/workflows/port/create_port.py b/workflows/port/create_port.py index be96008..beb7875 100644 --- a/workflows/port/create_port.py +++ b/workflows/port/create_port.py @@ -56,9 +56,7 @@ class SelectNodeForm(FormPage): _product = get_product_by_id(product) speed = int(_product.fixed_input_value("speed")) - FreePortChoice: TypeAlias = cast( - type[Choice], free_port_selector(node_subscription_id, speed) - ) + FreePortChoice: TypeAlias = cast(type[Choice], free_port_selector(node_subscription_id, speed)) class CreatePortForm(FormPage): model_config = ConfigDict(title=product_name) @@ -76,13 +74,7 @@ class CreatePortForm(FormPage): user_input = yield CreatePortForm user_input_dict = user_input.model_dump() - summary_fields = [ - "port_ims_id", - "port_description", - "port_mode", - "auto_negotiation", - "lldp", - ] + summary_fields = ["port_ims_id", "port_description", "port_mode", "auto_negotiation", "lldp"] yield from create_summary_form(user_input_dict, product_name, summary_fields) return user_input_dict | {"node_subscription_id": node_subscription_id} @@ -115,9 +107,7 @@ def construct_port_model( subscription.port.enabled = False subscription.port.ims_id = port_ims_id - subscription = PortProvisioning.from_other_lifecycle( - subscription, SubscriptionLifecycle.PROVISIONING - ) + subscription = PortProvisioning.from_other_lifecycle(subscription, SubscriptionLifecycle.PROVISIONING) subscription.description = description(subscription) return { diff --git a/workflows/shared.py b/workflows/shared.py index 5c8dff4..06f88e7 100644 --- a/workflows/shared.py +++ b/workflows/shared.py @@ -34,18 +34,12 @@ Vlan = Annotated[int, Ge(2), Le(4094), doc("VLAN ID.")] -AllowedNumberOfL2vpnPorts = Annotated[ - int, Ge(2), Le(8), doc("Allowed number of L2vpn ports.") -] +AllowedNumberOfL2vpnPorts = Annotated[int, Ge(2), Le(8), doc("Allowed number of L2vpn ports.")] -AllowedNumberOfNsistpPorts = Annotated[ - int, Ge(1), Le(1), doc("Allowed number of Nsistp ports.") -] +AllowedNumberOfNsistpPorts = Annotated[int, Ge(1), Le(1), doc("Allowed number of Nsistp ports.")] -def subscriptions_by_product_type( - product_type: str, status: List[SubscriptionLifecycle] -) -> List[SubscriptionTable]: +def subscriptions_by_product_type(product_type: str, status: List[SubscriptionLifecycle]) -> List[SubscriptionTable]: """ retrieve_subscription_list_by_product This function lets you retreive a list of all subscriptions of a given product type. For example, you could @@ -76,10 +70,7 @@ def subscriptions_by_product_type( def subscriptions_by_product_type_and_instance_value( - product_type: str, - resource_type: str, - value: str, - status: List[SubscriptionLifecycle], + product_type: str, resource_type: str, value: str, status: List[SubscriptionLifecycle] ) -> List[SubscriptionTable]: """Retrieve a list of Subscriptions by product_type, resource_type and value. @@ -106,35 +97,25 @@ def subscriptions_by_product_type_and_instance_value( def node_selector(enum: str = "NodesEnum") -> type[Choice]: - node_subscriptions = subscriptions_by_product_type( - "Node", [SubscriptionLifecycle.ACTIVE] - ) + node_subscriptions = subscriptions_by_product_type("Node", [SubscriptionLifecycle.ACTIVE]) nodes = { str(subscription.subscription_id): subscription.description - for subscription in sorted( - node_subscriptions, key=lambda node: node.description - ) + for subscription in sorted(node_subscriptions, key=lambda node: node.description) } return Choice(enum, zip(nodes.keys(), nodes.items())) # type:ignore -def free_port_selector( - node_subscription_id: UUIDstr, speed: int, enum: str = "PortsEnum" -) -> type[Choice]: +def free_port_selector(node_subscription_id: UUIDstr, speed: int, enum: str = "PortsEnum") -> type[Choice]: node = Node.from_subscription(node_subscription_id) interfaces = { str(interface.id): interface.name - for interface in netbox.get_interfaces( - device=node.node.node_name, speed=speed * 1000, enabled=False - ) + for interface in netbox.get_interfaces(device=node.node.node_name, speed=speed * 1000, enabled=False) } return Choice(enum, zip(interfaces.keys(), interfaces.items())) # type:ignore def summary_form(product_name: str, summary_data: SummaryData) -> Generator: - ProductSummary: TypeAlias = cast( - type[MigrationSummary], migration_summary(summary_data) - ) + ProductSummary: TypeAlias = cast(type[MigrationSummary], migration_summary(summary_data)) class SummaryForm(FormPage): model_config = ConfigDict(title=f"{product_name} summary") @@ -144,23 +125,17 @@ class SummaryForm(FormPage): yield SummaryForm -def create_summary_form( - user_input: dict, product_name: str, fields: List[str] -) -> Generator: +def create_summary_form(user_input: dict, product_name: str, fields: List[str]) -> Generator: columns = [[str(user_input[nm]) for nm in fields]] yield from summary_form(product_name, SummaryData(labels=fields, columns=columns)) # type: ignore -def modify_summary_form( - user_input: dict, block: ProductBlockModel, fields: List[str] -) -> Generator: +def modify_summary_form(user_input: dict, block: ProductBlockModel, fields: List[str]) -> Generator: before = [str(getattr(block, nm)) for nm in fields] # type: ignore[attr-defined] after = [str(user_input[nm]) for nm in fields] yield from summary_form( block.subscription.product.name, - SummaryData( - labels=fields, headers=["Before", "After"], columns=[before, after] - ), + SummaryData(labels=fields, headers=["Before", "After"], columns=[before, after]), ) From 548f0807675a92de059aadc303bbfba0c2ff8be7 Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 12:57:10 +0200 Subject: [PATCH 14/23] Revert "Removed unused exceptions" This reverts commit 0a9c1cc0ded1fbd4c6f922631262d688fa5d67c9. At least PortsValueError is needed ... --- utils/exceptions.py | 111 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/utils/exceptions.py b/utils/exceptions.py index f672132..6da2ac9 100644 --- a/utils/exceptions.py +++ b/utils/exceptions.py @@ -14,13 +14,124 @@ """Provides ValueError Exception classes.""" +class AllowedStatusValueError(ValueError): + pass + + +class ASNValueError(ValueError): + pass + + +class BGPPolicyValueError(ValueError): + pass + + +class BlackHoleCommunityValueError(ValueError): + pass + + +class ChoiceValueError(ValueError): + pass + + class DuplicateValueError(ValueError): pass +class EndpointTypeValueError(ValueError): + pass + + class FieldValueError(ValueError): pass +class FreeSpaceValueError(ValueError): + pass + + +class InSyncValueError(ValueError): + pass + + +class IPAddressValueError(ValueError): + pass + + +class IPPrefixValueError(ValueError): + pass + + +class LocationValueError(ValueError): + pass + + +class NodesValueError(ValueError): + pass + + +class CustomerValueError(ValueError): + pass + + +class PeeringValueError(ValueError): + pass + + +class PeerGroupNameError(ValueError): + pass + + +class PeerNameValueError(ValueError): + pass + + +class PeerPortNameValueError(ValueError): + pass + + +class PeerPortValueError(ValueError): + pass + + +class PortsModeValueError(ValueError): + def __init__(self, mode: str = "", message: str = ""): + super().__init__(message) + self.message = message + self.mode = mode + + +class PortsValueError(ValueError): + pass + + +class ProductValueError(ValueError): + pass + + +class ServicesActiveValueError(ValueError): + pass + + +class SubscriptionTypeValueError(ValueError): + pass + + +class UnsupportedSpeedValueError(ValueError): + pass + + +class UnsupportedTypeValueError(ValueError): + pass + + +class VlanRetaggingValueError(ValueError): + pass + + class VlanValueError(ValueError): pass + + +class InUseByAzError(ValueError): + pass From 7b28c3f8bff3e9fa880f6e9821d921afc7a8c23f Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 20:55:35 +0200 Subject: [PATCH 15/23] move flake8 options from pyproject.toml to .flake8 where they are recognized --- .flake8 | 2 ++ pyproject.toml | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6deafc2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/pyproject.toml b/pyproject.toml index 38964c1..356f2e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,6 @@ line-length = 120 [tool.isort] profile = "black" -[tool.flake8] -max-line-length = 120 - [tool.ruff] # Exclude a variety of commonly ignored directories. exclude = [ From c72c45a10f661a11cdb0dee4d375864d5de065e0 Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 20:57:49 +0200 Subject: [PATCH 16/23] make nsistp description uniform with other descriptions + add node name --- products/services/description.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/products/services/description.py b/products/services/description.py index e54ff2d..df6f760 100644 --- a/products/services/description.py +++ b/products/services/description.py @@ -84,10 +84,10 @@ def _(l2vpn: L2vpnProvisioning) -> str: @description.register def _(nsistp: NsistpProvisioning) -> str: - product_tag = nsistp.product.tag - stp_id = nsistp.nsistp.stp_id - topology = nsistp.nsistp.topology - - service_speed = nsistp.nsistp.bandwidth - - return f"{product_tag} {stp_id} {topology} {service_speed}" + return ( + f"{nsistp.product.tag} " + f"{nsistp.nsistp.stp_id} " + f"topology {nsistp.nsistp.topology} " + f"{nsistp.nsistp.sap.port.node.node_name} " + f"{nsistp.nsistp.bandwidth} Mbit/s" + ) From 4963ef6189327d1907dc44f318db7ac92304267b Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 21:02:27 +0200 Subject: [PATCH 17/23] refactor input form field used to get port --- workflows/nsistp/create_nsistp.py | 29 +++++++++++--------------- workflows/nsistp/shared/forms.py | 34 ++++++++++--------------------- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index 571f995..ec0767c 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -3,7 +3,7 @@ import uuid -from typing import Annotated +from typing import Annotated, TypeAlias, cast import structlog from orchestrator.forms import FormPage @@ -15,6 +15,7 @@ from orchestrator.workflows.utils import create_workflow from pydantic import AfterValidator, ConfigDict, model_validator from pydantic_forms.types import FormGenerator, State, UUIDstr +from pydantic_forms.validators import Choice from products.product_types.nsistp import NsistpInactive, NsistpProvisioning from products.services.description import description @@ -27,7 +28,7 @@ StpId, Topology, nsistp_fill_sap, - ports_selector, + port_selector, validate_both_aliases_empty_or_not, ) from workflows.nsistp.shared.vlan import ( @@ -38,8 +39,11 @@ logger = structlog.get_logger(__name__) +PortChoiceList: TypeAlias = cast(type[Choice], port_selector()) + def initial_input_form_generator(product_name: str) -> FormGenerator: + class CreateNsiStpForm(FormPage): model_config = ConfigDict(title=product_name) @@ -47,10 +51,7 @@ class CreateNsiStpForm(FormPage): nsistp_settings: Label - subscription_id: Annotated[ - UUIDstr, - ports_selector(), - ] + port: PortChoiceList vlan: Annotated[ int, AfterValidator(validate_vlan), @@ -76,7 +77,7 @@ def validate_is_alias_in_out(self) -> "CreateNsiStpForm": user_input_dict = user_input.dict() summary_fields = [ - "subscription_id", + "port", "vlan", "topology", "stp_id", @@ -95,7 +96,7 @@ def validate_is_alias_in_out(self) -> "CreateNsiStpForm": def construct_nsistp_model( product: UUIDstr, # customer_id: UUIDstr, - subscription_id, + port: UUIDstr, vlan: int, topology: str, stp_id: str, @@ -121,11 +122,9 @@ def construct_nsistp_model( # TODO: change to support CustomVlanRanges vlan_int = int(vlan) if not isinstance(vlan, int) else vlan - nsistp_fill_sap(nsistp, subscription_id, vlan_int) + nsistp_fill_sap(nsistp, port, vlan_int) - nsistp = NsistpProvisioning.from_other_lifecycle( - nsistp, SubscriptionLifecycle.PROVISIONING - ) + nsistp = NsistpProvisioning.from_other_lifecycle(nsistp, SubscriptionLifecycle.PROVISIONING) nsistp.description = description(nsistp) return { @@ -146,11 +145,7 @@ def ims_create_vlans(subscription: NsistpProvisioning) -> State: additional_steps = begin -@create_workflow( - "Create nsistp", - initial_input_form=initial_input_form_generator, - additional_steps=additional_steps, -) +@create_workflow("Create nsistp", initial_input_form=initial_input_form_generator, additional_steps=additional_steps) def create_nsistp() -> StepList: return ( begin diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index 0c38d50..e65fb92 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -45,18 +45,18 @@ TOPOLOGY_REGEX = r"^[-a-z0-9+,.;=_]+$" STP_ID_REGEX = r"^[-a-z0-9+,.;=_:]+$" NURN_REGEX = r"^urn:ogf:network:([^:]+):([0-9]+):([a-z0-9+,-.:;_!$()*@~&]*)$" -FQDN_REQEX = r"^(?!.{255}|.{253}[^.])([a-z0-9](?:[-a-z-0-9]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?[.]?$" +FQDN_REQEX = ( + r"^(?!.{255}|.{253}[^.])([a-z0-9](?:[-a-z-0-9]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?[.]?$" +) -def ports_selector() -> type[list[Choice]]: +def port_selector() -> type[Choice]: port_subscriptions = subscriptions_by_product_type_and_instance_value( "Port", "port_mode", PortMode.TAGGED, [SubscriptionLifecycle.ACTIVE] ) ports = { str(subscription.subscription_id): subscription.description - for subscription in sorted( - port_subscriptions, key=lambda port: port.description - ) + for subscription in sorted(port_subscriptions, key=lambda port: port.description) } return Choice("Port", zip(ports.keys(), ports.items())) @@ -137,9 +137,7 @@ def _get_nsistp_subscriptions(subscription_id: UUID | None) -> Iterator[Nsistp]: return (Nsistp.from_subscription(subscription_id) for subscription_id in result) -def validate_stp_id_uniqueness( - subscription_id: UUID | None, stp_id: str, info: ValidationInfo -) -> str: +def validate_stp_id_uniqueness(subscription_id: UUID | None, stp_id: str, info: ValidationInfo) -> str: values = info.data customer_id = values.get("customer_id") @@ -155,20 +153,14 @@ def is_not_unique(nsistp: Nsistp) -> bool: subscriptions = _get_nsistp_subscriptions(subscription_id) if any(is_not_unique(nsistp) for nsistp in subscriptions): - raise DuplicateValueError( - f"STP identifier `{stp_id}` already exists for topology `{topology}`" - ) + raise DuplicateValueError(f"STP identifier `{stp_id}` already exists for topology `{topology}`") return stp_id -def validate_both_aliases_empty_or_not( - is_alias_in: str | None, is_alias_out: str | None -) -> None: +def validate_both_aliases_empty_or_not(is_alias_in: str | None, is_alias_out: str | None) -> None: if bool(is_alias_in) != bool(is_alias_out): - raise FieldValueError( - "NSI inbound and outbound isAlias should either both have a value or be empty" - ) + raise FieldValueError("NSI inbound and outbound isAlias should either both have a value or be empty") def validate_nurn(nurn: str | None) -> str | None: @@ -180,13 +172,9 @@ def validate_nurn(nurn: str | None) -> str | None: return nurn -def nsistp_fill_sap( - subscription: NsistpInactive, subscription_id: UUIDstr, vlan: CustomVlanRanges | int -) -> None: +def nsistp_fill_sap(subscription: NsistpInactive, subscription_id: UUIDstr, vlan: CustomVlanRanges | int) -> None: subscription.nsistp.sap.vlan = vlan - subscription.nsistp.sap.port = SubscriptionModel.from_subscription( - subscription_id - ).port # type: ignore + subscription.nsistp.sap.port = SubscriptionModel.from_subscription(subscription_id).port # type: ignore def merge_uniforms(schema: dict[str, Any], *, to_merge: dict[str, Any]) -> None: From 2bccc6a028d8a374615a49f3285001ed9765f4aa Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 21:06:03 +0200 Subject: [PATCH 18/23] add form input field validation to modify nsistp + fix subscription description --- workflows/nsistp/modify_nsistp.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/workflows/nsistp/modify_nsistp.py b/workflows/nsistp/modify_nsistp.py index a0e1b8f..3a64f2c 100644 --- a/workflows/nsistp/modify_nsistp.py +++ b/workflows/nsistp/modify_nsistp.py @@ -1,7 +1,6 @@ # workflows/nsistp/modify_nsistp.py import structlog -from orchestrator.domain import SubscriptionModel from orchestrator.forms import FormPage from orchestrator.forms.validators import Divider from orchestrator.types import SubscriptionLifecycle @@ -12,16 +11,10 @@ from pydantic_forms.validators import read_only_field from products.product_types.nsistp import Nsistp, NsistpProvisioning +from products.services.description import description +from workflows.nsistp.shared.forms import IsAlias, ServiceSpeed, StpDescription, Topology from workflows.shared import modify_summary_form - -def subscription_description(subscription: SubscriptionModel) -> str: - """The suggested pattern is to implement a subscription service that generates a subscription specific - description, in case that is not present the description will just be set to the product name. - """ - return f"{subscription.product.name} subscription" - - logger = structlog.get_logger(__name__) @@ -34,12 +27,12 @@ class ModifyNsistpForm(FormPage): divider_1: Divider - topology: str = nsistp.topology - stp_description: str | None = nsistp.stp_description - is_alias_in: str | None = nsistp.is_alias_in - is_alias_out: str | None = nsistp.is_alias_out + topology: Topology = nsistp.topology + stp_description: StpDescription | None = nsistp.stp_description + is_alias_in: IsAlias | None = nsistp.is_alias_in + is_alias_out: IsAlias | None = nsistp.is_alias_out expose_in_topology: bool | None = nsistp.expose_in_topology - bandwidth: int | None = nsistp.bandwidth + bandwidth: ServiceSpeed | None = nsistp.bandwidth user_input = yield ModifyNsistpForm user_input_dict = user_input.dict() @@ -80,18 +73,14 @@ def update_subscription( @step("Update subscription description") def update_subscription_description(subscription: Nsistp) -> State: - subscription.description = subscription_description(subscription) + subscription.description = description(subscription) return {"subscription": subscription} additional_steps = begin -@modify_workflow( - "Modify nsistp", - initial_input_form=initial_input_form_generator, - additional_steps=additional_steps, -) +@modify_workflow("Modify nsistp", initial_input_form=initial_input_form_generator, additional_steps=additional_steps) def modify_nsistp() -> StepList: return ( begin From 35290f393010dc1bfadb854304d43843f0cfa6ed Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 21:08:07 +0200 Subject: [PATCH 19/23] remove unneeded AllowedNumberOfNsistpPorts --- workflows/shared.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflows/shared.py b/workflows/shared.py index 06f88e7..50381f4 100644 --- a/workflows/shared.py +++ b/workflows/shared.py @@ -36,8 +36,6 @@ AllowedNumberOfL2vpnPorts = Annotated[int, Ge(2), Le(8), doc("Allowed number of L2vpn ports.")] -AllowedNumberOfNsistpPorts = Annotated[int, Ge(1), Le(1), doc("Allowed number of Nsistp ports.")] - def subscriptions_by_product_type(product_type: str, status: List[SubscriptionLifecycle]) -> List[SubscriptionTable]: """ From 846c106eaefa58b7dc43791a421fd5aabb735b10 Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 21:08:46 +0200 Subject: [PATCH 20/23] reformat with line length set to 120 --- workflows/nsistp/shared/shared.py | 9 ++------ workflows/nsistp/shared/vlan.py | 33 ++++++++-------------------- workflows/nsistp/terminate_nsistp.py | 7 +++--- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/workflows/nsistp/shared/shared.py b/workflows/nsistp/shared/shared.py index 95683d9..f20c6b0 100644 --- a/workflows/nsistp/shared/shared.py +++ b/workflows/nsistp/shared/shared.py @@ -43,15 +43,10 @@ class CustomVlanRanges(VlanRanges): def __str__(self) -> str: # `range` objects have an exclusive `stop`. VlanRanges is expressed using terms that use an inclusive stop, # which is one less then the exclusive one we use for the internal representation. Hence the `-1` - return ", ".join( - str(vr.start) if len(vr) == 1 else f"{vr.start}-{vr.stop - 1}" - for vr in self._vlan_ranges - ) + return ", ".join(str(vr.start) if len(vr) == 1 else f"{vr.start}-{vr.stop - 1}" for vr in self._vlan_ranges) @classmethod - def __get_pydantic_json_schema__( - cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: + def __get_pydantic_json_schema__(cls, core_schema_: CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: parent_schema = super().__get_pydantic_json_schema__(core_schema_, handler) parent_schema["format"] = "custom-vlan" diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index babd43f..365362a 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -50,18 +50,14 @@ def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRan if not subscription_id: return vlan - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) + subscription = subscriptions.get_subscription(subscription_id, model=SubscriptionTable) port_mode = _get_port_mode(subscription) if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): raise VlanValueError(f"{port_mode} {subscription.product.tag} must have a vlan") elif port_mode == PortMode.UNTAGGED and vlan != CustomVlanRanges(0): # noqa: RET506 - raise VlanValueError( - f"{port_mode} {subscription.product.tag} can not have a vlan" - ) + raise VlanValueError(f"{port_mode} {subscription.product.tag} can not have a vlan") return vlan @@ -93,12 +89,8 @@ def check_vlan_already_used( # Remove currently chosen vlans for this port to prevent tripping on in used by itself current_selected_vlan_ranges: list[str] = [] if current: - current_selected_service_port = filter( - lambda c: str(c["subscription_id"]) == str(subscription_id), current - ) - current_selected_vlans = list( - map(operator.itemgetter("vlan"), current_selected_service_port) - ) + current_selected_service_port = filter(lambda c: str(c["subscription_id"]) == str(subscription_id), current) + current_selected_vlans = list(map(operator.itemgetter("vlan"), current_selected_service_port)) for current_selected_vlan in current_selected_vlans: # We assume an empty string is untagged and thus 0 if not current_selected_vlan: @@ -111,9 +103,7 @@ def check_vlan_already_used( *list(current_selected_vlan_range), ] - subscription = subscriptions.get_subscription( - subscription_id, model=SubscriptionTable - ) + subscription = subscriptions.get_subscription(subscription_id, model=SubscriptionTable) # Handle both int and CustomVlanRanges if isinstance(vlan, int): @@ -128,9 +118,7 @@ def check_vlan_already_used( # for tagged only; for link_member/untagged say "SP already in use" if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: raise PortsValueError("Port already in use") - raise VlanValueError( - f"Vlan(s) {', '.join(map(str, sorted(used_vlans)))} already in use" - ) + raise VlanValueError(f"Vlan(s) {', '.join(map(str, sorted(used_vlans)))} already in use") return vlan @@ -150,18 +138,15 @@ def find_allocated_vlans( select(SubscriptionInstanceValueTable.value) .join( ResourceTypeTable, - SubscriptionInstanceValueTable.resource_type_id - == ResourceTypeTable.resource_type_id, + SubscriptionInstanceValueTable.resource_type_id == ResourceTypeTable.resource_type_id, ) .join( SubscriptionInstanceRelationTable, - SubscriptionInstanceValueTable.subscription_instance_id - == SubscriptionInstanceRelationTable.in_use_by_id, + SubscriptionInstanceValueTable.subscription_instance_id == SubscriptionInstanceRelationTable.in_use_by_id, ) .join( SubscriptionInstanceTable, - SubscriptionInstanceRelationTable.depends_on_id - == SubscriptionInstanceTable.subscription_instance_id, + SubscriptionInstanceRelationTable.depends_on_id == SubscriptionInstanceTable.subscription_instance_id, ) .filter( SubscriptionInstanceTable.subscription_id == subscription_id, diff --git a/workflows/nsistp/terminate_nsistp.py b/workflows/nsistp/terminate_nsistp.py index a7a4900..6516ee0 100644 --- a/workflows/nsistp/terminate_nsistp.py +++ b/workflows/nsistp/terminate_nsistp.py @@ -11,9 +11,7 @@ logger = structlog.get_logger(__name__) -def terminate_initial_input_form_generator( - subscription_id: UUIDstr, customer_id: UUIDstr -) -> InputForm: +def terminate_initial_input_form_generator(subscription_id: UUIDstr, customer_id: UUIDstr) -> InputForm: temp_subscription_id = subscription_id class TerminateNsistpForm(FormPage): @@ -39,6 +37,7 @@ def delete_subscription_from_oss_bss(subscription: Nsistp) -> State: ) def terminate_nsistp() -> StepList: return ( - begin >> delete_subscription_from_oss_bss + begin + >> delete_subscription_from_oss_bss # TODO: fill in additional steps if needed ) From a2cdc7a7ec338f8293543db921c0b7923aeddab0 Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 9 Oct 2025 21:28:37 +0200 Subject: [PATCH 21/23] improve resource type descriptions + fix nsistp generator yaml --- .../2025-09-30_a87d11eb8dd1_add_nsistp.py | 10 +++++----- templates/nsistp.yaml | 17 +++++++---------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py b/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py index ada9278..3c0d471 100644 --- a/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py +++ b/migrations/versions/schema/2025-09-30_a87d11eb8dd1_add_nsistp.py @@ -37,13 +37,13 @@ "tag": "NSISTP", "status": "active", "resources": { - "topology": "Topology type or identifier for the service instance", + "topology": "Name of the topology this Service Termination Point is exposed in", "stp_id": "Unique identifier for the Service Termination Point", "stp_description": "Description of the Service Termination Point", - "is_alias_in": "Indicates if the incoming SAP is an alias", - "is_alias_out": "Indicates if the outgoing SAP is an alias", - "expose_in_topology": "Whether to expose this STP in the topology view", - "bandwidth": "Requested bandwidth for the service instance (in Mbps)", + "is_alias_in": "Inbound port from the other topology in case this STP is part of a SDP", + "is_alias_out": "Outbound port from the other topology in case this STP is part of a SDP", + "expose_in_topology": "Whether to actively expose this STP in the topology", + "bandwidth": "Maximum bandwidth for the combined set of STP (in Mbps)", }, "depends_on_block_relations": [ "SAP", diff --git a/templates/nsistp.yaml b/templates/nsistp.yaml index e8510c6..ab1ac0a 100644 --- a/templates/nsistp.yaml +++ b/templates/nsistp.yaml @@ -27,14 +27,11 @@ product_blocks: description: "nsistp product block" fields: - name: sap - type: list - description: "Virtual circuit service access points" - list_type: SAP - min_items: 2 - max_items: 8 + type: SAP + description: "NSI STP service access points" required: provisioning - name: topology - description: "Topology type or identifier for the service instance" + description: "Name of the topology this Service Termination Point is exposed in" type: str required: provisioning modifiable: @@ -47,18 +44,18 @@ product_blocks: type: str modifiable: - name: is_alias_in - description: "Indicates if the incoming SAP is an alias" + description: "Unique identifier for the Service Termination Point" type: str modifiable: - name: is_alias_out - description: "Indicates if the outgoing SAP is an alias" + description: "Outbound port from the other topology in case this STP is part of a SDP" type: str modifiable: - name: expose_in_topology - description: "Whether to expose this STP in the topology view" + description: "Whether to actively expose this STP in the topology" type: bool modifiable: - name: bandwidth - description: "Requested bandwidth for the service instance (in Mbps)" + description: "Maximum bandwidth for the combined set of STP (in Mbps)" type: int modifiable: From a1ec70eba542867a53c84da4bb0ca61e440df8e7 Mon Sep 17 00:00:00 2001 From: Hans Trompert Date: Thu, 16 Oct 2025 13:05:41 +0200 Subject: [PATCH 22/23] address review done by @Mark90 --- products/product_blocks/nsistp.py | 24 +++-- products/product_blocks/port.py | 2 +- products/product_types/nsistp.py | 25 +++-- pyproject.toml | 3 +- utils/exceptions.py | 137 --------------------------- workflows/nsistp/create_nsistp.py | 39 ++++---- workflows/nsistp/modify_nsistp.py | 19 +++- workflows/nsistp/shared/__init__.py | 12 +++ workflows/nsistp/shared/forms.py | 23 ++--- workflows/nsistp/shared/shared.py | 8 +- workflows/nsistp/shared/vlan.py | 11 ++- workflows/nsistp/terminate_nsistp.py | 27 +++--- workflows/nsistp/validate_nsistp.py | 15 ++- 13 files changed, 125 insertions(+), 220 deletions(-) delete mode 100644 utils/exceptions.py diff --git a/products/product_blocks/nsistp.py b/products/product_blocks/nsistp.py index df35c4e..82c4c26 100644 --- a/products/product_blocks/nsistp.py +++ b/products/product_blocks/nsistp.py @@ -1,4 +1,16 @@ -# products/product_blocks/nsistp.py +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from orchestrator.domain.base import ProductBlockModel from orchestrator.types import SubscriptionLifecycle @@ -6,9 +18,6 @@ from products.product_blocks.sap import SAPBlock, SAPBlockInactive, SAPBlockProvisioning -# NOTE: ListOfSap is not required here??? -# ListOfSap = Annotated[list[SI], Len(min_length=2, max_length=8)] - class NsistpBlockInactive(ProductBlockModel, product_block_name="Nsistp"): sap: SAPBlockInactive @@ -21,9 +30,7 @@ class NsistpBlockInactive(ProductBlockModel, product_block_name="Nsistp"): bandwidth: int | None = None -class NsistpBlockProvisioning( - NsistpBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] -): +class NsistpBlockProvisioning(NsistpBlockInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): sap: SAPBlockProvisioning topology: str stp_id: str @@ -36,8 +43,7 @@ class NsistpBlockProvisioning( @computed_field @property def title(self) -> str: - # TODO: format correct title string - return f"{self.name}" + return f"NSISTP {self.stp_id} on {self.sap.title}" class NsistpBlock(NsistpBlockProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): diff --git a/products/product_blocks/port.py b/products/product_blocks/port.py index b1bf122..b918119 100644 --- a/products/product_blocks/port.py +++ b/products/product_blocks/port.py @@ -12,12 +12,12 @@ # limitations under the License. -from pydantic_forms.types import strEnum from typing import List from orchestrator.domain.base import ProductBlockModel from orchestrator.types import SubscriptionLifecycle from pydantic import computed_field +from pydantic_forms.types import strEnum from products.product_blocks.node import NodeBlock, NodeBlockInactive, NodeBlockProvisioning diff --git a/products/product_types/nsistp.py b/products/product_types/nsistp.py index 426e55a..d7b20ca 100644 --- a/products/product_types/nsistp.py +++ b/products/product_types/nsistp.py @@ -1,12 +1,21 @@ -# products/product_types/nsistp.py +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from orchestrator.domain.base import SubscriptionModel from orchestrator.types import SubscriptionLifecycle -from products.product_blocks.nsistp import ( - NsistpBlock, - NsistpBlockInactive, - NsistpBlockProvisioning, -) +from products.product_blocks.nsistp import NsistpBlock, NsistpBlockInactive, NsistpBlockProvisioning from workflows.nsistp.shared.shared import CustomVlanRanges @@ -14,9 +23,7 @@ class NsistpInactive(SubscriptionModel, is_base=True): nsistp: NsistpBlockInactive -class NsistpProvisioning( - NsistpInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING] -): +class NsistpProvisioning(NsistpInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): nsistp: NsistpBlockProvisioning diff --git a/pyproject.toml b/pyproject.toml index 356f2e3..18cf011 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ line-length = 120 [tool.isort] profile = "black" +line_length = 120 [tool.ruff] # Exclude a variety of commonly ignored directories. @@ -47,7 +48,7 @@ exclude = [ ] # Same as Black. -line-length = 88 +line-length = 120 indent-width = 4 # Assume Python 3.9 diff --git a/utils/exceptions.py b/utils/exceptions.py deleted file mode 100644 index 6da2ac9..0000000 --- a/utils/exceptions.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2019-2024 SURF. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides ValueError Exception classes.""" - - -class AllowedStatusValueError(ValueError): - pass - - -class ASNValueError(ValueError): - pass - - -class BGPPolicyValueError(ValueError): - pass - - -class BlackHoleCommunityValueError(ValueError): - pass - - -class ChoiceValueError(ValueError): - pass - - -class DuplicateValueError(ValueError): - pass - - -class EndpointTypeValueError(ValueError): - pass - - -class FieldValueError(ValueError): - pass - - -class FreeSpaceValueError(ValueError): - pass - - -class InSyncValueError(ValueError): - pass - - -class IPAddressValueError(ValueError): - pass - - -class IPPrefixValueError(ValueError): - pass - - -class LocationValueError(ValueError): - pass - - -class NodesValueError(ValueError): - pass - - -class CustomerValueError(ValueError): - pass - - -class PeeringValueError(ValueError): - pass - - -class PeerGroupNameError(ValueError): - pass - - -class PeerNameValueError(ValueError): - pass - - -class PeerPortNameValueError(ValueError): - pass - - -class PeerPortValueError(ValueError): - pass - - -class PortsModeValueError(ValueError): - def __init__(self, mode: str = "", message: str = ""): - super().__init__(message) - self.message = message - self.mode = mode - - -class PortsValueError(ValueError): - pass - - -class ProductValueError(ValueError): - pass - - -class ServicesActiveValueError(ValueError): - pass - - -class SubscriptionTypeValueError(ValueError): - pass - - -class UnsupportedSpeedValueError(ValueError): - pass - - -class UnsupportedTypeValueError(ValueError): - pass - - -class VlanRetaggingValueError(ValueError): - pass - - -class VlanValueError(ValueError): - pass - - -class InUseByAzError(ValueError): - pass diff --git a/workflows/nsistp/create_nsistp.py b/workflows/nsistp/create_nsistp.py index ec0767c..6cfe0b1 100644 --- a/workflows/nsistp/create_nsistp.py +++ b/workflows/nsistp/create_nsistp.py @@ -1,5 +1,15 @@ -# workflows/nsistp/create_nsistp.py -# from typing import TypeAlias, cast +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import uuid @@ -31,27 +41,22 @@ port_selector, validate_both_aliases_empty_or_not, ) -from workflows.nsistp.shared.vlan import ( - validate_vlan, - validate_vlan_not_in_use, -) +from workflows.nsistp.shared.vlan import validate_vlan, validate_vlan_not_in_use from workflows.shared import create_summary_form logger = structlog.get_logger(__name__) -PortChoiceList: TypeAlias = cast(type[Choice], port_selector()) - def initial_input_form_generator(product_name: str) -> FormGenerator: + PortChoiceList: TypeAlias = cast(type[Choice], port_selector()) class CreateNsiStpForm(FormPage): model_config = ConfigDict(title=product_name) - # customer_id: CustomerId - nsistp_settings: Label port: PortChoiceList + # TODO: change to support CustomVlanRanges vlan: Annotated[ int, AfterValidator(validate_vlan), @@ -95,7 +100,6 @@ def validate_is_alias_in_out(self) -> "CreateNsiStpForm": @step("Construct Subscription model") def construct_nsistp_model( product: UUIDstr, - # customer_id: UUIDstr, port: UUIDstr, vlan: int, topology: str, @@ -142,15 +146,6 @@ def ims_create_vlans(subscription: NsistpProvisioning) -> State: return {"subscription": subscription, "payloads": [payload]} -additional_steps = begin - - -@create_workflow("Create nsistp", initial_input_form=initial_input_form_generator, additional_steps=additional_steps) +@create_workflow("Create nsistp", initial_input_form=initial_input_form_generator) def create_nsistp() -> StepList: - return ( - begin - >> construct_nsistp_model - >> store_process_subscription(Target.CREATE) - >> ims_create_vlans - # TODO add provision step(s) - ) + return begin >> construct_nsistp_model >> store_process_subscription(Target.CREATE) >> ims_create_vlans diff --git a/workflows/nsistp/modify_nsistp.py b/workflows/nsistp/modify_nsistp.py index 3a64f2c..86ed10b 100644 --- a/workflows/nsistp/modify_nsistp.py +++ b/workflows/nsistp/modify_nsistp.py @@ -1,4 +1,16 @@ -# workflows/nsistp/modify_nsistp.py +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import structlog from orchestrator.forms import FormPage @@ -77,10 +89,7 @@ def update_subscription_description(subscription: Nsistp) -> State: return {"subscription": subscription} -additional_steps = begin - - -@modify_workflow("Modify nsistp", initial_input_form=initial_input_form_generator, additional_steps=additional_steps) +@modify_workflow("Modify nsistp", initial_input_form=initial_input_form_generator) def modify_nsistp() -> StepList: return ( begin diff --git a/workflows/nsistp/shared/__init__.py b/workflows/nsistp/shared/__init__.py index e69de29..0da72f0 100644 --- a/workflows/nsistp/shared/__init__.py +++ b/workflows/nsistp/shared/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index e65fb92..661062a 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -10,6 +10,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import re from collections.abc import Iterator from datetime import datetime @@ -19,9 +21,7 @@ from annotated_types import BaseMetadata, Ge, Le from orchestrator.db import ProductTable, db -from orchestrator.db.models import ( - SubscriptionTable, -) +from orchestrator.db.models import SubscriptionTable from orchestrator.domain.base import SubscriptionModel from orchestrator.types import SubscriptionLifecycle from pydantic import AfterValidator, Field, ValidationInfo @@ -32,14 +32,7 @@ from products.product_blocks.port import PortMode from products.product_types.nsistp import Nsistp, NsistpInactive -from utils.exceptions import ( - DuplicateValueError, - FieldValueError, -) -from workflows.nsistp.shared.shared import ( - MAX_SPEED_POSSIBLE, - CustomVlanRanges, -) +from workflows.nsistp.shared.shared import MAX_SPEED_POSSIBLE, CustomVlanRanges from workflows.shared import subscriptions_by_product_type_and_instance_value TOPOLOGY_REGEX = r"^[-a-z0-9+,.;=_]+$" @@ -118,7 +111,7 @@ def validate_regex( return field if not re.match(regex, field, re.IGNORECASE): - raise FieldValueError(f"{message} must match: {regex}") + raise ValueError(f"{message} must match: {regex}") return field @@ -153,21 +146,21 @@ def is_not_unique(nsistp: Nsistp) -> bool: subscriptions = _get_nsistp_subscriptions(subscription_id) if any(is_not_unique(nsistp) for nsistp in subscriptions): - raise DuplicateValueError(f"STP identifier `{stp_id}` already exists for topology `{topology}`") + raise ValueError(f"STP identifier `{stp_id}` already exists for topology `{topology}`") return stp_id def validate_both_aliases_empty_or_not(is_alias_in: str | None, is_alias_out: str | None) -> None: if bool(is_alias_in) != bool(is_alias_out): - raise FieldValueError("NSI inbound and outbound isAlias should either both have a value or be empty") + raise ValueError("NSI inbound and outbound isAlias should either both have a value or be empty") def validate_nurn(nurn: str | None) -> str | None: if nurn: valid, message = valid_nurn(nurn) if not valid: - raise FieldValueError(message) + raise ValueError(message) return nurn diff --git a/workflows/nsistp/shared/shared.py b/workflows/nsistp/shared/shared.py index f20c6b0..4dea7f7 100644 --- a/workflows/nsistp/shared/shared.py +++ b/workflows/nsistp/shared/shared.py @@ -10,15 +10,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + from collections.abc import Callable from enum import StrEnum from uuid import UUID import structlog from nwastdlib.vlans import VlanRanges -from orchestrator.db import ( - SubscriptionTable, -) +from orchestrator.db import SubscriptionTable from pydantic import GetJsonSchemaHandler from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema @@ -27,10 +27,10 @@ GetSubscriptionByIdFunc = Callable[[UUID], SubscriptionTable] -PORT_SPEED = "port_speed" MAX_SPEED_POSSIBLE = 400_000 +# TODO: remove unneeded PortTag class class PortTag(StrEnum): SP = "SP" AGGSP = "AGGSP" diff --git a/workflows/nsistp/shared/vlan.py b/workflows/nsistp/shared/vlan.py index 365362a..9e457da 100644 --- a/workflows/nsistp/shared/vlan.py +++ b/workflows/nsistp/shared/vlan.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import operator from uuid import UUID @@ -29,12 +30,12 @@ from sqlalchemy import select from products.product_blocks.port import PortMode -from utils.exceptions import PortsValueError, VlanValueError from workflows.nsistp.shared.shared import CustomVlanRanges, PortTag logger = structlog.get_logger(__name__) +# TODO: remove unneeded _get_port_mode() def _get_port_mode(subscription: SubscriptionTable) -> PortMode: if subscription.product.tag in [PortTag.AGGSP + PortTag.SP]: return subscription.port_mode @@ -55,9 +56,9 @@ def validate_vlan(vlan: CustomVlanRanges, info: ValidationInfo) -> CustomVlanRan port_mode = _get_port_mode(subscription) if port_mode == PortMode.TAGGED and vlan == CustomVlanRanges(0): - raise VlanValueError(f"{port_mode} {subscription.product.tag} must have a vlan") + raise ValueError(f"{port_mode} {subscription.product.tag} must have a vlan") elif port_mode == PortMode.UNTAGGED and vlan != CustomVlanRanges(0): # noqa: RET506 - raise VlanValueError(f"{port_mode} {subscription.product.tag} can not have a vlan") + raise ValueError(f"{port_mode} {subscription.product.tag} can not have a vlan") return vlan @@ -117,8 +118,8 @@ def check_vlan_already_used( # for tagged only; for link_member/untagged say "SP already in use" if port_mode == PortMode.UNTAGGED or port_mode == PortMode.LINK_MEMBER: - raise PortsValueError("Port already in use") - raise VlanValueError(f"Vlan(s) {', '.join(map(str, sorted(used_vlans)))} already in use") + raise ValueError("Port already in use") + raise ValueError(f"Vlan(s) {', '.join(map(str, sorted(used_vlans)))} already in use") return vlan diff --git a/workflows/nsistp/terminate_nsistp.py b/workflows/nsistp/terminate_nsistp.py index 6516ee0..990b29f 100644 --- a/workflows/nsistp/terminate_nsistp.py +++ b/workflows/nsistp/terminate_nsistp.py @@ -1,4 +1,17 @@ -# workflows/nsistp/terminate_nsistp.py +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import structlog from orchestrator.forms import FormPage from orchestrator.forms.validators import DisplaySubscription @@ -27,17 +40,9 @@ def delete_subscription_from_oss_bss(subscription: Nsistp) -> State: return {} -additional_steps = begin - - -@terminate_workflow( - "Terminate nsistp", - initial_input_form=terminate_initial_input_form_generator, - additional_steps=additional_steps, -) +@terminate_workflow("Terminate nsistp", initial_input_form=terminate_initial_input_form_generator) def terminate_nsistp() -> StepList: return ( - begin - >> delete_subscription_from_oss_bss + begin >> delete_subscription_from_oss_bss # TODO: fill in additional steps if needed ) diff --git a/workflows/nsistp/validate_nsistp.py b/workflows/nsistp/validate_nsistp.py index d863a49..451aa5d 100644 --- a/workflows/nsistp/validate_nsistp.py +++ b/workflows/nsistp/validate_nsistp.py @@ -1,4 +1,17 @@ -# workflows/nsistp/validate_nsistp.py +# Copyright 2019-2023 SURF. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import structlog from orchestrator.workflow import StepList, begin, step from orchestrator.workflows.utils import validate_workflow From 443f40c10caf020d14b3e9cc4a2b5ad330131bcf Mon Sep 17 00:00:00 2001 From: tvdven Date: Fri, 17 Oct 2025 13:39:29 +0200 Subject: [PATCH 23/23] Processed comment Mark --- workflows/nsistp/shared/forms.py | 37 +++++++++++--------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/workflows/nsistp/shared/forms.py b/workflows/nsistp/shared/forms.py index 661062a..3b4f9e0 100644 --- a/workflows/nsistp/shared/forms.py +++ b/workflows/nsistp/shared/forms.py @@ -41,6 +41,12 @@ FQDN_REQEX = ( r"^(?!.{255}|.{253}[^.])([a-z0-9](?:[-a-z-0-9]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?[.]?$" ) +VALID_DATE_FORMATS = { + 4: '%Y', + 6: '%Y%m', + 8: '%Y%m%d', +} + def port_selector() -> type[Choice]: @@ -59,33 +65,14 @@ def is_fqdn(hostname: str) -> bool: def valid_date(date: str) -> tuple[bool, str | None]: - def valid_month() -> tuple[bool, str | None]: - month_str = date[4:6] - month = int(month_str) - if month < 1 or month > 12: - return False, f"{month_str} is not a valid month number" - return True, None + if not (date_format := VALID_DATE_FORMATS.get(len(date))): + return False, f"Invalid date length, expected one of: {list(VALID_DATE_FORMATS)}" - def valid_day() -> tuple[bool, str | None]: - try: - datetime.fromisoformat(f"{date[0:4]}-{date[4:6]}-{date[6:8]}") - except ValueError: - return False, f"`{date}` is not a valid date" + try: + _ = datetime.strptime(date, date_format) return True, None - - length = len(date) - if length == 4: # year - pass # No checks on reasonable year, so 9999 is allowed - elif length in (6, 8): - valid, message = valid_month() - if not valid: - return valid, message - if length == 8: # year + month + day - return valid_day() - else: - return False, f"date `{date}` has invalid length" - - return True, None + except ValueError as exc: + return False, f"Invalid date for format {date_format!r}: {exc}" def valid_nurn(nurn: str) -> tuple[bool, str | None]: