Closes #3799: Remove NaturalOrderingManager

docs/release-notes/version-2.7.md

@@ -2,6 +2,7 @@
 
 ## Enhancements
 
+* [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components
 * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views
 * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components
 * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views

netbox/dcim/managers.py

@@ -1,18 +1,7 @@
 from django.db.models import Manager, QuerySet
-from django.db.models.expressions import RawSQL
 
 from .constants import NONCONNECTABLE_IFACE_TYPES
 
-# Regular expressions for parsing Interface names
-TYPE_RE = r"SUBSTRING({} FROM '^([^0-9\.:]+)')"
-SLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(\d{{1,9}})/') AS integer), NULL)"
-SUBSLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?\d{{1,9}}/(\d{{1,9}})') AS integer), NULL)"
-POSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{2}}(\d{{1,9}})') AS integer), NULL)"
-SUBPOSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{3}}(\d{{1,9}})') AS integer), NULL)"
-ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?(\d{{1,9}})([^/]|$)') AS integer)"
-CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*:(\d{{1,9}})(\.\d{{1,9}})?$') AS integer), 0)"
-VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*\.(\d{{1,9}})$') AS integer), 0)"
-
 
 class InterfaceQuerySet(QuerySet):
 
@@ -27,47 +16,4 @@ def connectable(self):
 class InterfaceManager(Manager):
 
     def get_queryset(self):
-        """
-        Naturally order interfaces by their type and numeric position. To order interfaces naturally, the `name` field
-        is split into eight distinct components: leading text (type), slot, subslot, position, subposition, ID, channel,
-        and virtual circuit:
-
-            {type}{slot or ID}/{subslot}/{position}/{subposition}:{channel}.{vc}
-
-        Components absent from the interface name are coalesced to zero or null. For example, an interface named
-        GigabitEthernet1/2/3 would be parsed as follows:
-
-            type = 'GigabitEthernet'
-            slot =  1
-            subslot = 2
-            position = 3
-            subposition = None
-            id = None
-            channel = 0
-            vc = 0
-
-        The original `name` field is considered in its entirety to serve as a fallback in the event interfaces do not
-        match any of the prescribed fields.
-
-        The `id` field is included to enforce deterministic ordering of interfaces in similar vein of other device
-        components.
-        """
-
-        sql_col = '{}.name'.format(self.model._meta.db_table)
-        ordering = [
-            '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name', 'pk'
-
-        ]
-
-        fields = {
-            '_type': RawSQL(TYPE_RE.format(sql_col), []),
-            '_id': RawSQL(ID_RE.format(sql_col), []),
-            '_slot': RawSQL(SLOT_RE.format(sql_col), []),
-            '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []),
-            '_position': RawSQL(POSITION_RE.format(sql_col), []),
-            '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []),
-            '_channel': RawSQL(CHANNEL_RE.format(sql_col), []),
-            '_vc': RawSQL(VC_RE.format(sql_col), []),
-        }
-
-        return InterfaceQuerySet(self.model, using=self._db).annotate(**fields).order_by(*ordering)
+        return InterfaceQuerySet(self.model, using=self._db)

netbox/dcim/migrations/0093_device_component_ordering.py

@@ -0,0 +1,147 @@
+from django.db import migrations
+import utilities.fields
+import utilities.ordering
+
+
+def _update_model_names(model):
+    # Update each unique field value in bulk
+    for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name))
+
+
+def naturalize_consoleports(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'ConsolePort'))
+
+
+def naturalize_consoleserverports(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'ConsoleServerPort'))
+
+
+def naturalize_powerports(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'PowerPort'))
+
+
+def naturalize_poweroutlets(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'PowerOutlet'))
+
+
+def naturalize_frontports(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'FrontPort'))
+
+
+def naturalize_rearports(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'RearPort'))
+
+
+def naturalize_devicebays(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'DeviceBay'))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dcim', '0092_fix_rack_outer_unit'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='consoleport',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='consoleserverport',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='devicebay',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='frontport',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='inventoryitem',
+            options={'ordering': ('device__id', 'parent__id', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='poweroutlet',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='powerport',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='rearport',
+            options={'ordering': ('device', '_name')},
+        ),
+        migrations.AddField(
+            model_name='consoleport',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='consoleserverport',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='devicebay',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='frontport',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='inventoryitem',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='poweroutlet',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='powerport',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='rearport',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.RunPython(
+            code=naturalize_consoleports,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_consoleserverports,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_powerports,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_poweroutlets,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_frontports,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_rearports,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_devicebays,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

netbox/dcim/migrations/0094_device_component_template_ordering.py

@@ -0,0 +1,138 @@
+from django.db import migrations
+import utilities.fields
+import utilities.ordering
+
+
+def _update_model_names(model):
+    # Update each unique field value in bulk
+    for name in model.objects.values_list('name', flat=True).order_by('name').distinct():
+        model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name))
+
+
+def naturalize_consoleporttemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'ConsolePortTemplate'))
+
+
+def naturalize_consoleserverporttemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'ConsoleServerPortTemplate'))
+
+
+def naturalize_powerporttemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'PowerPortTemplate'))
+
+
+def naturalize_poweroutlettemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'PowerOutletTemplate'))
+
+
+def naturalize_frontporttemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'FrontPortTemplate'))
+
+
+def naturalize_rearporttemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'RearPortTemplate'))
+
+
+def naturalize_devicebaytemplates(apps, schema_editor):
+    _update_model_names(apps.get_model('dcim', 'DeviceBayTemplate'))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dcim', '0093_device_component_ordering'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='consoleporttemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='consoleserverporttemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='devicebaytemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='frontporttemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='poweroutlettemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='powerporttemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AlterModelOptions(
+            name='rearporttemplate',
+            options={'ordering': ('device_type', '_name')},
+        ),
+        migrations.AddField(
+            model_name='consoleporttemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='consoleserverporttemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='devicebaytemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='frontporttemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='poweroutlettemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='powerporttemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.AddField(
+            model_name='rearporttemplate',
+            name='_name',
+            field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize),
+        ),
+        migrations.RunPython(
+            code=naturalize_consoleporttemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_consoleserverporttemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_powerporttemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_poweroutlettemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_frontporttemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_rearporttemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+        migrations.RunPython(
+            code=naturalize_devicebaytemplates,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]