[IM] Improve speed of Peering Manager

On INEX today, this means ~75secs -> 2secs // we've added "bs.timestamp >= NOW() - INTERVAL 7 DAY" below as we don't // dump old date (yet) and the time to run the query is O(n) on number // of rows... // also: CREATE INDEX idx_timestamp ON bgpsessiondata (timestamp)
inex · Jan 19, 2016 · 9dbc314 · 9dbc314
1 parent 1a66ec6
commit 9dbc314
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 21 deletions.
diff --git a/database/IXP-Manager.ormdesigner2 b/database/IXP-Manager.ormdesigner2
@@ -346,6 +346,9 @@
       <field name="packetcount" type="integer" default="0" uuid="eb604e5d-18c0-4050-83c5-6a837d6b4b5d"/>
       <field name="timestamp" type="datetime" uuid="c71b199c-d25d-4208-9b64-7fff911b9b87"/>
       <field name="source" type="string" size="40" uuid="3ea5f9ae-e383-4532-9edf-ce430bd93255"/>
+      <index name="idx_timestamp">
+        <index-field name="timestamp"/>
+      </index>
       <orm-attributes>
         <attribute name="repository-class">Repositories\BGPSessionData</attribute>
         <attribute name="table">bgpsessiondata</attribute>
@@ -1162,7 +1165,7 @@
     <entity uuid="04f83672-40da-4802-ad22-809cfd3ab549" bg-color="4294967295" hdr-color="4292006610" position-x="250" position-y="1009" size-x="0" size-x2="97" size-y="0" size-y2="73"/>
     <entity uuid="0617cfe2-0a29-4671-a9d6-5401bc911e43" bg-color="#FFFFFF" hdr-color="#D2D2D2" position-x="595" position-y="350" size-x="0" size-x2="186" size-y="0" size-y2="465"/>
     <entity uuid="09ab0078-0545-4696-a0cf-f825906ff6b8" bg-color="4294967295" hdr-color="4292006610" position-x="802" position-y="145" size-x="0" size-x2="183" size-y="0" size-y2="143"/>
-    <entity uuid="0aaacba6-6bf1-4b5f-b3ad-41f36ca7bd7d" bg-color="4294967295" hdr-color="4292006610" position-x="35" position-y="36" size-x="0" size-x2="117" size-y="0" size-y2="101"/>
+    <entity uuid="0aaacba6-6bf1-4b5f-b3ad-41f36ca7bd7d" bg-color="#FFFFFF" hdr-color="#D2D2D2" position-x="35" position-y="36" size-x="0" size-x2="117" size-y="0" size-y2="101"/>
     <entity uuid="0be9b477-a199-4d85-a892-4287cf858f4d" bg-color="#FFFFFF" hdr-color="#D2D2D2" position-x="47" position-y="-145" size-x="0" size-x2="124" size-y="0" size-y2="59"/>
     <entity uuid="0c1cb743-444a-4fea-ab68-7150272fc55b" bg-color="4294967295" hdr-color="4292006610" position-x="5" position-y="417" size-x="0" size-x2="123" size-y="0" size-y2="115"/>
     <entity uuid="13cd2e44-8bf8-41d0-b105-7ae62dc1d189" bg-color="4294967295" hdr-color="4292006610" position-x="1417" position-y="16" size-x="0" size-x2="119" size-y="0" size-y2="129"/>

diff --git a/database/Proxies/__CG__EntitiesCustomer.php b/database/Proxies/__CG__EntitiesCustomer.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace Proxies\__CG__\Entities;
+namespace DoctrineProxies\__CG__\Entities;
 
 /**
  * DO NOT EDIT THIS FILE - IT WAS CREATED BY DOCTRINE'S PROXY GENERATOR

diff --git a/database/Proxies/__CG__EntitiesUser.php b/database/Proxies/__CG__EntitiesUser.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace Proxies\__CG__\Entities;
+namespace DoctrineProxies\__CG__\Entities;
 
 /**
  * DO NOT EDIT THIS FILE - IT WAS CREATED BY DOCTRINE'S PROXY GENERATOR

diff --git a/database/Repositories/BGPSessionData.php b/database/Repositories/BGPSessionData.php
@@ -59,24 +59,29 @@ public function getPeers( $vlan = null, $protocol = 6, $asn = null, $forceDb = f
 
         if( !$forceDb && ( $apeers = \Zend_Registry::get( 'd2cache' )->fetch( $key ) ) )
             return $apeers;
-        
+
         if( !in_array( $protocol, [ 4, 6 ] ) )
             throw new \IXP_Exception( 'Invalid protocol' );
-        
+
         if( $vlan !== null && !( $evlan = $this->getEntityManager()->getRepository( '\Entities\Vlan' )->find( $vlan ) ) )
             throw new \IXP_Exception( 'Invalid VLAN' );
-            
+
         $conn = $this->getEntityManager()->getConnection();
         $conn->setFetchMode( \PDO::FETCH_ASSOC );
-
+
+        // we've added "bs.timestamp >= NOW() - INTERVAL 7 DAY" below as we don't
+        // dump old date (yet) and the time to run the query is O(n) on number
+        // of rows...
+        // also: CREATE INDEX idx_timestamp ON bgpsessiondata (timestamp)
+
         // need to construct a raw SQL here due to the schema design by NH
         $sql = "SELECT bs.*, srcip.*, dstip.*,
             vlis.virtualinterfaceid as visid, vlid.virtualinterfaceid as vidid,
             cs.shortname AS csshortname, cs.name AS csname, cs.autsys AS csautsys,
             cd.shortname AS cdshortname, cd.name AS cdname, cd.autsys AS cdautsys,
             vlan.id AS vlanid, vlan.name AS vlanname, vlan.number AS vlantag,
             COUNT( bs.packetcount ) AS packetcount
-                
+
             FROM bgpsessiondata AS bs
                 LEFT JOIN ipv{$protocol}address AS srcip ON bs.srcipaddressid = srcip.id
                 LEFT JOIN ipv{$protocol}address AS dstip ON bs.dstipaddressid = dstip.id
@@ -87,23 +92,24 @@ public function getPeers( $vlan = null, $protocol = 6, $asn = null, $forceDb = f
                 LEFT JOIN cust AS cs ON vis.custid = cs.id
                 LEFT JOIN cust AS cd ON vid.custid = cd.id
                 LEFT JOIN vlan AS vlan ON vlan.number = bs.vlan
-         
+
         WHERE
-            bs.protocol = {$protocol}
+            bs.timestamp >= NOW() - INTERVAL 7 DAY
+            AND bs.protocol = {$protocol}
             AND packetcount >= 1";
-        
+
         if( $vlan !== null && $evlan )
             $sql .= "\n            AND vlan.id = " . $evlan->getId();
-        
+
         if( $asn !== null )
             $sql .= "\n            AND cs.autsys = " . intval( $asn );
-            
+
         $sql .= "\n        GROUP BY bs.srcipaddressid, bs.dstipaddressid";
-        
+
         $peers = $conn->fetchAll( $sql );
-        
+
         $apeers = [];
-    
+
         foreach( $peers as $p )
         {
             if( !isset( $apeers[ $p['csautsys'] ] ) )
@@ -113,18 +119,18 @@ public function getPeers( $vlan = null, $protocol = 6, $asn = null, $forceDb = f
                 $apeers[ $p['csautsys'] ]['name']      = $p['csname'];
                 $apeers[ $p['csautsys'] ]['peers']     = [];
             }
-    
+
             $apeers[ $p['csautsys'] ]['peers'][ $p['cdautsys'] ] = $p['cdautsys'];
         }
-    
+
         ksort( $apeers, SORT_NUMERIC );
-    
+
         foreach( $apeers as $asn => $p )
             ksort( $apeers[ $asn ][ 'peers' ], SORT_NUMERIC );
 
         \Zend_Registry::get( 'd2cache' )->save( $key, $apeers, 3600 );
-                
+
         return $apeers;
     }
-    
+
 }
diff --git a/database/xml/Entities.BGPSessionData.dcm.xml b/database/xml/Entities.BGPSessionData.dcm.xml
@@ -15,5 +15,8 @@
     </field>
     <field name="timestamp" type="datetime" nullable="true"/>
     <field name="source" type="string" length="40" nullable="true"/>
+    <indexes>
+      <index name="idx_timestamp" columns="timestamp"/>
+    </indexes>
   </entity>
 </doctrine-mapping>
diff --git a/schema-v3-to-v4.sql b/schema-v3-to-v4.sql
@@ -28,3 +28,4 @@ ALTER TABLE vlan CHANGE private private TINYINT(1) DEFAULT '0' NOT NULL;
 ALTER TABLE vlaninterface CHANGE ipv4enabled ipv4enabled TINYINT(1) DEFAULT '0', CHANGE ipv6enabled ipv6enabled TINYINT(1) DEFAULT '0', CHANGE mcastenabled mcastenabled TINYINT(1) DEFAULT '0', CHANGE irrdbfilter irrdbfilter TINYINT(1) DEFAULT '1';
 DROP TABLE change_log;
 DROP TABLE sec_event;
+CREATE INDEX idx_timestamp ON bgpsessiondata (timestamp);