Fix slow aggregation and slow cleaning

console

@@ -11,6 +11,7 @@ $console->setSilex($app);
 
 $console->add(new Pinboard\Command\InitCommand());
 $console->add(new Pinboard\Command\AggregateCommand());
+$console->add(new Pinboard\Command\CleanCommand());
 $console->add(new Pinboard\Command\AddUserCommand());
 
 $app->register(

src/Pinboard/Command/AggregateCommand.php

@@ -28,6 +28,12 @@ protected function configure()
         ;
     }
 
+    protected function currentTime()
+    {
+        $now = new \DateTime();
+        return $now->format('Y-m-d H:i:s');
+    }
+
     protected function initMailer()
     {
         if (isset($this->params['smtp'])) {
@@ -137,6 +143,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         $db = $this->app['db'];
 
+        $output->writeln('<comment>[' . $this->currentTime() . ']</comment> <info>Starting aggregation</info>');
+
         try {
             $this->initMailer();
         }
@@ -182,46 +190,18 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $now = new \DateTime();
         $now = $now->format('Y-m-d H:i:s');
 
-        $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M');
-        $date = new \DateTime();
-        $date->sub($delta);
-
-        $params = array(
-            'created_at' => $date->format('Y-m-d H:i:s'),
-        );
-
-        $tablesForClear = array(
-            "ipm_report_2_by_hostname_and_server",
-            "ipm_report_by_hostname",
-            "ipm_report_by_hostname_and_server",
-            "ipm_report_by_server_name",
-            "ipm_req_time_details",
-            "ipm_mem_peak_usage_details",
-            "ipm_status_details",
-            "ipm_cpu_usage_details",
-            "ipm_timer",
-        );
-
         $sql = '';
+        $sql .= 'TRUNCATE TABLE `ipm_request`;';
+        $sql .= 'INSERT INTO `ipm_request` SELECT * FROM `request`;';
 
-        foreach ($tablesForClear as $value) {
-            $sql .= '
-            DELETE
-            FROM
-                ' . $value . '
-            WHERE
-                created_at < :created_at
-            ;';
-        }
-        if ($sql != '')
-            $db->executeQuery($sql, $params);
+        $db->executeQuery($sql);
 
         if (isset($this->params['notification']['enable']) && $this->params['notification']['enable']) {
             $sql = '
                 SELECT
                     server_name, script_name, status, max(hostname) AS hostname, count(*) AS count
                 FROM
-                    request
+                    ipm_request
                 WHERE
                     status >= 500
                 GROUP BY
@@ -247,7 +227,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             SELECT
                 server_name, hostname, COUNT(*) AS cnt
             FROM
-                request
+                ipm_request
             GROUP BY
                 server_name, hostname
         ';
@@ -259,7 +239,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 SELECT
                     r.%s
                 FROM
-                    request r
+                    ipm_request r
                 WHERE
                     r.server_name = r2.server_name AND r.hostname = r2.hostname
                 ORDER BY
@@ -297,7 +277,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                     ' . sprintf($subselectTemplate, 'doc_size', 'doc_size', $server['cnt'] * (1 - 1.00), 'doc_size_100') . ',
                     \'' . $now . '\'
                 FROM
-                    request r2
+                    ipm_request r2
                 WHERE
                     r2.server_name = "' . $server['server_name'] . '" and r2.hostname = "' . $server['hostname'] . '"
                 LIMIT 1
@@ -451,7 +431,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             SELECT
                 server_name, hostname, script_name, status, tags, tags_cnt, FROM_UNIXTIME(max(timestamp))
             FROM
-                request
+                ipm_request
             WHERE
                 status >= 500
             GROUP BY
@@ -478,7 +458,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 SELECT
                     id, server_name, hostname, script_name, max(req_time), max(mem_peak_usage), max(tags), max(tags_cnt), max(timers_cnt), FROM_UNIXTIME(timestamp)
                 FROM
-                    request
+                    ipm_request
                 WHERE
                     server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND req_time > ' . (float)$maxReqTime . '
                 GROUP BY
@@ -518,7 +498,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                     FROM
                         timer t
                     JOIN
-                        request r ON t.request_id = r.id
+                        ipm_request r ON t.request_id = r.id
                     JOIN
                         timertag tt ON tt.timer_id = t.id
                     JOIN
@@ -547,7 +527,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                 SELECT
                     server_name, hostname, script_name, max(mem_peak_usage), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp))
                 FROM
-                    request
+                    ipm_request
                 WHERE
                     server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND mem_peak_usage > ' . (int)$maxMemoryUsage . '
                 GROUP BY
@@ -577,7 +557,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
                   SELECT
                       server_name, hostname, script_name, max(ru_utime), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp))
                   FROM
-                      request
+                      ipm_request
                   WHERE
                       server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND ru_utime > ' . (int)$maxCPUUsage . '
                   GROUP BY
@@ -595,7 +575,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $values = $this->getBorderOutValues($db, $servers);
         $this->sendBorderOutEmails($values);
 
-        $output->writeln('<info>Data are aggregated successfully</info>');
+        $output->writeln('<comment>[' . $this->currentTime() . ']</comment> <info>Data are aggregated successfully</info>');
 
         if (!unlink( __FILE__ . '.lock')) {
             $output->writeln('<error>Error: cannot remove ' . __FILE__ . '.lock file, you must remove it manually and check server settings.</error>');

src/Pinboard/Command/CleanCommand.php

@@ -0,0 +1,89 @@
+<?php
+namespace Pinboard\Command;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Yaml\Yaml;
+
+class CleanCommand extends Command
+{
+    protected $params;
+    protected $output;
+
+    protected function configure()
+    {
+        $this
+            ->setName('clean')
+            ->setDescription('Clean old data from tables')
+        ;
+    }
+
+    protected function currentTime()
+    {
+        $now = new \DateTime();
+        return $now->format('Y-m-d H:i:s');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $this->app = $this->getApplication()->getSilex();
+        $this->app->boot();
+        $this->params = $this->app['params'];
+        $this->output = $output;
+
+        $db = $this->app['db'];
+
+        $output->writeln('<comment>[' . $this->currentTime() . ']</comment> <info>Starting clean</info>');
+
+        try {
+            $db->connect();
+        }
+        catch(\PDOException $e) {
+            $output->writeln('<error>Can\'t connect to MySQL server</error>');
+
+            return;
+        }
+
+        $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M');
+        $date = new \DateTime();
+        $date->sub($delta);
+
+        $params = array(
+            'created_at' => $date->format('Y-m-d H:i:s'),
+        );
+
+        $tablesForClear = array(
+            "ipm_report_2_by_hostname_and_server",
+            "ipm_report_by_hostname",
+            "ipm_report_by_hostname_and_server",
+            "ipm_report_by_server_name",
+            "ipm_req_time_details",
+            "ipm_mem_peak_usage_details",
+            "ipm_status_details",
+            "ipm_cpu_usage_details",
+            "ipm_timer",
+            "ipm_tag_info",
+        );
+
+
+        foreach ($tablesForClear as $value) {
+            $output->write('<comment>[' . $this->currentTime() . ']</comment> <info>Clean </info>"' . $value . '"<comment>....</comment>');
+            $sql = '
+            DELETE
+            FROM
+                ' . $value . '
+            WHERE
+                created_at < :created_at
+            ;';
+
+            if ($db->executeQuery($sql, $params)->closeCursor()) {
+                $output->writeln('<info>Done</info>');
+            }
+        }
+
+        $output->writeln('<comment>[' . $this->currentTime() . ']</comment> <info>Finished</info>');
+    }
+}

src/Pinboard/Command/InitCommand.php

@@ -20,6 +20,7 @@ protected function configure()
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $changeFlag = False;
         $output->writeln('<info>Defining crontab task...</info>');
         $output->writeln('<info>Please enter the frequency of data aggregating</info> <comment>(frequency must be equal "pinba_stats_history" of the pinba engine config)</comment>.');
 
@@ -46,24 +47,42 @@ function ($answer) {
         $crontabString = $process->isSuccessful() ? $process->getOutput() : '';
 
         $path = realpath(__DIR__ . '/../../../console');
-        $command = '*/' . $frequency . ' * * * * ' . $path . ' aggregate';
 
+        $crontabString .= "\n# Pinboard aggregating & clean\n";
+
+        // Add the aggregate command in to crontab if not exist
+        $execTime = '*/' . $frequency . ' * * * * ';
+        $command = $path . ' aggregate';
         if (strpos($crontabString, $command) === false) {
-            $crontabString .= "\n" . $command . "\n";
+            $crontabString .= $execTime . $command . "\n";
+            $changeFlag = True;
         }
 
-        $file = tempnam(sys_get_temp_dir(), 'ipm');
-        file_put_contents($file, $crontabString);
+        // Add the clean command in to crontab if not exist
+        $execTime = '3 0 * * * ';
+        $command = $path . ' clean';
+        if (strpos($crontabString, $command) === false) {
+            $crontabString .= $execTime . $command . "\n";
+            $changeFlag = True;
+        }
 
-        $process = new Process('crontab ' . $file);
-        $process->setTimeout(20);
-        $process->run();
+        if ($changeFlag) {
+            $file = tempnam(sys_get_temp_dir(), 'ipm');
+            file_put_contents($file, $crontabString);
 
-        if (!$process->isSuccessful()) {
-            throw new \RuntimeException($process->getErrorOutput());
-        }
+            $process = new Process('crontab ' . $file);
+            $process->setTimeout(20);
+            $process->run();
+
+            if (!$process->isSuccessful()) {
+                throw new \RuntimeException($process->getErrorOutput());
+            }
 
-        $output->writeln('<info>Crontab task are defined successfully</info>');
-        $output->writeln('<info>Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml</info>');
+            $output->writeln('<info>Crontab task are defined successfully</info>');
+            $output->writeln('<info>Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml</info>');
+        }
+        else {
+            $output->writeln('<comment>The crons already exist</comment>');
+        }
     }
-}
+}

src/Pinboard/DoctrineMigrations/Version20191018165422.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace Pinboard\DoctrineMigrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+class Version20191018165422 extends AbstractMigration
+{
+    public function up(Schema $schema)
+    {
+        $this->addSql("
+            CREATE TABLE `ipm_request` (
+              `id` int(11) NOT NULL DEFAULT '0',
+              `hostname` varchar(32) DEFAULT NULL,
+              `req_count` int(11) DEFAULT NULL,
+              `server_name` varchar(64) DEFAULT NULL,
+              `script_name` varchar(128) DEFAULT NULL,
+              `doc_size` float DEFAULT NULL,
+              `mem_peak_usage` float DEFAULT NULL,
+              `req_time` float DEFAULT NULL,
+              `ru_utime` float DEFAULT NULL,
+              `ru_stime` float DEFAULT NULL,
+              `timers_cnt` int(11) DEFAULT NULL,
+              `status` int(11) DEFAULT NULL,
+              `memory_footprint` float DEFAULT NULL,
+              `schema` varchar(16) DEFAULT NULL,
+              `tags_cnt` int(11) DEFAULT NULL,
+              `tags` varchar(1024) DEFAULT NULL,
+              `timestamp` int(11) DEFAULT NULL,
+              KEY `ir_sss` (`server_name`,`script_name`,`status`),
+              KEY `ir_shst` (`server_name`,`hostname`,`script_name`,`timestamp`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+        ");
+    }
+
+    public function down(Schema $schema)
+    {
+        $this->addSql("DROP TABLE `ipm_request`;");
+    }
+}

src/Pinboard/DoctrineMigrations/Version20191021124252.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Pinboard\DoctrineMigrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+
+class Version20191021124252 extends AbstractMigration
+{
+    public function up(Schema $schema)
+    {
+        $this->addSql("ALTER TABLE `ipm_tag_info` ADD INDEX `iti_c` (`created_at`);");
+        $this->addSql("ALTER TABLE `ipm_timer` ADD INDEX `it_c` (`created_at`);");
+    }
+
+    public function down(Schema $schema)
+    {
+        $this->addSql("ALTER TABLE `ipm_tag_info` DROP INDEX `iti_c`;");
+        $this->addSql("ALTER TABLE `ipm_timer` DROP INDEX `it_c`;");
+    }
+
+}

views/server.html.twig

@@ -8,7 +8,7 @@
       <div class="span{{ period == '1 month' or period == '1 week' ? '12' : '6' }}">
         <h2>Request time <small><a href="{{ path('server_req_time', { serverName: server_name, hostName: hostname }) }}">view details</a></small></h2>
         {% if req|length > 0 %}
-        <div id="req-time-chart" style="height: 450px;"></div>
+        <div id="req-time-chart" style="height: 455px;"></div>
         <script type="text/javascript">
             var rtChart;
 
@@ -122,7 +122,7 @@
       <div class="span{{ period == '1 month' or period == '1 week' ? '12' : '6' }}">
         <h2>Requests per sec {# <small><a href="{{ path('server_request_time', { serverName: server_name }) }}">view details</a></small> #}</h2>
         {% if req_per_sec.data|length > 0 %}
-        <div id="req-per-sec-chart" style="height: {{ 346 + req_per_sec.hosts|length * 19 + (req_per_sec.hosts|length > 1 ? 19 : 0) }}px;"></div>
+        <div id="req-per-sec-chart" style="height: {{ 315 + req_per_sec.hosts|length * 28 + (req_per_sec.hosts|length > 0 ? 0 : 28) }}px;"></div>
         <script type="text/javascript">
             var rpsChart;