Graphing refactor

.gitignore

@@ -18,6 +18,10 @@
 /include/config/global.inc.scrypt.php
 /include/config/global.inc.sha.php
 
+# Test files
+/scripts/test.php
+/cronjobs/test.php
+
 # IDE Settings
 /.idea/*
 .buildpath

cronjobs/statistics.php

@@ -27,7 +27,7 @@
 
 // Header
 $log->logInfo('Running statistical queries, errors may just mean no shares were available');
-$strLogMask = "| %-26.26s | %8.8s | %-6.6s |";
+$strLogMask = "| %-33.33s | %8.8s | %-6.6s |";
 $log->logInfo(sprintf($strLogMask, 'Method', 'Runtime', 'Status'));
 
 // Per user share statistics based on all shares submitted
@@ -37,9 +37,15 @@
 
 // Get all user hashrate statistics for caching
 $start = microtime(true);
-$statistics->getAllUserMiningStats() ? $status = 'OK' : $status = 'ERROR';
-$log->logInfo(sprintf($strLogMask, 'getAllUserMiningStats', number_format(microtime(true) - $start, 3), $status));
+$statistics->fetchAllUserMiningStats() ? $status = 'OK' : $status = 'ERROR';
+$log->logInfo(sprintf($strLogMask, 'fetchAllUserMiningStats', number_format(microtime(true) - $start, 3), $status));
 
+// Store our statistical data into our `statistics_users` table
+$start = microtime(true);
+$statistics->storeAllUserMiningStatsSnapshot($statistics->getAllUserMiningStats()) ? $status = 'OK' : $status = 'ERROR';
+$log->logInfo(sprintf($strLogMask, 'storeAllUserMiningStatsSnapshot', number_format(microtime(true) - $start, 3), $status));
+
+// Get stats for pool overview
 $start = microtime(true);
 $statistics->getTopContributors('hashes') ? $status = 'OK' : $status = 'ERROR';
 $log->logInfo(sprintf($strLogMask, 'getTopContributors(hashes)', number_format(microtime(true) - $start, 3), $status));

cronjobs/tables_cleanup.php

@@ -65,14 +65,27 @@
 $message = '';
 $affected = $share->purgeArchive();
 if ($affected === false) {
-  $message = 'Failed to delete notifications: ' . $oToken->getCronError();
+  $message = 'Failed to delete shares: ' . $share->getCronError();
   $status = 'ERROR';
   $monitoring->endCronjob($cron_name, 'E0008', 0, false, false);
 } else {
   $affected == 0 ? $message = 'No shares deleted' : $message = 'Deleted old shares';
 }
 $log->logInfo(sprintf($strLogMask, 'purgeArchive', $affected, number_format(microtime(true) - $start, 3), $status, $message));
 
+// Clenaup shares archive
+$start = microtime(true);
+$status = 'OK';
+$message = '';
+$affected = $statistics->purgeUserStats($setting->getValue('statistics_graphing_days', 1));
+if ($affected === false) {
+  $message = 'Failed to delete entries: ' . $statistics->getCronError();
+  $status = 'ERROR';
+  $monitoring->endCronjob($cron_name, 'E0008', 0, false, false);
+} else {
+  $affected == 0 ? $message = 'No entries deleted' : $message = 'Deleted old entries';
+}
+$log->logInfo(sprintf($strLogMask, 'purgeUserStats', $affected, number_format(microtime(true) - $start, 3), $status, $message));
 
 // Cron cleanup and monitoring
 require_once('cron_end.inc.php');

include/classes/statistics.class.php

@@ -9,6 +9,7 @@
  **/
 class Statistics extends Base {
   protected $table = 'statistics_shares';
+  protected $table_user_stats = 'statistics_users';
   private $getcache = true;
 
   // Disable fetching values from cache
@@ -18,6 +19,12 @@ public function setGetCache($set=false) {
   public function getGetCache() {
     return $this->getcache;
   }
+  public function getAllUserMiningStats() {
+    return $this->allUserMiningStats;
+  }
+  public function getUserStatsTableName() {
+    return $this->table_user_stats;
+  }
 
   /**
    * Get our first block found
@@ -185,7 +192,7 @@ public function getBlocksSolvedbyWorker($account_id, $limit=25) {
       return $this->memcache->setCache(__FUNCTION__ . $account_id . $limit, $result->fetch_all(MYSQLI_ASSOC), 5);
     return $this->sqlError();
   }
-  
+
   /**
    * Currently the only function writing to the database
    * Stored per block user statistics of valid and invalid shares
@@ -254,12 +261,12 @@ public function getCurrentShareRate($interval=180) {
       SELECT
       (
         (
-          SELECT ROUND(COUNT(id) / ?, 2) AS sharerate
+          SELECT ROUND(SUM(difficulty) / ?, 2) AS sharerate
           FROM " . $this->share->getTableName() . "
           WHERE time > DATE_SUB(now(), INTERVAL ? SECOND)
           AND our_result = 'Y'
         ) + (
-          SELECT ROUND(COUNT(id) / ?, 2) AS sharerate
+          SELECT ROUND(SUM(difficulty) / ?, 2) AS sharerate
           FROM " . $this->share->getArchiveTableName() . "
           WHERE time > DATE_SUB(now(), INTERVAL ? SECOND)
           AND our_result = 'Y'
@@ -451,16 +458,19 @@ public function getAllUserStats($filter='%',$limit=1,$start=0) {
 
   /**
    * Fetch all user hashrates based on shares and archived shares
+   * Store it in cache, also keep a copy of the data internally to
+   * return it for further processing
    * @return data array Set of all user stats
    **/
-  public function getAllUserMiningStats($interval=180) {
+  public function fetchAllUserMiningStats($interval=180) {
     $this->debug->append("STA " . __METHOD__, 4);
     $stmt = $this->mysqli->prepare("
       SELECT
         a.id AS id,
         a.username AS account,
+        COUNT(DISTINCT t1.username) AS workers,
         IFNULL(SUM(t1.difficulty), 0) AS shares,
-        ROUND(COUNT(t1.id) / ?, 2) AS sharerate,
+        ROUND(SUM(t1.difficulty) / ?, 2) AS sharerate,
         IFNULL(AVG(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS avgsharediff
       FROM (
         SELECT
@@ -489,12 +499,45 @@ public function getAllUserMiningStats($interval=180) {
         $aData['data'][$row['id']] = $row;
         $aData['data'][$row['id']]['hashrate'] = $this->coin->calcHashrate($row['shares'], $interval);
       }
+      $this->allUserMiningStats = $aData;
       return $this->memcache->setStaticCache(STATISTICS_ALL_USER_HASHRATES, $aData, 600);
     } else {
       return $this->sqlError();
     }
   }
 
+  /**
+   * Store our gathered data into our statistic table for users
+   * @param aData array Data created by fetchAllUserMiningStats
+   * @return bool true or false
+   **/
+  public function storeAllUserMiningStatsSnapshot($aData) {
+    $this->debug->append("STA " . __METHOD__, 4);
+    if (!isset($aData['data'])) return false;
+    // initilize
+    $timestamp = time();    // Store all entries with the same timestamp to reduce cardinality
+    $ok = 0;
+    $failed = 0;
+    foreach ($aData['data'] as $key => $aUserData) {
+      $stmt = $this->mysqli->prepare("
+        INSERT INTO " . $this->getUserStatsTableName() . "
+          ( account_id, hashrate, workers, sharerate, timestamp ) VALUES ( ?, ?, ?, ?, ?)");
+      if ($this->checkStmt($stmt) && $stmt->bind_param("ididi", $aUserData['id'], $aUserData['hashrate'], $aUserData['workers'], $aUserData['sharerate'], $timestamp) && $stmt->execute() ) {
+        $ok++;
+      } else {
+        $failed++;
+      }
+    }
+    return array('ok' => $ok, 'failed' => $failed);
+  }
+
+  /**
+   * Fetch unpaid PPS shares for an account
+   * @param username string Username
+   * @param account_id int User ID
+   * @param last_paid_pps_id int Last paid out share by pps_payout cron
+   * @return data int Sum of unpaid diff1 shares
+   **/
   public function getUserUnpaidPPSShares($username, $account_id=NULL, $last_paid_pps_id) {
     $this->debug->append("STA " . __METHOD__, 4);
     if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data;
@@ -515,7 +558,7 @@ public function getUserUnpaidPPSShares($username, $account_id=NULL, $last_paid_p
    * Get Shares per x interval by user
    * @param username string username
    * @param $account_id int account id   
-   * @return data integer Current Sharerate in shares/s
+   * @return data integer Current Sharerate in diff1 shares/s
    **/
   public function getUserMiningStats($username, $account_id=NULL, $interval=180) {
     $this->debug->append("STA " . __METHOD__, 4);
@@ -532,7 +575,7 @@ public function getUserMiningStats($username, $account_id=NULL, $interval=180) {
     if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data;
     $stmt = $this->mysqli->prepare("
       SELECT
-        IFNULL(COUNT(*) / ?, 0) AS sharerate,
+        IFNULL(SUM(difficulty) / ?, 0) AS sharerate,
         IFNULL(SUM(difficulty), 0) AS shares,
         IFNULL(AVG(difficulty), 0) AS avgsharediff
       FROM (
@@ -655,78 +698,24 @@ public function getTopContributors($type='shares', $limit=15) {
    * @param $account_id int account id
    * @return data array NOT FINISHED YET
    **/
-  public function getHourlyHashrateByAccount($username, $account_id=NULL) {
+  public function getHourlyMiningStatsByAccount($account_id, $format='array', $days = 1) {
     $this->debug->append("STA " . __METHOD__, 4);
     if ($data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data;
     $stmt = $this->mysqli->prepare("
       SELECT
-        id,
-        IFNULL(SUM(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS shares,
-        HOUR(time) AS hour
-      FROM " . $this->share->getTableName() . "
-      WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60))
-        AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR
-        AND our_result = 'Y'
-        AND username LIKE ?
-      GROUP BY HOUR(time)
-      UNION
-      SELECT
-        share_id,
-        IFNULL(SUM(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS shares,
-        HOUR(time) AS hour
-      FROM " . $this->share->getArchiveTableName() . "
-      WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60))
-        AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR
-        AND our_result = 'Y'
-        AND username LIKE ?
-      GROUP BY HOUR(time)");
-    $username = $username . ".%";
-    if ($this->checkStmt($stmt) && $stmt->bind_param('ss', $username, $username) && $stmt->execute() && $result = $stmt->get_result()) {
-      $iStartHour = date('G');
-      // Initilize array
-      for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0;
-      // Fill data
-      while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int) $this->coin->calcHashrate($row['shares'], 3600);
-      return $this->memcache->setCache(__FUNCTION__ . $account_id, $aData);
-    }
-    return $this->sqlError();
-  }
-
-  /**
-   * get Hourly hashrate for the pool 
-   * @param none
-   * @return data array NOT FINISHED YET
-   **/
-  public function getHourlyHashrateByPool() {
-    $this->debug->append("STA " . __METHOD__, 4);
-    if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data;
-    $stmt = $this->mysqli->prepare("
-      SELECT
-        id,
-      	IFNULL(SUM(IF(s.difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty)), 0) AS shares,
-        HOUR(s.time) AS hour
-      FROM " . $this->share->getTableName() . " AS s
-      WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60))
-        AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR
-        AND our_result = 'Y'
-      GROUP BY HOUR(time)
-      UNION
-      SELECT
-        share_id,
-      	IFNULL(SUM(IF(s.difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty)), 0) AS shares,
-        HOUR(s.time) AS hour
-      FROM " . $this->share->getArchiveTableName() . " AS s
-      WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60))
-        AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR
-        AND our_result = 'Y'
-      GROUP BY HOUR(time)");
-    if ($this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result()) {
-      $iStartHour = date('G');
-      // Initilize array
-      for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0;
-      // Fill data
-      while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int) $this->coin->calcHashrate($row['shares'], 3600);
-      return $this->memcache->setCache(__FUNCTION__, $aData);
+        timestamp,
+        FROM_UNIXTIME(timestamp, '%Y-%m-%d %H:%i') AS time,
+        AVG(hashrate) AS hashrate,
+        AVG(workers) AS workers,
+        AVG(sharerate) AS sharerate
+      FROM " . $this->getUserStatsTableName() . "
+      WHERE FROM_UNIXTIME(timestamp) >= DATE_SUB(NOW(), INTERVAL $days DAY)
+        AND account_id = ?
+      GROUP BY DAY(FROM_UNIXTIME(timestamp)), HOUR(FROM_UNIXTIME(timestamp))");
+    if ($this->checkStmt($stmt) && $stmt->bind_param('i', $account_id) && $stmt->execute() && $result = $stmt->get_result()) {
+      $aData = $result->fetch_all(MYSQLI_ASSOC);
+      if ($format == 'json') $aData = json_encode($aData);
+      return $this->memcache->setCache(__FUNCTION__ . $account_id . $format, $aData);
     }
     return $this->sqlError();
   }
@@ -914,6 +903,17 @@ public function getCountAllActiveUsers($interval=120) {
       return $this->memcache->setCache(__FUNCTION__, $result->fetch_object()->total);
     return $this->sqlError();
   }
+
+  /**
+   * Purge older entries from our statistics_users table
+   **/
+  public function purgeUserStats($days = 1) {
+    // Fallbacks if unset
+    $stmt = $this->mysqli->prepare("DELETE FROM " . $this->getUserStatsTableName() . " WHERE FROM_UNIXTIME(timestamp) <= DATE_SUB(NOW(), INTERVAL ? DAY)");
+    if ($this->checkStmt($stmt) && $stmt->bind_param('i', $days) && $stmt->execute())
+      return $stmt->affected_rows;
+    return $this->sqlError();
+  }
 }
 
 $statistics = new Statistics();

include/config/admin_settings.inc.php

@@ -160,6 +160,13 @@
   'name' => 'statistics_ajax_data_interval', 'value' => $setting->getValue('statistics_ajax_data_interval'),
   'tooltip' => 'Time in minutes, interval for hashrate and sharerate calculations. Higher intervals allow for better accuracy at a higer server load.'
 );
+$aSettings['statistics'][] = array(
+  'display' => 'Graphing Days', 'type' => 'text',
+  'size' => 25,
+  'default' => 1,
+  'name' => 'statistics_graphing_days', 'value' => $setting->getValue('statistics_graphing_days'),
+  'tooltip' => 'How many days to graph out on the statistics -> graphs page.'
+);
 $aSettings['statistics'][] = array(
   'display' => 'Block Statistics Count', 'type' => 'text',
   'size' => 25,

include/pages/api.inc.php

@@ -5,7 +5,7 @@
 $api->isActive();
 
 // Check for valid API key
-$id = $user->checkApiKey($_REQUEST['api_key']);
+$id = $user->checkApiKey(@$_REQUEST['api_key']);
 
 header('HTTP/1.1 400 Bad Request');
 die('400 Bad Request');

include/pages/statistics/graphs.inc.php

@@ -4,11 +4,9 @@
 if (!$smarty->isCached('master.tpl', $smarty_cache_key)) {
   $debug->append('No cached version available, fetching from backend', 3);
   if ($user->isAuthenticated()) {
-    $aHourlyHashRates = $statistics->getHourlyHashrateByAccount($_SESSION['USERDATA']['username'], $_SESSION['USERDATA']['id']);
-    $aPoolHourlyHashRates = $statistics->getHourlyHashrateByPool();
+    $aHourlyMiningStats = $statistics->getHourlyMiningStatsByAccount($_SESSION['USERDATA']['id'], 'json', $setting->getValue('statistics_graphing_days', 1));
   }
-  $smarty->assign("YOURHASHRATES", @$aHourlyHashRates);
-  $smarty->assign("POOLHASHRATES", @$aPoolHourlyHashRates);
+  $smarty->assign('YOURMININGSTATS', @$aHourlyMiningStats);
 } else {
   $debug->append('Using cached page', 3);
 }

include/version.inc.php

@@ -2,7 +2,7 @@
 $defflip = (!cfip()) ? exit(header('HTTP/1.1 401 Unauthorized')) : 1;
 
 define('MPOS_VERSION', '0.0.4');
-define('DB_VERSION', '0.0.12');
+define('DB_VERSION', '0.0.13');
 define('CONFIG_VERSION', '0.0.8');
 define('HASH_VERSION', 1);
 

sql/000_base_structure.sql

@@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS `settings` (
   UNIQUE KEY `setting` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-INSERT INTO `settings` (`name`, `value`) VALUES ('DB_VERSION', '0.0.12');
+INSERT INTO `settings` (`name`, `value`) VALUES ('DB_VERSION', '0.0.13');
 
 CREATE TABLE IF NOT EXISTS `shares` (
   `id` bigint(30) NOT NULL AUTO_INCREMENT,
@@ -239,12 +239,15 @@ CREATE TABLE IF NOT EXISTS `transactions` (
   KEY `account_id_archived` (`account_id`,`archived`)
 ) ENGINE=InnoDB  DEFAULT CHARSET=utf8;
 
-CREATE TABLE IF NOT EXISTS `templates` (
-  `template` varchar(255) NOT NULL,
-  `active` tinyint(1) NOT NULL DEFAULT 0,
-  `content` mediumtext,
-  `modified_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  PRIMARY KEY (`template`)
+CREATE TABLE `statistics_users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `account_id` int(11) NOT NULL,
+  `hashrate` int(11) NOT NULL,
+  `workers` int(11) NOT NULL,
+  `sharerate` float NOT NULL,
+  `timestamp` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `account_id_timestamp` (`account_id`,`timestamp`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;