From b758ee9395b807c3ca82847cf3c8c1782de4a35a Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Thu, 10 Jul 2014 11:49:33 +0200 Subject: [PATCH 01/10] [PoC] Cronjob based Hashrate graphs --- cronjobs/statistics.php | 12 +- include/classes/statistics.class.php | 113 +++++++++++-------- include/pages/statistics/graphs.inc.php | 2 +- include/version.inc.php | 2 +- sql/000_base_structure.sql | 17 +-- upgrade/definitions/0.0.12_to_0.0.13.inc.php | 39 +++++++ 6 files changed, 128 insertions(+), 57 deletions(-) create mode 100644 upgrade/definitions/0.0.12_to_0.0.13.inc.php diff --git a/cronjobs/statistics.php b/cronjobs/statistics.php index 6530fab71..eb564c1a8 100755 --- a/cronjobs/statistics.php +++ b/cronjobs/statistics.php @@ -27,7 +27,7 @@ // Header $log->logInfo('Running statistical queries, errors may just mean no shares were available'); -$strLogMask = "| %-26.26s | %8.8s | %-6.6s |"; +$strLogMask = "| %-33.33s | %8.8s | %-6.6s |"; $log->logInfo(sprintf($strLogMask, 'Method', 'Runtime', 'Status')); // Per user share statistics based on all shares submitted @@ -37,9 +37,15 @@ // Get all user hashrate statistics for caching $start = microtime(true); -$statistics->getAllUserMiningStats() ? $status = 'OK' : $status = 'ERROR'; -$log->logInfo(sprintf($strLogMask, 'getAllUserMiningStats', number_format(microtime(true) - $start, 3), $status)); +$statistics->fetchAllUserMiningStats() ? $status = 'OK' : $status = 'ERROR'; +$log->logInfo(sprintf($strLogMask, 'fetchAllUserMiningStats', number_format(microtime(true) - $start, 3), $status)); +// Store our statistical data into our `statistics_users` table +$start = microtime(true); +$statistics->storeAllUserMiningStatsSnapshot($statistics->getAllUserMiningStats()) ? $status = 'OK' : $status = 'ERROR'; +$log->logInfo(sprintf($strLogMask, 'storeAllUserMiningStatsSnapshot', number_format(microtime(true) - $start, 3), $status)); + +// Get stats for pool overview $start = microtime(true); $statistics->getTopContributors('hashes') ? $status = 'OK' : $status = 'ERROR'; $log->logInfo(sprintf($strLogMask, 'getTopContributors(hashes)', number_format(microtime(true) - $start, 3), $status)); diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index ae6736b45..51a63410a 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -9,6 +9,7 @@ **/ class Statistics extends Base { protected $table = 'statistics_shares'; + protected $table_user_stats = 'statistics_users'; private $getcache = true; // Disable fetching values from cache @@ -18,6 +19,12 @@ public function setGetCache($set=false) { public function getGetCache() { return $this->getcache; } + public function getAllUserMiningStats() { + return $this->allUserMiningStats; + } + public function getUserStatsTableName() { + return $this->table_user_stats; + } /** * Get our first block found @@ -451,14 +458,17 @@ public function getAllUserStats($filter='%',$limit=1,$start=0) { /** * Fetch all user hashrates based on shares and archived shares + * Store it in cache, also keep a copy of the data internally to + * return it for further processing * @return data array Set of all user stats **/ - public function getAllUserMiningStats($interval=180) { + public function fetchAllUserMiningStats($interval=180) { $this->debug->append("STA " . __METHOD__, 4); $stmt = $this->mysqli->prepare(" SELECT a.id AS id, a.username AS account, + COUNT(DISTINCT t1.username) AS workers, IFNULL(SUM(t1.difficulty), 0) AS shares, ROUND(COUNT(t1.id) / ?, 2) AS sharerate, IFNULL(AVG(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS avgsharediff @@ -489,12 +499,43 @@ public function getAllUserMiningStats($interval=180) { $aData['data'][$row['id']] = $row; $aData['data'][$row['id']]['hashrate'] = $this->coin->calcHashrate($row['shares'], $interval); } + $this->allUserMiningStats = $aData; return $this->memcache->setStaticCache(STATISTICS_ALL_USER_HASHRATES, $aData, 600); } else { return $this->sqlError(); } } + /** + * Store our gathered data into our statistic table for users + * @param aData array Data created by fetchAllUserMiningStats + * @return bool true or false + **/ + public function storeAllUserMiningStatsSnapshot($aData) { + // initilize + $timestamp = time(); // Store all entries with the same timestamp to reduce cardinality + $ok = 0; + $failed = 0; + foreach ($aData['data'] as $key => $aUserData) { + $stmt = $this->mysqli->prepare(" + INSERT INTO " . $this->getUserStatsTableName() . " + ( account_id, hashrate, workers, sharerate, timestamp ) VALUES ( ?, ?, ?, ?, ?)"); + if ($this->checkStmt($stmt) && $stmt->bind_param("ididi", $aUserData['id'], $aUserData['hashrate'], $aUserData['workers'], $aUserData['sharerate'], $timestamp) && $stmt->execute() ) { + $ok++; + } else { + $failed++; + } + } + return array('ok' => $ok, 'failed' => $failed); + } + + /** + * Fetch unpaid PPS shares for an account + * @param username string Username + * @param account_id int User ID + * @param last_paid_pps_id int Last paid out share by pps_payout cron + * @return data int Sum of unpaid diff1 shares + **/ public function getUserUnpaidPPSShares($username, $account_id=NULL, $last_paid_pps_id) { $this->debug->append("STA " . __METHOD__, 4); if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; @@ -655,38 +696,26 @@ public function getTopContributors($type='shares', $limit=15) { * @param $account_id int account id * @return data array NOT FINISHED YET **/ - public function getHourlyHashrateByAccount($username, $account_id=NULL) { + public function getHourlyHashrateByAccount($account_id) { $this->debug->append("STA " . __METHOD__, 4); if ($data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT - id, - IFNULL(SUM(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS shares, - HOUR(time) AS hour - FROM " . $this->share->getTableName() . " - WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR - AND our_result = 'Y' - AND username LIKE ? - GROUP BY HOUR(time) - UNION - SELECT - share_id, - IFNULL(SUM(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS shares, - HOUR(time) AS hour - FROM " . $this->share->getArchiveTableName() . " - WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR - AND our_result = 'Y' - AND username LIKE ? - GROUP BY HOUR(time)"); - $username = $username . ".%"; - if ($this->checkStmt($stmt) && $stmt->bind_param('ss', $username, $username) && $stmt->execute() && $result = $stmt->get_result()) { + account_id AS id, + AVG(hashrate) AS hashrate, + HOUR(FROM_UNIXTIME(timestamp)) AS hour + FROM " . $this->getUserStatsTableName() . " + WHERE + account_id = ? + AND timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) + AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) + GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); + if ($this->checkStmt($stmt) && $stmt->bind_param('i', $account_id) && $stmt->execute() && $result = $stmt->get_result()) { $iStartHour = date('G'); // Initilize array for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0; - // Fill data - while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int) $this->coin->calcHashrate($row['shares'], 3600); + // Fill data in proper hour order, result in SQL was ordered 0 - 23 + while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int)$row['hashrate']; return $this->memcache->setCache(__FUNCTION__ . $account_id, $aData); } return $this->sqlError(); @@ -702,30 +731,24 @@ public function getHourlyHashrateByPool() { if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data; $stmt = $this->mysqli->prepare(" SELECT - id, - IFNULL(SUM(IF(s.difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty)), 0) AS shares, - HOUR(s.time) AS hour - FROM " . $this->share->getTableName() . " AS s - WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR - AND our_result = 'Y' - GROUP BY HOUR(time) - UNION - SELECT - share_id, - IFNULL(SUM(IF(s.difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty)), 0) AS shares, - HOUR(s.time) AS hour - FROM " . $this->share->getArchiveTableName() . " AS s - WHERE time <= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - AND time >= FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(NOW())/(60*60))*(60*60)) - INTERVAL 24 HOUR - AND our_result = 'Y' - GROUP BY HOUR(time)"); + SUM(hashrate) / ( + SELECT + COUNT(DISTINCT timestamp) + FROM " . $this->getUserStatsTableName() . " + WHERE timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) + AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) + ) AS hashrate, + HOUR(FROM_UNIXTIME(timestamp)) AS hour + FROM " . $this->getUserStatsTableName() . " + WHERE timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) + AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) + GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); if ($this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result()) { $iStartHour = date('G'); // Initilize array for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0; // Fill data - while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int) $this->coin->calcHashrate($row['shares'], 3600); + while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int)$row['hashrate']; return $this->memcache->setCache(__FUNCTION__, $aData); } return $this->sqlError(); diff --git a/include/pages/statistics/graphs.inc.php b/include/pages/statistics/graphs.inc.php index 0a19492d9..291229cfd 100644 --- a/include/pages/statistics/graphs.inc.php +++ b/include/pages/statistics/graphs.inc.php @@ -4,7 +4,7 @@ if (!$smarty->isCached('master.tpl', $smarty_cache_key)) { $debug->append('No cached version available, fetching from backend', 3); if ($user->isAuthenticated()) { - $aHourlyHashRates = $statistics->getHourlyHashrateByAccount($_SESSION['USERDATA']['username'], $_SESSION['USERDATA']['id']); + $aHourlyHashRates = $statistics->getHourlyHashrateByAccount($_SESSION['USERDATA']['id']); $aPoolHourlyHashRates = $statistics->getHourlyHashrateByPool(); } $smarty->assign("YOURHASHRATES", @$aHourlyHashRates); diff --git a/include/version.inc.php b/include/version.inc.php index a4e0d533f..2b7ea18af 100644 --- a/include/version.inc.php +++ b/include/version.inc.php @@ -2,7 +2,7 @@ $defflip = (!cfip()) ? exit(header('HTTP/1.1 401 Unauthorized')) : 1; define('MPOS_VERSION', '0.0.4'); -define('DB_VERSION', '0.0.12'); +define('DB_VERSION', '0.0.13'); define('CONFIG_VERSION', '0.0.8'); define('HASH_VERSION', 1); diff --git a/sql/000_base_structure.sql b/sql/000_base_structure.sql index a52829b5a..b9887a519 100644 --- a/sql/000_base_structure.sql +++ b/sql/000_base_structure.sql @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS `settings` ( UNIQUE KEY `setting` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -INSERT INTO `settings` (`name`, `value`) VALUES ('DB_VERSION', '0.0.12'); +INSERT INTO `settings` (`name`, `value`) VALUES ('DB_VERSION', '0.0.13'); CREATE TABLE IF NOT EXISTS `shares` ( `id` bigint(30) NOT NULL AUTO_INCREMENT, @@ -239,12 +239,15 @@ CREATE TABLE IF NOT EXISTS `transactions` ( KEY `account_id_archived` (`account_id`,`archived`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE IF NOT EXISTS `templates` ( - `template` varchar(255) NOT NULL, - `active` tinyint(1) NOT NULL DEFAULT 0, - `content` mediumtext, - `modified_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`template`) +CREATE TABLE `statistics_users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `account_id` int(11) NOT NULL, + `hashrate` int(11) NOT NULL, + `workers` int(11) NOT NULL, + `sharerate` float NOT NULL, + `timestamp` int(11) NOT NULL, + PRIMARY KEY (`id`), + KEY `account_id_timestamp` (`account_id`,`timestamp`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; diff --git a/upgrade/definitions/0.0.12_to_0.0.13.inc.php b/upgrade/definitions/0.0.12_to_0.0.13.inc.php new file mode 100644 index 000000000..e661c4d15 --- /dev/null +++ b/upgrade/definitions/0.0.12_to_0.0.13.inc.php @@ -0,0 +1,39 @@ +getValue('DB_VERSION'); // Our actual version installed + + // Upgrade specific variables + $aSql[] = "CREATE TABLE `statistics_users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `account_id` int(11) NOT NULL, + `hashrate` int(11) NOT NULL, + `workers` int(11) NOT NULL, + `sharerate` float NOT NULL, + `timestamp` int(11) NOT NULL, + PRIMARY KEY (`id`), + KEY `account_id_timestamp` (`account_id`,`timestamp`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8"; + $aSql[] = "UPDATE " . $setting->getTableName() . " SET value = '0.0.13' WHERE name = 'DB_VERSION'"; + + if ($db_version_now == $db_version_old && version_compare($db_version_now, DB_VERSION, '<')) { + // Run the upgrade + echo '- Starting database migration to version ' . $db_version_new . PHP_EOL; + foreach ($aSql as $sql) { + echo '- Preparing: ' . $sql . PHP_EOL; + $stmt = $mysqli->prepare($sql); + if ($stmt && $stmt->execute()) { + echo '- success' . PHP_EOL; + } else { + echo '- failed: ' . $mysqli->error . PHP_EOL; + exit(1); + } + } + } +} +?> From bf2429ab2f7a6f3ad306d3d85d8a3ab246950751 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Thu, 10 Jul 2014 13:26:03 +0200 Subject: [PATCH 02/10] [FIX] Skip stat storage on missing data --- include/classes/statistics.class.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index 51a63410a..b7a65782a 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -512,6 +512,8 @@ public function fetchAllUserMiningStats($interval=180) { * @return bool true or false **/ public function storeAllUserMiningStatsSnapshot($aData) { + $this->debug->append("STA " . __METHOD__, 4); + if (!isset($aData['data'])) return false; // initilize $timestamp = time(); // Store all entries with the same timestamp to reduce cardinality $ok = 0; From 77c6abed05891a0553d0a00c253f7a3616006648 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 13 Jul 2014 13:55:37 +0200 Subject: [PATCH 03/10] [REFACTOR] Use raw statistical data --- .gitignore | 4 + include/classes/statistics.class.php | 53 ++++------- include/pages/api.inc.php | 2 +- include/pages/statistics/graphs.inc.php | 4 +- .../bootstrap/statistics/graphs/default.tpl | 88 ++++--------------- .../bootstrap/statistics/graphs/mine.tpl | 2 - .../bootstrap/statistics/graphs/pool.tpl | 2 - 7 files changed, 44 insertions(+), 111 deletions(-) diff --git a/.gitignore b/.gitignore index b4976dfc0..6c1d5b6fc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ /include/config/global.inc.scrypt.php /include/config/global.inc.sha.php +# Test files +/scripts/test.php +/cronjobs/test.php + # IDE Settings /.idea/* .buildpath diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index b7a65782a..c7b2c5939 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -192,7 +192,7 @@ public function getBlocksSolvedbyWorker($account_id, $limit=25) { return $this->memcache->setCache(__FUNCTION__ . $account_id . $limit, $result->fetch_all(MYSQLI_ASSOC), 5); return $this->sqlError(); } - + /** * Currently the only function writing to the database * Stored per block user statistics of valid and invalid shares @@ -698,61 +698,44 @@ public function getTopContributors($type='shares', $limit=15) { * @param $account_id int account id * @return data array NOT FINISHED YET **/ - public function getHourlyHashrateByAccount($account_id) { + public function getHashrateByAccount($account_id, $format='array') { $this->debug->append("STA " . __METHOD__, 4); if ($data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT - account_id AS id, - AVG(hashrate) AS hashrate, - HOUR(FROM_UNIXTIME(timestamp)) AS hour + timestamp, + hashrate FROM " . $this->getUserStatsTableName() . " WHERE - account_id = ? - AND timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) - AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) - GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); + account_id = ?"); if ($this->checkStmt($stmt) && $stmt->bind_param('i', $account_id) && $stmt->execute() && $result = $stmt->get_result()) { - $iStartHour = date('G'); - // Initilize array - for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0; - // Fill data in proper hour order, result in SQL was ordered 0 - 23 - while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int)$row['hashrate']; - return $this->memcache->setCache(__FUNCTION__ . $account_id, $aData); + $aData = $result->fetch_all(MYSQLI_ASSOC); + if ($format == 'json') $aData = json_encode($aData); + return $this->memcache->setCache(__FUNCTION__ . $account_id . $format, $aData); } return $this->sqlError(); } /** - * get Hourly hashrate for the pool + * get Hourly hashrate for the pool * @param none * @return data array NOT FINISHED YET **/ - public function getHourlyHashrateByPool() { + public function getHashrateForPool($format='array') { $this->debug->append("STA " . __METHOD__, 4); if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data; $stmt = $this->mysqli->prepare(" SELECT - SUM(hashrate) / ( - SELECT - COUNT(DISTINCT timestamp) - FROM " . $this->getUserStatsTableName() . " - WHERE timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) - AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) - ) AS hashrate, - HOUR(FROM_UNIXTIME(timestamp)) AS hour - FROM " . $this->getUserStatsTableName() . " - WHERE timestamp <= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 HOUR)) - AND timestamp >= UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 24 HOUR)) - GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); + timestamp, + SUM(hashrate) AS hashrate + FROM " . $this->getUserStatsTableName() . " + GROUP BY timestamp"); if ($this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result()) { - $iStartHour = date('G'); - // Initilize array - for ($i = 0; $i < 24; $i++) $aData[($iStartHour + $i) % 24] = 0; - // Fill data - while ($row = $result->fetch_assoc()) $aData[$row['hour']] += (int)$row['hashrate']; - return $this->memcache->setCache(__FUNCTION__, $aData); + $aData = $result->fetch_all(MYSQLI_ASSOC); + if ($format == 'json') $aData = json_encode($aData); + return $this->memcache->setCache(__FUNCTION__ . $format, $aData); } + var_dump($this->mysqli->error); return $this->sqlError(); } diff --git a/include/pages/api.inc.php b/include/pages/api.inc.php index 01cd20f52..32fd0a34b 100644 --- a/include/pages/api.inc.php +++ b/include/pages/api.inc.php @@ -5,7 +5,7 @@ $api->isActive(); // Check for valid API key -$id = $user->checkApiKey($_REQUEST['api_key']); +$id = $user->checkApiKey(@$_REQUEST['api_key']); header('HTTP/1.1 400 Bad Request'); die('400 Bad Request'); diff --git a/include/pages/statistics/graphs.inc.php b/include/pages/statistics/graphs.inc.php index 291229cfd..c970b8d84 100644 --- a/include/pages/statistics/graphs.inc.php +++ b/include/pages/statistics/graphs.inc.php @@ -4,8 +4,8 @@ if (!$smarty->isCached('master.tpl', $smarty_cache_key)) { $debug->append('No cached version available, fetching from backend', 3); if ($user->isAuthenticated()) { - $aHourlyHashRates = $statistics->getHourlyHashrateByAccount($_SESSION['USERDATA']['id']); - $aPoolHourlyHashRates = $statistics->getHourlyHashrateByPool(); + $aHourlyHashRates = $statistics->getHashrateByAccount($_SESSION['USERDATA']['id'], 'json'); + $aPoolHourlyHashRates = $statistics->getHashrateForPool('json'); } $smarty->assign("YOURHASHRATES", @$aHourlyHashRates); $smarty->assign("POOLHASHRATES", @$aPoolHourlyHashRates); diff --git a/templates/bootstrap/statistics/graphs/default.tpl b/templates/bootstrap/statistics/graphs/default.tpl index 5de1b86b4..65d03503e 100644 --- a/templates/bootstrap/statistics/graphs/default.tpl +++ b/templates/bootstrap/statistics/graphs/default.tpl @@ -8,88 +8,38 @@ $(function () { // You can't draw here chart directly, because it's on hidden tab, instead let's do the workaround $('a[data-toggle="tab"]').on('shown.bs.tab', function (e) { - // this ain't pretty, but you should get the idea - if ($(e.target).attr('href') == '#pool' && $('#pool-area-chart').html().length == 0) { - Morris.Area({ - element: 'pool-area-chart', - data: [ - {foreach $POOLHASHRATES as $hour=>$hashrate} - { - period: '{$hour|default:"0"}:00', - Pool: '{$hashrate|default:"0"}', - }, - {/foreach} - ], - parseTime: false, - behaveLikeLine: true, - xkey: 'period', - ykeys: ['Pool'], - labels: ['Hashrate'], - pointSize: 2, - hideHover: 'auto', - lineColors: ['#0b62a4'], - pointFillColors: ['#FFFFFF'], - resize: true, - fillOpacity: 1.00, - postUnits: ' KH/s' - }); - } - if ($(e.target).attr('href') == '#mine' && $('#mine-area-chart').html().length == 0) { - Morris.Area({ + var chart = Morris.Line({ + // ID of the element in which to draw the chart. element: 'mine-area-chart', - data: [ - {foreach $YOURHASHRATES as $yourhour=>$yourhashrate} - { - period: '{$yourhour|default:"0"}:00', - Mine: '{$yourhashrate|default:"0"}', - }, - {/foreach} - ], - parseTime: false, - behaveLikeLine: true, - xkey: 'period', - ykeys: ['Mine'], + data: {$YOURHASHRATES}, + xkey: 'timestamp', + ykeys: ['hashrate'], labels: ['Hashrate'], pointSize: 2, hideHover: 'auto', - lineColors: ['#24A665'], - pointFillColors: ['#FFFFFF'], resize: true, fillOpacity: 1.00, - postUnits: ' KH/s' + lineColors: ['#24A665'], + pointFillColors: ['#FFFFFF'], }); } - - if ($(e.target).attr('href') == '#both' && $('#both-area-chart').html().length == 0) { - Morris.Area({ - element: 'both-area-chart', - data: [ - {foreach $YOURHASHRATES as $yourhour=>$yourhashrate} - { - period: '{$yourhour|default:"0"}:00', - Mine: '{$yourhashrate|default:"0"}', - {foreach $POOLHASHRATES as $poolhour=>$poolhashrate} - {if $yourhour eq $poolhour} - Pool: '{$poolhashrate|default:"0"}', - {/if} - {/foreach} - }, - {/foreach} - ], - parseTime: false, - behaveLikeLine: true, - xkey: 'period', - ykeys: ['Mine', 'Pool'], - labels: ['Your Hashrate', 'Pool Hashrate'], + if ($(e.target).attr('href') == '#pool' && $('#pool-area-chart').html().length == 0) { + var chart = Morris.Line({ + // ID of the element in which to draw the chart. + element: 'pool-area-chart', + data: {$POOLHASHRATES}, + xkey: 'timestamp', + ykeys: ['hashrate'], + labels: ['Hashrate'], pointSize: 2, hideHover: 'auto', resize: true, - fillOpacity: 0.1, - postUnits: ' KH/s' + fillOpacity: 1.00, + lineColors: ['#24A665'], + pointFillColors: ['#FFFFFF'], }); } - }); }); @@ -99,7 +49,7 @@ $(function () {
Stats -
+
From 226d2c8b54f2d233d90cdd9cbbef71f5d93cd1e2 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Thu, 17 Jul 2014 07:41:23 +0200 Subject: [PATCH 08/10] [ADDED] Flush statistics_users entries after 7 days default --- cronjobs/tables_cleanup.php | 15 ++++++++++++++- include/classes/statistics.class.php | 11 +++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/cronjobs/tables_cleanup.php b/cronjobs/tables_cleanup.php index 04386133d..231de9bef 100755 --- a/cronjobs/tables_cleanup.php +++ b/cronjobs/tables_cleanup.php @@ -65,7 +65,7 @@ $message = ''; $affected = $share->purgeArchive(); if ($affected === false) { - $message = 'Failed to delete notifications: ' . $oToken->getCronError(); + $message = 'Failed to delete shares: ' . $share->getCronError(); $status = 'ERROR'; $monitoring->endCronjob($cron_name, 'E0008', 0, false, false); } else { @@ -73,6 +73,19 @@ } $log->logInfo(sprintf($strLogMask, 'purgeArchive', $affected, number_format(microtime(true) - $start, 3), $status, $message)); +// Clenaup shares archive +$start = microtime(true); +$status = 'OK'; +$message = ''; +$affected = $statistics->purgeUserStats(); +if ($affected === false) { + $message = 'Failed to delete entries: ' . $statistics->getCronError(); + $status = 'ERROR'; + $monitoring->endCronjob($cron_name, 'E0008', 0, false, false); +} else { + $affected == 0 ? $message = 'No entries deleted' : $message = 'Deleted old entries'; +} +$log->logInfo(sprintf($strLogMask, 'purgeUserStats', $affected, number_format(microtime(true) - $start, 3), $status, $message)); // Cron cleanup and monitoring require_once('cron_end.inc.php'); diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index dc4a79011..216231421 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -927,6 +927,17 @@ public function getCountAllActiveUsers($interval=120) { return $this->memcache->setCache(__FUNCTION__, $result->fetch_object()->total); return $this->sqlError(); } + + /** + * Purge older entries from our statistics_users table + **/ + public function purgeUserStats($days = 7) { + // Fallbacks if unset + $stmt = $this->mysqli->prepare("DELETE FROM " . $this->getUserStatsTableName() . " WHERE FROM_UNIXTIME(timestamp) <= DATE_SUB(NOW(), INTERVAL ? DAY)"); + if ($this->checkStmt($stmt) && $stmt->bind_param('i', $days) && $stmt->execute()) + return $stmt->affected_rows; + return $this->sqlError(); + } } $statistics = new Statistics(); From 2c773fca4285e07c30a3eab0003512d31177fca6 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Thu, 17 Jul 2014 08:13:31 +0200 Subject: [PATCH 09/10] [UPDATE] Overhauled graphing stats * [REMOVED] Pool/Combined hashrate graph until we have a proper SQL to generate this data. Volunteers? * [ADDED] Worker and share rate statistics over time * [ADDED] Admin setting to change graphing days (default: 1 day) * [ADDED] Purge entries older than admin setting * [REMOVED] Template files that aren't used anymore --- cronjobs/tables_cleanup.php | 2 +- include/classes/statistics.class.php | 40 ++---- include/config/admin_settings.inc.php | 7 + include/pages/statistics/graphs.inc.php | 6 +- .../bootstrap/statistics/graphs/both.tpl | 10 -- .../bootstrap/statistics/graphs/default.tpl | 127 +++++++++++------- .../bootstrap/statistics/graphs/mine.tpl | 8 -- .../bootstrap/statistics/graphs/pool.tpl | 8 -- 8 files changed, 96 insertions(+), 112 deletions(-) delete mode 100644 templates/bootstrap/statistics/graphs/both.tpl delete mode 100644 templates/bootstrap/statistics/graphs/mine.tpl delete mode 100644 templates/bootstrap/statistics/graphs/pool.tpl diff --git a/cronjobs/tables_cleanup.php b/cronjobs/tables_cleanup.php index 231de9bef..42189f2f2 100755 --- a/cronjobs/tables_cleanup.php +++ b/cronjobs/tables_cleanup.php @@ -77,7 +77,7 @@ $start = microtime(true); $status = 'OK'; $message = ''; -$affected = $statistics->purgeUserStats(); +$affected = $statistics->purgeUserStats($setting->getValue('statistics_graphing_days', 1)); if ($affected === false) { $message = 'Failed to delete entries: ' . $statistics->getCronError(); $status = 'ERROR'; diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index 216231421..923d8cfa3 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -698,19 +698,21 @@ public function getTopContributors($type='shares', $limit=15) { * @param $account_id int account id * @return data array NOT FINISHED YET **/ - public function getHashrateByAccount($account_id, $format='array') { + public function getHourlyMiningStatsByAccount($account_id, $format='array', $days = 1) { $this->debug->append("STA " . __METHOD__, 4); if ($data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT timestamp, FROM_UNIXTIME(timestamp, '%Y-%m-%d %H:%i') AS time, - AVG(hashrate) AS hashrate + AVG(hashrate) AS hashrate, + AVG(workers) AS workers, + AVG(sharerate) AS sharerate FROM " . $this->getUserStatsTableName() . " - WHERE FROM_UNIXTIME(timestamp) >= DATE_SUB(NOW(), INTERVAL 24 HOUR) + WHERE FROM_UNIXTIME(timestamp) >= DATE_SUB(NOW(), INTERVAL $days DAY) AND account_id = ? - GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); - if ($this->checkStmt($stmt) && $stmt->bind_param('i', $account_id ) && $stmt->execute() && $result = $stmt->get_result()) { + GROUP BY DAY(FROM_UNIXTIME(timestamp)), HOUR(FROM_UNIXTIME(timestamp))"); + if ($this->checkStmt($stmt) && $stmt->bind_param('i', $account_id) && $stmt->execute() && $result = $stmt->get_result()) { $aData = $result->fetch_all(MYSQLI_ASSOC); if ($format == 'json') $aData = json_encode($aData); return $this->memcache->setCache(__FUNCTION__ . $account_id . $format, $aData); @@ -718,32 +720,6 @@ public function getHashrateByAccount($account_id, $format='array') { return $this->sqlError(); } - /** - * get Hourly hashrate for the pool - * @param none - * @return data array NOT FINISHED YET - **/ - public function getHashrateForPool($format='array') { - $this->debug->append("STA " . __METHOD__, 4); - if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data; - $stmt = $this->mysqli->prepare(" - SELECT - timestamp, - FROM_UNIXTIME(timestamp, '%Y-%m-%d %T') AS time, - SUM(DISTINCT account_id) - FROM " . $this->getUserStatsTableName() . " - WHERE timestamp >= DATE_SUB(NOW(), INTERVAL 24 HOUR) - GROUP BY HOUR(FROM_UNIXTIME(timestamp))"); - if ($this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result()) { - // return json_encode(array(time() * 1000, 1000)); - $aData = $result->fetch_all(MYSQLI_ASSOC); - var_dump($aData); - if ($format == 'json') $aData = json_encode($aData); - return $this->memcache->setCache(__FUNCTION__ . $format, $aData); - } - return $this->sqlError(); - } - /** * get user estimated payouts based on share counts * @param value1 mixed Round shares OR share rate @@ -931,7 +907,7 @@ public function getCountAllActiveUsers($interval=120) { /** * Purge older entries from our statistics_users table **/ - public function purgeUserStats($days = 7) { + public function purgeUserStats($days = 1) { // Fallbacks if unset $stmt = $this->mysqli->prepare("DELETE FROM " . $this->getUserStatsTableName() . " WHERE FROM_UNIXTIME(timestamp) <= DATE_SUB(NOW(), INTERVAL ? DAY)"); if ($this->checkStmt($stmt) && $stmt->bind_param('i', $days) && $stmt->execute()) diff --git a/include/config/admin_settings.inc.php b/include/config/admin_settings.inc.php index 4f8618305..2ae6716e3 100644 --- a/include/config/admin_settings.inc.php +++ b/include/config/admin_settings.inc.php @@ -160,6 +160,13 @@ 'name' => 'statistics_ajax_data_interval', 'value' => $setting->getValue('statistics_ajax_data_interval'), 'tooltip' => 'Time in minutes, interval for hashrate and sharerate calculations. Higher intervals allow for better accuracy at a higer server load.' ); +$aSettings['statistics'][] = array( + 'display' => 'Graphing Days', 'type' => 'text', + 'size' => 25, + 'default' => 1, + 'name' => 'statistics_graphing_days', 'value' => $setting->getValue('statistics_graphing_days'), + 'tooltip' => 'How many days to graph out on the statistics -> graphs page.' +); $aSettings['statistics'][] = array( 'display' => 'Block Statistics Count', 'type' => 'text', 'size' => 25, diff --git a/include/pages/statistics/graphs.inc.php b/include/pages/statistics/graphs.inc.php index c970b8d84..a212e941d 100644 --- a/include/pages/statistics/graphs.inc.php +++ b/include/pages/statistics/graphs.inc.php @@ -4,11 +4,9 @@ if (!$smarty->isCached('master.tpl', $smarty_cache_key)) { $debug->append('No cached version available, fetching from backend', 3); if ($user->isAuthenticated()) { - $aHourlyHashRates = $statistics->getHashrateByAccount($_SESSION['USERDATA']['id'], 'json'); - $aPoolHourlyHashRates = $statistics->getHashrateForPool('json'); + $aHourlyMiningStats = $statistics->getHourlyMiningStatsByAccount($_SESSION['USERDATA']['id'], 'json', $setting->getValue('statistics_graphing_days', 1)); } - $smarty->assign("YOURHASHRATES", @$aHourlyHashRates); - $smarty->assign("POOLHASHRATES", @$aPoolHourlyHashRates); + $smarty->assign('YOURMININGSTATS', @$aHourlyMiningStats); } else { $debug->append('Using cached page', 3); } diff --git a/templates/bootstrap/statistics/graphs/both.tpl b/templates/bootstrap/statistics/graphs/both.tpl deleted file mode 100644 index a4943f857..000000000 --- a/templates/bootstrap/statistics/graphs/both.tpl +++ /dev/null @@ -1,10 +0,0 @@ -{if is_array($YOURHASHRATES) && is_array($POOLHASHRATES)} -
-
- Your vs. Pool Hashrate -
-
-
-
-
-{/if} diff --git a/templates/bootstrap/statistics/graphs/default.tpl b/templates/bootstrap/statistics/graphs/default.tpl index 679b68853..b2007f715 100644 --- a/templates/bootstrap/statistics/graphs/default.tpl +++ b/templates/bootstrap/statistics/graphs/default.tpl @@ -1,47 +1,48 @@ @@ -50,17 +51,45 @@ $(function () {
- Stats + Average Hashrate past 24h +
+
+
+
+
+ Your average hashrate per hour, updated every backend cron run. +
+
+
+ + +
+
+
+
+ Average Workers past 24h
- -
- {include file="{$smarty.request.page|escape}/{$smarty.request.action|escape}/mine.tpl"} - {include file="{$smarty.request.page|escape}/{$smarty.request.action|escape}/pool.tpl"} -
+
+
+
+ Your average active workers per hour, updated every backend cron run. +
+
+
+
+ +
+
+
+
+ Average Sharerate past 24h +
+
+
+
+
+ Your share rate per hour, updated every backend cron run.
diff --git a/templates/bootstrap/statistics/graphs/mine.tpl b/templates/bootstrap/statistics/graphs/mine.tpl deleted file mode 100644 index 91c8c53ad..000000000 --- a/templates/bootstrap/statistics/graphs/mine.tpl +++ /dev/null @@ -1,8 +0,0 @@ -
-
- Your Hashrate -
-
-
-
-
diff --git a/templates/bootstrap/statistics/graphs/pool.tpl b/templates/bootstrap/statistics/graphs/pool.tpl deleted file mode 100644 index 95450674c..000000000 --- a/templates/bootstrap/statistics/graphs/pool.tpl +++ /dev/null @@ -1,8 +0,0 @@ -
-
- Pool Hashrate -
-
-
-
-
From feb958677455ba2af9c54aded821ecf1d70b058a Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Thu, 17 Jul 2014 09:37:40 +0200 Subject: [PATCH 10/10] [CHANGE] Shares/s to diff1 shares/s --- include/classes/statistics.class.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/classes/statistics.class.php b/include/classes/statistics.class.php index 923d8cfa3..e1005dade 100644 --- a/include/classes/statistics.class.php +++ b/include/classes/statistics.class.php @@ -261,12 +261,12 @@ public function getCurrentShareRate($interval=180) { SELECT ( ( - SELECT ROUND(COUNT(id) / ?, 2) AS sharerate + SELECT ROUND(SUM(difficulty) / ?, 2) AS sharerate FROM " . $this->share->getTableName() . " WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' ) + ( - SELECT ROUND(COUNT(id) / ?, 2) AS sharerate + SELECT ROUND(SUM(difficulty) / ?, 2) AS sharerate FROM " . $this->share->getArchiveTableName() . " WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' @@ -470,7 +470,7 @@ public function fetchAllUserMiningStats($interval=180) { a.username AS account, COUNT(DISTINCT t1.username) AS workers, IFNULL(SUM(t1.difficulty), 0) AS shares, - ROUND(COUNT(t1.id) / ?, 2) AS sharerate, + ROUND(SUM(t1.difficulty) / ?, 2) AS sharerate, IFNULL(AVG(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS avgsharediff FROM ( SELECT @@ -558,7 +558,7 @@ public function getUserUnpaidPPSShares($username, $account_id=NULL, $last_paid_p * Get Shares per x interval by user * @param username string username * @param $account_id int account id - * @return data integer Current Sharerate in shares/s + * @return data integer Current Sharerate in diff1 shares/s **/ public function getUserMiningStats($username, $account_id=NULL, $interval=180) { $this->debug->append("STA " . __METHOD__, 4); @@ -575,7 +575,7 @@ public function getUserMiningStats($username, $account_id=NULL, $interval=180) { if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT - IFNULL(COUNT(*) / ?, 0) AS sharerate, + IFNULL(SUM(difficulty) / ?, 0) AS sharerate, IFNULL(SUM(difficulty), 0) AS shares, IFNULL(AVG(difficulty), 0) AS avgsharediff FROM (