Permalink
Browse files

[REDIS DATA BREAKING CHANGES] More efficient block storage format for…

… redis. Limited number of blocks sent to front-end. Expanded admin page. More realistic 'luck' value. Run redisBlocksUpgrade.js script to convert block data to new format.
  • Loading branch information...
1 parent b3f0af0 commit b4f49e953ab9e6138f33c48bac824d0ceb4eba3c @zone117x committed Jul 4, 2014
Showing with 611 additions and 180 deletions.
  1. +3 −1 README.md
  2. +1 −0 config_example.json
  3. +1 −18 init.js
  4. +81 −33 lib/api.js
  5. +58 −39 lib/blockUnlocker.js
  6. +20 −0 lib/configReader.js
  7. +24 −5 lib/pool.js
  8. +191 −0 redisBlocksUpgrade.js
  9. +113 −12 website/admin.html
  10. +119 −72 website/index.html
View
@@ -256,7 +256,9 @@ Explanation for each field:
"enabled": true,
"hashrateWindow": 600, //how many second worth of shares used to estimate hash rate
"updateInterval": 3, //gather stats and broadcast every this many seconds
- "port": 8117
+ "port": 8117,
+ "blocks": 30, //amount of blocks to send at a time
+ "password": "test" //password required for admin stats
},
/* Coin daemon connection details. */
View
@@ -82,6 +82,7 @@
"hashrateWindow": 600,
"updateInterval": 5,
"port": 8117,
+ "blocks": 30,
"password": "your_password"
},
View
@@ -5,24 +5,7 @@ var os = require('os');
var redis = require('redis');
-var configFile = (function(){
- for (var i = 0; i < process.argv.length; i++){
- if (process.argv[i].indexOf('-config=') === 0)
- return process.argv[i].split('=')[1];
- }
- return 'config.json';
-})();
-
-
-try {
- global.config = JSON.parse(fs.readFileSync(configFile));
-}
-catch(e){
- console.error('Failed to read config file ' + configFile + '\n\n' + e);
- return;
-}
-
-config.version = "v0.99.0.6";
+require('./lib/configReader.js');
require('./lib/logger.js');
View
@@ -13,13 +13,13 @@ require('./exceptionWriter.js')(logSystem);
var redisCommands = [
['zremrangebyscore', config.coin + ':hashrate', '-inf', ''],
- ['zrangebyscore', config.coin + ':hashrate', '', '+inf'],
- ['hgetall', config.coin + ':stats'],
- ['smembers', config.coin + ':blocksPending'],
- ['smembers', config.coin + ':blocksUnlocked'],
- ['smembers', config.coin + ':blocksOrphaned'],
+ ['zrange', config.coin + ':hashrate', 0, -1],
+ ['hgetall', config.coin + ':stats'],
+ ['zrange', config.coin + ':blocks:candidates', 0, -1, 'WITHSCORES'],
+ ['zrevrange', config.coin + ':blocks:matured', 0, config.api.blocks, 'WITHSCORES'],
['hgetall', config.coin + ':shares:roundCurrent'],
- ['hgetall', config.coin + ':stats']
+ ['hgetall', config.coin + ':stats'],
+ ['zcard', config.coin + ':blocks:matured']
];
var currentStats = "";
@@ -56,11 +56,8 @@ function collectStats(){
var data = {
stats: replies[2],
- blocks: {
- pending: replies[3],
- unlocked: replies[4],
- orphaned: replies[5]
- }
+ blocks: replies[3].concat(replies[4]),
+ totalBlocks: parseInt(replies[7]) + replies[3].length
};
var hashrates = replies[1];
@@ -86,14 +83,14 @@ function collectStats(){
data.roundHashes = 0;
- if (replies[6]){
- for (var miner in replies[6]){
- data.roundHashes += parseInt(replies[6][miner]);
+ if (replies[5]){
+ for (var miner in replies[5]){
+ data.roundHashes += parseInt(replies[5][miner]);
}
}
- if (replies[7]) {
- data.lastBlockFound = replies[7].lastBlockFound;
+ if (replies[6]) {
+ data.lastBlockFound = replies[6].lastBlockFound;
}
callback(null, data);
@@ -227,6 +224,28 @@ function formatMinerStats(redisData, address){
}
+function handleGetBlocks(urlParts, response){
+ redisClient.zrevrangebyscore(config.coin + ':blocks:matured', '(' + urlParts.query.height, '-inf', 'WITHSCORES', 'LIMIT', 0, config.api.blocks, function(err, result){
+
+ var reply;
+
+ if (err)
+ reply = JSON.stringify({error: 'query failed'});
+ else
+ reply = JSON.stringify(result);
+
+ response.writeHead("200", {
+ 'Access-Control-Allow-Origin': '*',
+ 'Cache-Control': 'no-cache',
+ 'Content-Type': 'application/json',
+ 'Content-Length': reply.length
+ });
+ response.end(reply);
+
+ });
+}
+
+
collectStats();
function authorize(request, response){
@@ -252,21 +271,24 @@ function handleAdminStats(response){
async.waterfall([
- //Get worker keys
+ //Get worker keys & unlocked blocks
function(callback){
- redisClient.keys(config.coin + ':workers:*', function(error, result) {
+ redisClient.multi([
+ ['keys', config.coin + ':workers:*'],
+ ['zrange', config.coin + ':blocks:matured', 0, -1]
+ ]).exec(function(error, replies) {
if (error) {
- log('error', logSystem, 'Error trying to get worker balances from redis %j', [error]);
+ log('error', logSystem, 'Error trying to get admin data from redis %j', [error]);
callback(true);
return;
}
- callback(null, result);
+ callback(null, replies[0], replies[1]);
});
},
//Get worker balances
- function(keys, callback){
- var redisCommands = keys.map(function(k){
+ function(workerKeys, blocks, callback){
+ var redisCommands = workerKeys.map(function(k){
return ['hmget', k, 'balance', 'paid'];
});
redisClient.multi(redisCommands).exec(function(error, replies){
@@ -276,19 +298,42 @@ function handleAdminStats(response){
return;
}
- var stats = {
- totalOwed: 0,
- totalPaid: 0
- };
+ callback(null, replies, blocks);
+ });
+ },
+ function(workerData, blocks, callback){
+ var stats = {
+ totalOwed: 0,
+ totalPaid: 0,
+ totalRevenue: 0,
+ totalDiff: 0,
+ totalShares: 0,
+ blocksOrphaned: 0,
+ blocksUnlocked: 0,
+ totalWorkers: 0
+ };
+
+ for (var i = 0; i < workerData.length; i++){
+ stats.totalOwed += parseInt(workerData[i][0]) || 0;
+ stats.totalPaid += parseInt(workerData[i][1]) || 0;
+ stats.totalWorkers++;
+ }
- for (var i = 0; i < replies.length; i++){
- stats.totalOwed += parseInt(replies[i][0]) || 0;
- stats.totalPaid += parseInt(replies[i][1]) || 0;
+ for (var i = 0; i < blocks.length; i++){
+ var block = blocks[i].split(':');
+ if (block[5]) {
+ stats.blocksUnlocked++;
+ stats.totalDiff += parseInt(block[2]);
+ stats.totalShares += parseInt(block[3]);
+ stats.totalRevenue += parseInt(block[5]);
}
-
- callback(null, stats);
- });
- }], function(error, stats){
+ else{
+ stats.blocksOrphaned++;
+ }
+ }
+ callback(null, stats);
+ }
+ ], function(error, stats){
if (error){
response.end(JSON.stringify({error: 'error collecting stats'}));
return;
@@ -346,6 +391,9 @@ var server = http.createServer(function(request, response){
case '/stats_address':
handleMinerStats(urlParts, response);
break;
+ case '/get_blocks':
+ handleGetBlocks(urlParts, response);
+ break;
case '/admin_stats':
if (!authorize(request, response))
return;
View
@@ -32,28 +32,34 @@ var doDonations = config.blockUnlocker.devDonation > 0 && devDonationAddress[0]
function runInterval(){
async.waterfall([
- //Get all pending blocks in redis
+ //Get all block candidates in redis
function(callback){
- redisClient.smembers(config.coin + ':blocksPending', function(error, result){
+ redisClient.zrange(config.coin + ':blocks:candidates', 0, -1, 'WITHSCORES', function(error, results){
if (error){
log('error', logSystem, 'Error trying to get pending blocks from redis %j', [error]);
callback(true);
return;
}
- if (result.length === 0){
- log('info', logSystem, 'No pending blocks in redis');
+ if (results.length === 0){
+ log('info', logSystem, 'No blocks candidates in redis');
callback(true);
return;
}
- var blocks = result.map(function(item){
- var parts = item.split(':');
- return {
- height: parseInt(parts[0]),
- difficulty: parseInt(parts[1]),
- hash: parts[2],
- serialized: item
- };
- });
+
+ var blocks = [];
+
+ for (var i = 0; i < results.length; i += 2){
+ var parts = results[i].split(':');
+ blocks.push({
+ serialized: results[i],
+ height: parseInt(results[i + 1]),
+ hash: parts[0],
+ time: parts[1],
+ difficulty: parts[2],
+ shares: parts[3]
+ });
+ }
+
callback(null, blocks);
});
},
@@ -75,15 +81,15 @@ function runInterval(){
return;
}
var blockHeader = result.block_header;
- block.orphan = (blockHeader.hash !== block.hash);
+ block.orphaned = blockHeader.hash === block.hash ? 0 : 1;
block.unlocked = blockHeader.depth >= config.blockUnlocker.depth;
block.reward = blockHeader.reward;
mapCback(block.unlocked);
});
}, function(unlockedBlocks){
if (unlockedBlocks.length === 0){
- log('info', logSystem, 'No pending blocks are unlocked or orphaned yet (%d pending)', [blocks.length]);
+ log('info', logSystem, 'No pending blocks are unlocked yet (%d pending)', [blocks.length]);
callback(true);
return;
}
@@ -110,9 +116,6 @@ function runInterval(){
for (var i = 0; i < replies.length; i++){
var workerShares = replies[i];
blocks[i].workerShares = workerShares;
- blocks[i].totalShares = Object.keys(workerShares).reduce(function(p, c){
- return p + parseInt(workerShares[c])
- }, 0);
}
callback(null, blocks);
});
@@ -121,19 +124,29 @@ function runInterval(){
//Handle orphaned blocks
function(blocks, callback){
var orphanCommands = [];
+
blocks.forEach(function(block){
- if (!block.orphan) return;
- var workerShares = block.workerShares;
- orphanCommands.push(['del', config.coin + ':shares:round' + block.height]);
+ if (!block.orphaned) return;
- orphanCommands.push(['srem', config.coin + ':blocksPending', block.serialized]);
- orphanCommands.push(['sadd', config.coin + ':blocksOrphaned', block.serialized + ':' + block.totalShares]);
+ orphanCommands.push(['del', config.coin + ':shares:round' + block.height]);
- Object.keys(workerShares).forEach(function(worker){
- orphanCommands.push(['hincrby', config.coin + ':shares:roundCurrent',
- worker, workerShares[worker]]);
- });
+ orphanCommands.push(['zrem', config.coin + ':blocks:candidates', block.serialized]);
+ orphanCommands.push(['zadd', config.coin + ':blocks:matured', block.height, [
+ block.hash,
+ block.time,
+ block.difficulty,
+ block.shares,
+ block.orphaned
+ ].join(':')]);
+
+ if (block.workerShares) {
+ var workerShares = block.workerShares;
+ Object.keys(workerShares).forEach(function (worker) {
+ orphanCommands.push(['hincrby', config.coin + ':shares:roundCurrent', worker, workerShares[worker]]);
+ });
+ }
});
+
if (orphanCommands.length > 0){
redisClient.multi(orphanCommands).exec(function(error, replies){
if (error){
@@ -155,13 +168,19 @@ function runInterval(){
var payments = {};
var totalBlocksUnlocked = 0;
blocks.forEach(function(block){
- if (block.orphan) return;
+ if (block.orphaned) return;
totalBlocksUnlocked++;
unlockedBlocksCommands.push(['del', config.coin + ':shares:round' + block.height]);
-
- unlockedBlocksCommands.push(['srem', config.coin + ':blocksPending', block.serialized]);
- unlockedBlocksCommands.push(['sadd', config.coin + ':blocksUnlocked', block.serialized + ':' + block.totalShares]);
+ unlockedBlocksCommands.push(['zrem', config.coin + ':blocks:candidates', block.serialized]);
+ unlockedBlocksCommands.push(['zadd', config.coin + ':blocks:matured', block.height, [
+ block.hash,
+ block.time,
+ block.difficulty,
+ block.shares,
+ block.orphaned,
+ block.reward
+ ].join(':')]);
var feePercent = config.blockUnlocker.poolFee / 100;
@@ -173,12 +192,14 @@ function runInterval(){
var reward = block.reward - (block.reward * feePercent);
- var totalShares = block.totalShares;
- Object.keys(block.workerShares).forEach(function(worker){
- var percent = block.workerShares[worker] / totalShares;
- var workerReward = reward * percent;
- payments[worker] = (payments[worker] || 0) + workerReward;
- });
+ if (block.workerShares) {
+ var totalShares = parseInt(block.shares);
+ Object.keys(block.workerShares).forEach(function (worker) {
+ var percent = block.workerShares[worker] / totalShares;
+ var workerReward = reward * percent;
+ payments[worker] = (payments[worker] || 0) + workerReward;
+ });
+ }
});
for (var worker in payments) {
@@ -205,9 +226,7 @@ function runInterval(){
log('info', logSystem, 'Unlocked %d blocks and update balances for %d workers', [totalBlocksUnlocked, Object.keys(payments).length]);
callback(null);
});
-
}
-
], function(error, result){
setTimeout(runInterval, config.blockUnlocker.interval * 1000);
})
Oops, something went wrong.

1 comment on commit b4f49e9

@skygong
skygong commented on b4f49e9 Jul 4, 2014

The use of version 992,
Pool Hash Rate: 0 H/sec!
Your Stats Hash Rate: 0 H/sec!

Please sign in to comment.