Permalink
Browse files

[REDIS DATA BREAKING CHANGES] More efficient block storage format for…

… redis. Limited number of blocks sent to front-end. Expanded admin page. More realistic 'luck' value. Run redisBlocksUpgrade.js script to convert block data to new format.
  • Loading branch information...
zone117x committed Jul 4, 2014
1 parent b3f0af0 commit b4f49e953ab9e6138f33c48bac824d0ceb4eba3c
Showing with 611 additions and 180 deletions.
  1. +3 −1 README.md
  2. +1 −0 config_example.json
  3. +1 −18 init.js
  4. +81 −33 lib/api.js
  5. +58 −39 lib/blockUnlocker.js
  6. +20 −0 lib/configReader.js
  7. +24 −5 lib/pool.js
  8. +191 −0 redisBlocksUpgrade.js
  9. +113 −12 website/admin.html
  10. +119 −72 website/index.html
View
@@ -256,7 +256,9 @@ Explanation for each field:
"enabled": true,
"hashrateWindow": 600, //how many second worth of shares used to estimate hash rate
"updateInterval": 3, //gather stats and broadcast every this many seconds
"port": 8117
"port": 8117,
"blocks": 30, //amount of blocks to send at a time
"password": "test" //password required for admin stats
},
/* Coin daemon connection details. */
View
@@ -82,6 +82,7 @@
"hashrateWindow": 600,
"updateInterval": 5,
"port": 8117,
"blocks": 30,
"password": "your_password"
},
View
19 init.js
@@ -5,24 +5,7 @@ var os = require('os');
var redis = require('redis');
var configFile = (function(){
for (var i = 0; i < process.argv.length; i++){
if (process.argv[i].indexOf('-config=') === 0)
return process.argv[i].split('=')[1];
}
return 'config.json';
})();
try {
global.config = JSON.parse(fs.readFileSync(configFile));
}
catch(e){
console.error('Failed to read config file ' + configFile + '\n\n' + e);
return;
}
config.version = "v0.99.0.6";
require('./lib/configReader.js');
require('./lib/logger.js');
View
@@ -13,13 +13,13 @@ require('./exceptionWriter.js')(logSystem);
var redisCommands = [
['zremrangebyscore', config.coin + ':hashrate', '-inf', ''],
['zrangebyscore', config.coin + ':hashrate', '', '+inf'],
['hgetall', config.coin + ':stats'],
['smembers', config.coin + ':blocksPending'],
['smembers', config.coin + ':blocksUnlocked'],
['smembers', config.coin + ':blocksOrphaned'],
['zrange', config.coin + ':hashrate', 0, -1],
['hgetall', config.coin + ':stats'],
['zrange', config.coin + ':blocks:candidates', 0, -1, 'WITHSCORES'],
['zrevrange', config.coin + ':blocks:matured', 0, config.api.blocks, 'WITHSCORES'],
['hgetall', config.coin + ':shares:roundCurrent'],
['hgetall', config.coin + ':stats']
['hgetall', config.coin + ':stats'],
['zcard', config.coin + ':blocks:matured']
];
var currentStats = "";
@@ -56,11 +56,8 @@ function collectStats(){
var data = {
stats: replies[2],
blocks: {
pending: replies[3],
unlocked: replies[4],
orphaned: replies[5]
}
blocks: replies[3].concat(replies[4]),
totalBlocks: parseInt(replies[7]) + replies[3].length
};
var hashrates = replies[1];
@@ -86,14 +83,14 @@ function collectStats(){
data.roundHashes = 0;
if (replies[6]){
for (var miner in replies[6]){
data.roundHashes += parseInt(replies[6][miner]);
if (replies[5]){
for (var miner in replies[5]){
data.roundHashes += parseInt(replies[5][miner]);
}
}
if (replies[7]) {
data.lastBlockFound = replies[7].lastBlockFound;
if (replies[6]) {
data.lastBlockFound = replies[6].lastBlockFound;
}
callback(null, data);
@@ -227,6 +224,28 @@ function formatMinerStats(redisData, address){
}
function handleGetBlocks(urlParts, response){
redisClient.zrevrangebyscore(config.coin + ':blocks:matured', '(' + urlParts.query.height, '-inf', 'WITHSCORES', 'LIMIT', 0, config.api.blocks, function(err, result){
var reply;
if (err)
reply = JSON.stringify({error: 'query failed'});
else
reply = JSON.stringify(result);
response.writeHead("200", {
'Access-Control-Allow-Origin': '*',
'Cache-Control': 'no-cache',
'Content-Type': 'application/json',
'Content-Length': reply.length
});
response.end(reply);
});
}
collectStats();
function authorize(request, response){
@@ -252,21 +271,24 @@ function handleAdminStats(response){
async.waterfall([
//Get worker keys
//Get worker keys & unlocked blocks
function(callback){
redisClient.keys(config.coin + ':workers:*', function(error, result) {
redisClient.multi([
['keys', config.coin + ':workers:*'],
['zrange', config.coin + ':blocks:matured', 0, -1]
]).exec(function(error, replies) {
if (error) {
log('error', logSystem, 'Error trying to get worker balances from redis %j', [error]);
log('error', logSystem, 'Error trying to get admin data from redis %j', [error]);
callback(true);
return;
}
callback(null, result);
callback(null, replies[0], replies[1]);
});
},
//Get worker balances
function(keys, callback){
var redisCommands = keys.map(function(k){
function(workerKeys, blocks, callback){
var redisCommands = workerKeys.map(function(k){
return ['hmget', k, 'balance', 'paid'];
});
redisClient.multi(redisCommands).exec(function(error, replies){
@@ -276,19 +298,42 @@ function handleAdminStats(response){
return;
}
var stats = {
totalOwed: 0,
totalPaid: 0
};
callback(null, replies, blocks);
});
},
function(workerData, blocks, callback){
var stats = {
totalOwed: 0,
totalPaid: 0,
totalRevenue: 0,
totalDiff: 0,
totalShares: 0,
blocksOrphaned: 0,
blocksUnlocked: 0,
totalWorkers: 0
};
for (var i = 0; i < workerData.length; i++){
stats.totalOwed += parseInt(workerData[i][0]) || 0;
stats.totalPaid += parseInt(workerData[i][1]) || 0;
stats.totalWorkers++;
}
for (var i = 0; i < replies.length; i++){
stats.totalOwed += parseInt(replies[i][0]) || 0;
stats.totalPaid += parseInt(replies[i][1]) || 0;
for (var i = 0; i < blocks.length; i++){
var block = blocks[i].split(':');
if (block[5]) {
stats.blocksUnlocked++;
stats.totalDiff += parseInt(block[2]);
stats.totalShares += parseInt(block[3]);
stats.totalRevenue += parseInt(block[5]);
}
callback(null, stats);
});
}], function(error, stats){
else{
stats.blocksOrphaned++;
}
}
callback(null, stats);
}
], function(error, stats){
if (error){
response.end(JSON.stringify({error: 'error collecting stats'}));
return;
@@ -346,6 +391,9 @@ var server = http.createServer(function(request, response){
case '/stats_address':
handleMinerStats(urlParts, response);
break;
case '/get_blocks':
handleGetBlocks(urlParts, response);
break;
case '/admin_stats':
if (!authorize(request, response))
return;
View
@@ -32,28 +32,34 @@ var doDonations = config.blockUnlocker.devDonation > 0 && devDonationAddress[0]
function runInterval(){
async.waterfall([
//Get all pending blocks in redis
//Get all block candidates in redis
function(callback){
redisClient.smembers(config.coin + ':blocksPending', function(error, result){
redisClient.zrange(config.coin + ':blocks:candidates', 0, -1, 'WITHSCORES', function(error, results){
if (error){
log('error', logSystem, 'Error trying to get pending blocks from redis %j', [error]);
callback(true);
return;
}
if (result.length === 0){
log('info', logSystem, 'No pending blocks in redis');
if (results.length === 0){
log('info', logSystem, 'No blocks candidates in redis');
callback(true);
return;
}
var blocks = result.map(function(item){
var parts = item.split(':');
return {
height: parseInt(parts[0]),
difficulty: parseInt(parts[1]),
hash: parts[2],
serialized: item
};
});
var blocks = [];
for (var i = 0; i < results.length; i += 2){
var parts = results[i].split(':');
blocks.push({
serialized: results[i],
height: parseInt(results[i + 1]),
hash: parts[0],
time: parts[1],
difficulty: parts[2],
shares: parts[3]
});
}
callback(null, blocks);
});
},
@@ -75,15 +81,15 @@ function runInterval(){
return;
}
var blockHeader = result.block_header;
block.orphan = (blockHeader.hash !== block.hash);
block.orphaned = blockHeader.hash === block.hash ? 0 : 1;
block.unlocked = blockHeader.depth >= config.blockUnlocker.depth;
block.reward = blockHeader.reward;
mapCback(block.unlocked);
});
}, function(unlockedBlocks){
if (unlockedBlocks.length === 0){
log('info', logSystem, 'No pending blocks are unlocked or orphaned yet (%d pending)', [blocks.length]);
log('info', logSystem, 'No pending blocks are unlocked yet (%d pending)', [blocks.length]);
callback(true);
return;
}
@@ -110,9 +116,6 @@ function runInterval(){
for (var i = 0; i < replies.length; i++){
var workerShares = replies[i];
blocks[i].workerShares = workerShares;
blocks[i].totalShares = Object.keys(workerShares).reduce(function(p, c){
return p + parseInt(workerShares[c])
}, 0);
}
callback(null, blocks);
});
@@ -121,19 +124,29 @@ function runInterval(){
//Handle orphaned blocks
function(blocks, callback){
var orphanCommands = [];
blocks.forEach(function(block){
if (!block.orphan) return;
var workerShares = block.workerShares;
orphanCommands.push(['del', config.coin + ':shares:round' + block.height]);
if (!block.orphaned) return;
orphanCommands.push(['srem', config.coin + ':blocksPending', block.serialized]);
orphanCommands.push(['sadd', config.coin + ':blocksOrphaned', block.serialized + ':' + block.totalShares]);
orphanCommands.push(['del', config.coin + ':shares:round' + block.height]);
Object.keys(workerShares).forEach(function(worker){
orphanCommands.push(['hincrby', config.coin + ':shares:roundCurrent',
worker, workerShares[worker]]);
});
orphanCommands.push(['zrem', config.coin + ':blocks:candidates', block.serialized]);
orphanCommands.push(['zadd', config.coin + ':blocks:matured', block.height, [
block.hash,
block.time,
block.difficulty,
block.shares,
block.orphaned
].join(':')]);
if (block.workerShares) {
var workerShares = block.workerShares;
Object.keys(workerShares).forEach(function (worker) {
orphanCommands.push(['hincrby', config.coin + ':shares:roundCurrent', worker, workerShares[worker]]);
});
}
});
if (orphanCommands.length > 0){
redisClient.multi(orphanCommands).exec(function(error, replies){
if (error){
@@ -155,13 +168,19 @@ function runInterval(){
var payments = {};
var totalBlocksUnlocked = 0;
blocks.forEach(function(block){
if (block.orphan) return;
if (block.orphaned) return;
totalBlocksUnlocked++;
unlockedBlocksCommands.push(['del', config.coin + ':shares:round' + block.height]);
unlockedBlocksCommands.push(['srem', config.coin + ':blocksPending', block.serialized]);
unlockedBlocksCommands.push(['sadd', config.coin + ':blocksUnlocked', block.serialized + ':' + block.totalShares]);
unlockedBlocksCommands.push(['zrem', config.coin + ':blocks:candidates', block.serialized]);
unlockedBlocksCommands.push(['zadd', config.coin + ':blocks:matured', block.height, [
block.hash,
block.time,
block.difficulty,
block.shares,
block.orphaned,
block.reward
].join(':')]);
var feePercent = config.blockUnlocker.poolFee / 100;
@@ -173,12 +192,14 @@ function runInterval(){
var reward = block.reward - (block.reward * feePercent);
var totalShares = block.totalShares;
Object.keys(block.workerShares).forEach(function(worker){
var percent = block.workerShares[worker] / totalShares;
var workerReward = reward * percent;
payments[worker] = (payments[worker] || 0) + workerReward;
});
if (block.workerShares) {
var totalShares = parseInt(block.shares);
Object.keys(block.workerShares).forEach(function (worker) {
var percent = block.workerShares[worker] / totalShares;
var workerReward = reward * percent;
payments[worker] = (payments[worker] || 0) + workerReward;
});
}
});
for (var worker in payments) {
@@ -205,9 +226,7 @@ function runInterval(){
log('info', logSystem, 'Unlocked %d blocks and update balances for %d workers', [totalBlocksUnlocked, Object.keys(payments).length]);
callback(null);
});
}
], function(error, result){
setTimeout(runInterval, config.blockUnlocker.interval * 1000);
})
Oops, something went wrong.

1 comment on commit b4f49e9

@skygong

This comment has been minimized.

Show comment
Hide comment
@skygong

skygong Jul 4, 2014

The use of version 992,
Pool Hash Rate: 0 H/sec!
Your Stats Hash Rate: 0 H/sec!

skygong commented on b4f49e9 Jul 4, 2014

The use of version 992,
Pool Hash Rate: 0 H/sec!
Your Stats Hash Rate: 0 H/sec!

Please sign in to comment.