Skip to content

Commit

Permalink
Payout Operations Analysis and Patch for Setup
Browse files Browse the repository at this point in the history 
Two main additions:
* The operations types author_payout, curator_payout and benefactor_payout are now analysed, formatted and inserted to MongoDB. This gives the payout amounts for the analysis by application. "Upsert" has been used which creates a new record even if the initial comment has not been processed. This is to allow blocks to be processed in any order.
* A patch was required for the blockDates setup following the accidental move to hardfork20 and reset. This was simply due to the large number of missing blocks making the estimation of the first block of the day inaccurate. The workaround estimation technique has been improved.

mongoblock.js
* processAuthorReward, processBenefactorReward, processCuratorReward: These functions analyse and format each block operation of the stated type.
* mongoAuthorReward, mongoBenefactorReward, mongoCuratorReward: These functions upsert the records to MongoDB.
* reportBlocksProcessed: This function reports on the status of blocks processed.

blockOps.js
updateBlockDates: The blockDates setup patch is here.
  • Loading branch information
@miniature-tiger
miniature-tiger committed Sep 18, 2018
1 parent 0f2d51e commit 120fc51
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 13 deletions.
29 changes: 20 additions & 9 deletions blockOps.js
View file
Expand Up @@ -64,7 +64,7 @@ async function updateBlockDates() {
let latestB = await steemrequest.getLatestBlockNumber();
console.log('Latest block:', latestB.blockNumber, latestB.timestamp);

let loopBlock = 1, loopDate = new Date(), loopStartSet = false, record = {};
let loopBlock = 1, loopDate = new Date(), loopStartSet = false, record = {}, secondsPerBlock = 3;

// Checks whether blockDates have previously been loaded
// If true uses most recent blockDate as starting point for loading further blockDates
Expand All @@ -75,9 +75,10 @@ async function updateBlockDates() {
.then(function(maxDate) {
if(maxDate.length == 1) {
console.log('Latest blockdate loaded:', maxDate[0].blockNumber, maxDate[0].timestamp);
loopBlock = maxDate[0].blockNumber + (24*60*20);
loopBlock = Math.min(maxDate[0].blockNumber + (24*60*20), latestB.blockNumber);
loopDate = helperblock.forwardOneDay(maxDate[0].timestamp);
loopStartSet = true;
console.log('Attempted block:', loopBlock, loopDate);
}
}).catch(function(error) {
console.log(error)
Expand All @@ -99,7 +100,7 @@ async function updateBlockDates() {
console.log('block 1 - logged on db:')
// go forward one day
loopDate = helperblock.forwardOneDay(loopDate);
loopBlock = loopBlock - helperblock.blocksToMidnight(startB.timestamp, loopDate);
loopBlock = loopBlock - helperblock.blocksToMidnight(startB.timestamp, loopDate, 3);

collection.createIndex({timestamp: 1}, {unique:true})
.catch(function(error) {
Expand Down Expand Up @@ -127,18 +128,22 @@ async function updateBlockDates() {
loopBlock = loopBlock + (24*60*20);
loopDate = helperblock.forwardOneDay(loopDate);
counter = 0, attemptArray = [];
secondsPerBlock = 3;
// If block is not first block revise estimate of first blockNumber based on 3 second blocks
} else {
console.log('...recalculating');
counter += 1;
attemptArray.push(loopBlock);
attemptArray.push({blockNumber: loopBlock, timestamp: firstB.timestamp});
// Workaround for when estimation process gets stuck
if (counter % 5 == 0) {
loopBlock = Math.round((attemptArray[attemptArray.length-1] + attemptArray[attemptArray.length-2])/2);
console.log('moving to average of last two block numbers');
console.log(attemptArray);
secondsPerBlock = Math.round((attemptArray[attemptArray.length-1].timestamp - attemptArray[attemptArray.length-2].timestamp) / (attemptArray[attemptArray.length-1].blockNumber - attemptArray[attemptArray.length-2].blockNumber)/1000);
//console.log( (attemptArray[attemptArray.length-1].timestamp - attemptArray[attemptArray.length-2].timestamp) , (attemptArray[attemptArray.length-1].blockNumber - attemptArray[attemptArray.length-2].blockNumber) , secondsPerBlock );
console.log('adjusting seconds per block to ' + secondsPerBlock + ' seconds due to large number of missing blocks. Resets once first block found.');
loopBlock = Math.min(loopBlock - helperblock.blocksToMidnight(firstB.timestamp, loopDate, secondsPerBlock), latestB.blockNumber);
// blocksToMidnight revises estimate based on 3 second blocks
} else {
loopBlock = loopBlock - helperblock.blocksToMidnight(firstB.timestamp, loopDate);
loopBlock = Math.min(loopBlock - helperblock.blocksToMidnight(firstB.timestamp, loopDate, secondsPerBlock), latestB.blockNumber);
}
}
console.log('----------------');
Expand Down Expand Up @@ -173,10 +178,10 @@ async function removeCollection() {
// ----------------------------------------------------------
async function checkAllBlockDates() {
// Opening MongoDB
client = await MongoClient.connect(url, { useNewUrlParser: true });
console.log('Connected to server.');
client = await MongoClient.connect(url, { useNewUrlParser: true })
const db = client.db(dbName);
const collection = db.collection('blockDates');
console.log('Connected to server.');

// Gets first and last dates to check
let latestB = await steemrequest.getLatestBlockNumber();
Expand Down Expand Up @@ -283,6 +288,12 @@ async function fillOperations() {
}
if (operation.op[0] == 'comment') {
mongoblock.processComment(operation, mongoblock.mongoComment, db);
} else if (operation.op[0] == 'author_reward') {
mongoblock.processAuthorReward(operation, mongoblock.mongoAuthorReward, db);
} else if (operation.op[0] == 'comment_benefactor_reward') {
mongoblock.processBenefactorReward(operation, mongoblock.mongoBenefactorReward, db);
} else if (operation.op[0] == 'curation_reward') {
mongoblock.processCuratorReward(operation, mongoblock.mongoCuratorReward, db);
} else {
// Operations not handled:
if (!unknownOperations.includes(operation.op[0])) {
Expand Down
4 changes: 2 additions & 2 deletions helperblock.js
View file
Expand Up @@ -34,9 +34,9 @@ module.exports.forwardOneDay = forwardOneDay;

// Function calculates the number of 3 second blocks between the firstDate and Midnight of the secondDate
// ------------------------------------------------------------------------------------------------------
function blocksToMidnight(firstDate, secondDate) {
function blocksToMidnight(firstDate, secondDate, secondsPerBlock) {
let secondDateMidnight = new Date(Date.UTC(secondDate.getUTCFullYear(), secondDate.getUTCMonth(), secondDate.getUTCDate()));
let result = Math.round((firstDate - secondDateMidnight)/1000/3);
let result = Math.round((firstDate - secondDateMidnight)/1000/secondsPerBlock);
return result;
}

Expand Down
93 changes: 91 additions & 2 deletions mongoblock.js
View file
Expand Up @@ -15,7 +15,7 @@ async function checkCollectionExists(db, collectionName) {
}
}
}).catch(function(error) {
console.log(error);
console.log('checkCollectionExists', error);
});
return result;
}
Expand Down Expand Up @@ -75,6 +75,87 @@ module.exports.mongoComment = mongoComment;



// Author Reward - processing of block operation
// ---------------------------------------------
function processAuthorReward(operation, mongoAuthorReward, db) {
let record = {author: operation.op[1].author, permlink: operation.op[1].permlink, author_payout: {sbd: Number(operation.op[1].sbd_payout.split(' ', 1)[0]), steem: Number(operation.op[1].steem_payout.split(' ', 1)[0]), vests: Number(operation.op[1].vesting_payout.split(' ', 1)[0])}};
mongoAuthorReward(db, record);
}

module.exports.processAuthorReward = processAuthorReward;



// Author Reward - update / insert of mongo record
// -----------------------------------------------
function mongoAuthorReward(db, localRecord) {
// Uses upsert - blocks may be processed in any order so author/permlink record may already exist if another operation is processed first
db.collection('comments').updateOne({ author: localRecord.author, permlink: localRecord.permlink, author_payout: {$exists : false}}, {$set: localRecord, $addToSet: {operations: 'author_payout'}}, {upsert: true})
.catch(function(error) {
if(error.code != 11000) {
console.log(error); // ignore 11000 errors as there are many duplicated operations in AppBase
}
});
}

module.exports.mongoAuthorReward = mongoAuthorReward;



// Benefactor Reward - processing of block operation
// -------------------------------------------------
function processBenefactorReward(operation, mongoBenefactorReward, db) {
let record = {author: operation.op[1].author, permlink: operation.op[1].permlink, benefactor: operation.op[1].benefactor, benefactor_payout: {vests: Number(operation.op[1].reward.split(' ', 1)[0])}};
mongoBenefactorReward(db, record);
}

module.exports.processBenefactorReward = processBenefactorReward;



// Benefactor Reward - update / insert of mongo record
// -----------------------------------------------
function mongoBenefactorReward(db, localRecord) {
// Uses upsert - blocks may be processed in any order so author/permlink record may already exist if another operation is processed first
db.collection('comments').updateOne({ author: localRecord.author, permlink: localRecord.permlink, "benefactors.user": {$ne: localRecord.benefactor}}, {$inc: {"benefactor_payout.vests": localRecord.benefactor_payout.vests}, $addToSet: {operations: 'benefactor_payout', benefactors: {user: localRecord.benefactor, vests: localRecord.benefactor_payout.vests}}}, {upsert: true})
.catch(function(error) {
if(error.code != 11000) {
console.log(error); // ignore 11000 errors as there are many duplicated operations in AppBase
}
});
}

module.exports.mongoBenefactorReward = mongoBenefactorReward;



// Curator Reward - processing of block operation
// -------------------------------------------------
function processCuratorReward(operation, mongoCuratorReward, db) {
let record = {author: operation.op[1].comment_author, permlink: operation.op[1].comment_permlink, curator: operation.op[1].curator, curator_payout: {vests: Number(operation.op[1].reward.split(' ', 1)[0])}};
mongoCuratorReward(db, record);
}

module.exports.processCuratorReward = processCuratorReward;



// Curator Reward - update / insert of mongo record
// -----------------------------------------------
function mongoCuratorReward(db, localRecord) {
// Uses upsert - blocks may be processed in any order so author/permlink record may already exist if another operation is processed first
db.collection('comments').updateOne({ author: localRecord.author, permlink: localRecord.permlink, "curators.user": {$ne: localRecord.curator}}, {$inc: {"curator_payout.vests": localRecord.curator_payout.vests}, $addToSet: {operations: 'curator_payout', curators: {user: localRecord.curator, vests: localRecord.curator_payout.vests}}}, {upsert: true})
.catch(function(error) {
if(error.code != 11000) {
console.log(error); // ignore 11000 errors as there are many duplicated operations in AppBase
}
});
}

module.exports.mongoCuratorReward = mongoCuratorReward;



// Function reports on comments
// ----------------------------
async function reportComments(MongoClient, url, dbName) {
Expand All @@ -85,16 +166,24 @@ async function reportComments(MongoClient, url, dbName) {

collection.aggregate([
{ $match : {operations: 'comment'}},
{ $project : {_id: 0, application: 1, transactionType: 1}},
{ $project : {_id: 0, application: 1, transactionType: 1, author_payout: 1, benefactor_payout: 1, curator_payout: 1}},
{ $group : {_id : {application : "$application"},
posts: { $sum: 1 },
author_payout_sbd: {$sum: "$author_payout.sbd"}, author_payout_steem: {$sum: "$author_payout.steem"}, author_payout_vests: {$sum: "$author_payout.vests"},
benefactor_payout_vests: {$sum: "$benefactor_payout.vests"},
curator_payout_vests: {$sum: "$curator_payout.vests"},
}},
{ $sort : {posts:-1}}
]).toArray()
.then(function(records) {
for (let record of records) {
record.application = record._id.application;
delete record._id;
record.author_payout_sbd = Number(record.author_payout_sbd.toFixed(3));
record.author_payout_steem = Number(record.author_payout_steem.toFixed(3));
record.author_payout_vests = Number(record.author_payout_vests.toFixed(6));
record.benefactor_payout_vests = Number(record.benefactor_payout_vests.toFixed(6));
record.curator_payout_vests = Number(record.curator_payout_vests.toFixed(6));
console.log(record);
}
console.log('closing mongo db');
Expand Down

0 comments on commit 120fc51

Please sign in to comment.