Skip to content

Commit

Permalink
Improve db storage efficiency by ~2 times
Browse files Browse the repository at this point in the history 
Encodes and decodes keys and values for leveldb storage more efficiently.
  • Loading branch information
Braydon Fuller committed Sep 14, 2015
1 parent d3641f3 commit f88eee5
Show file tree
Hide file tree
Showing 2 changed files with 215 additions and 158 deletions.
257 changes: 179 additions & 78 deletions lib/services/address/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ var bitcore = require('bitcore');
var $ = bitcore.util.preconditions;
var _ = bitcore.deps._;
var Hash = bitcore.crypto.Hash;
var BufferReader = bitcore.encoding.BufferReader;
var EventEmitter = require('events').EventEmitter;
var PublicKey = bitcore.PublicKey;
var Address = bitcore.Address;
Expand All @@ -34,10 +35,13 @@ AddressService.dependencies = [
];

AddressService.PREFIXES = {
OUTPUTS: 'outs',
SPENTS: 'sp'
OUTPUTS: new Buffer('32', 'hex'),
SPENTS: new Buffer('33', 'hex')
};

AddressService.SPACER_MIN = new Buffer('00', 'hex');
AddressService.SPACER_MAX = new Buffer('ff', 'hex');

AddressService.prototype.getAPIMethods = function() {
return [
['getBalance', this, this.getBalance, 2],
Expand Down Expand Up @@ -153,6 +157,7 @@ AddressService.prototype._extractAddressInfoFromScript = function(script) {

AddressService.prototype.blockHandler = function(block, addOutput, callback) {
var txs = block.transactions;
var height = block.__height;

var action = 'put';
if (!addOutput) {
Expand Down Expand Up @@ -188,43 +193,28 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) {
continue;
}

addressInfo.hashHex = addressInfo.hashBuffer.toString('hex');

// We need to use the height for indexes (and not the timestamp) because the
// the timestamp has unreliable sequential ordering. The next block
// can have a time that is previous to the previous block (however not
// less than the mean of the 11 previous blocks) and not greater than 2
// hours in the future.
var height = block.__height;

var scriptHex = output._scriptBuffer.toString('hex');

// To lookup outputs by address and height
var key = [
AddressService.PREFIXES.OUTPUTS,
addressInfo.hashHex,
height,
txid,
outputIndex
].join('-');

// TODO use buffers directly to save on disk storage

var value = [output.satoshis, scriptHex].join(':');

var key = this._encodeOutputKey(addressInfo.hashBuffer, height, txid, outputIndex);
var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer);
operations.push({
type: action,
key: key,
value: value
});

addressInfo.hashHex = addressInfo.hashBuffer.toString('hex');

// Collect data for subscribers
if (txmessages[addressInfo.hashHex]) {
txmessages[addressInfo.hashHex].outputIndexes.push(outputIndex);
} else {
txmessages[addressInfo.hashHex] = {
tx: tx,
height: block.__height,
height: height,
outputIndexes: [outputIndex],
addressInfo: addressInfo,
timestamp: block.header.timestamp
Expand All @@ -247,30 +237,19 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) {
for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) {

var input = inputs[inputIndex];

var inputHashBuffer;
var inputHash;

if (input.script.isPublicKeyHashIn()) {
inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf);
inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf);
} else if (input.script.isScriptHashIn()) {
inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf);
inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf);
} else {
continue;
}

// To be able to query inputs by address and spent height
var inputKey = [
AddressService.PREFIXES.SPENTS,
inputHashBuffer.toString('hex'),
block.__height,
input.prevTxId.toString('hex'),
input.outputIndex
].join('-');

var inputValue = [
txid,
inputIndex
].join(':');
var inputKey = this._encodeInputKey(inputHash, height, input.prevTxId, input.outputIndex);
var inputValue = this._encodeInputValue(txid, inputIndex);

operations.push({
type: action,
Expand All @@ -285,6 +264,104 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) {
});
};

AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txid, outputIndex) {
var heightBuffer = new Buffer(4);
heightBuffer.writeUInt32BE(height);
var outputIndexBuffer = new Buffer(4);
outputIndexBuffer.writeUInt32BE(outputIndex);
var key = Buffer.concat([
AddressService.PREFIXES.OUTPUTS,
hashBuffer,
AddressService.SPACER_MIN,
heightBuffer,
new Buffer(txid, 'hex'), //TODO get buffer directly from tx
outputIndexBuffer
]);
return key;
};

AddressService.prototype._decodeOutputKey = function(buffer) {
var reader = new BufferReader(buffer);
var prefix = reader.read(1);
var hashBuffer = reader.read(20);
var spacer = reader.read(1);
var height = reader.readUInt32BE();
var txid = reader.read(32);
var outputIndex = reader.readUInt32BE();
return {
prefix: prefix,
hashBuffer: hashBuffer,
height: height,
txid: txid,
outputIndex: outputIndex
};
};

AddressService.prototype._encodeOutputValue = function(satoshis, scriptBuffer) {
var satoshisBuffer = new Buffer(8);
satoshisBuffer.writeDoubleBE(satoshis);
return Buffer.concat([satoshisBuffer, scriptBuffer]);
};

AddressService.prototype._decodeOutputValue = function(buffer) {
var satoshis = buffer.readDoubleBE(0);
var scriptBuffer = buffer.slice(8, buffer.length);
return {
satoshis: satoshis,
scriptBuffer: scriptBuffer
};
};

AddressService.prototype._encodeInputKey = function(hashBuffer, height, prevTxIdBuffer, outputIndex) {
var heightBuffer = new Buffer(4);
heightBuffer.writeUInt32BE(height);
var outputIndexBuffer = new Buffer(4);
outputIndexBuffer.writeUInt32BE(outputIndex);
return Buffer.concat([
AddressService.PREFIXES.SPENTS,
hashBuffer,
AddressService.SPACER_MIN,
heightBuffer,
prevTxIdBuffer,
outputIndexBuffer
]);
};

AddressService.prototype._decodeInputKey = function(buffer) {
var reader = new BufferReader(buffer);
var prefix = reader.read(1);
var hashBuffer = reader.read(20);
var spacer = reader.read(1);
var height = reader.readUInt32BE();
var prevTxId = reader.read(32);
var outputIndex = reader.readUInt32BE();
return {
prefix: prefix,
hashBuffer: hashBuffer,
height: height,
prevTxId: prevTxId,
outputIndex: outputIndex
};
};

AddressService.prototype._encodeInputValue = function(txid, inputIndex) {
var inputIndexBuffer = new Buffer(4);
inputIndexBuffer.writeUInt32BE(inputIndex);
return Buffer.concat([
new Buffer(txid, 'hex'),
inputIndexBuffer
]);
};

AddressService.prototype._decodeInputValue = function(buffer) {
var txid = buffer.slice(0, 32);
var inputIndex = buffer.readUInt32BE(32);
return {
txid: txid,
inputIndex: inputIndex
};
};

/**
* @param {Object} obj
* @param {Transaction} obj.tx - The transaction
Expand Down Expand Up @@ -424,45 +501,56 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) {
var inputs = [];
var stream;

var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex');
var hashBuffer = bitcore.Address(addressStr).hashBuffer;

if (options.start && options.end) {

// The positions will be flipped because the end position should be greater
// than the starting position for the stream, and we'll add one to the end key
// so that it's included in the results.
var endBuffer = new Buffer(4);
endBuffer.writeUInt32BE(options.end);

var endKey = [AddressService.PREFIXES.SPENTS, hashHex, options.start + 1].join('-');
var startKey = [AddressService.PREFIXES.SPENTS, hashHex, options.end].join('-');
var startBuffer = new Buffer(4);
startBuffer.writeUInt32BE(options.start + 1);

stream = this.node.services.db.store.createReadStream({
start: startKey,
end: endKey
gte: Buffer.concat([
AddressService.PREFIXES.SPENTS,
hashBuffer,
AddressService.SPACER_MIN,
endBuffer
]),
lte: Buffer.concat([
AddressService.PREFIXES.SPENTS,
hashBuffer,
AddressService.SPACER_MIN,
startBuffer
]),
valueEncoding: 'binary',
keyEncoding: 'binary'
});
} else {
var allKey = [AddressService.PREFIXES.SPENTS, hashHex].join('-');
var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer]);
stream = this.node.services.db.store.createReadStream({
start: allKey,
end: allKey + '~'
gte: Buffer.concat([allKey, AddressService.SPACER_MIN]),
lte: Buffer.concat([allKey, AddressService.SPACER_MAX]),
valueEncoding: 'binary',
keyEncoding: 'binary'
});
}

stream.on('data', function(data) {

var key = data.key.split('-');
var value = data.value.split(':');

var blockHeight = Number(key[2]);
var key = self._decodeInputKey(data.key);
var value = self._decodeInputValue(data.value);

var output = {
var input = {
address: addressStr,
txid: value[0],
inputIndex: Number(value[1]),
height: blockHeight,
confirmations: self.node.services.db.tip.__height - blockHeight + 1
txid: value.txid.toString('hex'),
inputIndex: value.inputIndex,
height: key.height,
confirmations: self.node.services.db.tip.__height - key.height + 1
};

inputs.push(output);
inputs.push(input);

});

Expand Down Expand Up @@ -502,44 +590,57 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) {
$.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.');
$.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.');

var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex');
var hashBuffer = bitcore.Address(addressStr).hashBuffer;

var outputs = [];
var stream;

if (options.start && options.end) {

// The positions will be flipped because the end position should be greater
// than the starting position for the stream, and we'll add one to the end key
// so that it's included in the results.
var endKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.start + 1].join('-');
var startKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.end].join('-');
var startBuffer = new Buffer(4);
startBuffer.writeUInt32BE(options.start + 1);
var endBuffer = new Buffer(4);
endBuffer.writeUInt32BE(options.end);

stream = this.node.services.db.store.createReadStream({
start: startKey,
end: endKey
gte: Buffer.concat([
AddressService.PREFIXES.OUTPUTS,
hashBuffer,
AddressService.SPACER_MIN,
endBuffer
]),
lte: Buffer.concat([
AddressService.PREFIXES.OUTPUTS,
hashBuffer,
AddressService.SPACER_MIN,
startBuffer
]),
valueEncoding: 'binary',
keyEncoding: 'binary'
});
} else {
var allKey = [AddressService.PREFIXES.OUTPUTS, hashHex].join('-');
var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer]);
stream = this.node.services.db.store.createReadStream({
start: allKey,
end: allKey + '~'
gte: Buffer.concat([allKey, AddressService.SPACER_MIN]),
lte: Buffer.concat([allKey, AddressService.SPACER_MAX]),
valueEncoding: 'binary',
keyEncoding: 'binary'
});
}

stream.on('data', function(data) {

var key = data.key.split('-');
var value = data.value.split(':');
var key = self._decodeOutputKey(data.key);
var value = self._decodeOutputValue(data.value);

var output = {
address: addressStr,
txid: key[3],
outputIndex: Number(key[4]),
height: Number(key[2]),
satoshis: Number(value[0]),
script: value[1],
confirmations: self.node.services.db.tip.__height - Number(key[2]) + 1
txid: key.txid.toString('hex'),
outputIndex: key.outputIndex,
height: key.height,
satoshis: value.satoshis,
script: value.scriptBuffer.toString('hex'),
confirmations: self.node.services.db.tip.__height - key.height + 1
};

outputs.push(output);
Expand Down
Loading

0 comments on commit f88eee5

Please sign in to comment.