Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Release v0.3.0b1

  • Loading branch information...
commit b2340a4257e34173bfa17d094023a62ab53eedad 1 parent 43496fa
Pedro Teixeira authored
2  docs/web/views/api/database.jade
View
@@ -12,7 +12,7 @@ p.note Everything here is experimental, and the API is no exception, so expect t
li <b>meta_compact_interval</b>: default average compact interval for the metadata key map, in miliseconds. Defaults to 1 hour (1000 * 60 * 60).
li <b>replication_master</b>: true if this database is going to act as a replication master. Defaults to false.
li <b>replication_port</b>: replication server port for this database. Defaults to 5293.
- li <b>replication_max_temp_file_size_kb</b>: maximum file size for replication temporary files on master. Defaults to 10000 KBytes.
+ li <b>replication_max_file_size_kb</b>: maximum file size for replication temporary files on master. Defaults to 10000 KBytes.
h3 Example:
pre
code
4 lib/alfred/meta/database.js
View
@@ -11,7 +11,7 @@ var default_meta_options = {
meta_compact_interval: 1000 * 60 * 60, // 1 hour, +- 50%
replication_master: false,
replication_port: 5293,
- replication_max_temp_file_size_kb: 10000
+ replication_max_file_size_kb: 10000
};
var Database = function(db_path, options, callback) {
@@ -151,7 +151,7 @@ Database.prototype._initialize = function(callback) {
var options = {
master: true,
port: self.options.replication_port,
- max_temp_file_size_kb: self.options.replication_max_temp_file_size_kb
+ max_file_size_kb: self.options.replication_max_file_size_kb
};
self.master_replicator = replication.start(self, options, this);
self.master_replicator.on('error', function(err) {
60 lib/alfred/meta/replication/logger.js
View
@@ -1,11 +1,18 @@
-var File = require('../../files/file'),
- fs = require('fs'),
- path = require('path'),
- EventEmitter = require('events').EventEmitter,
- util = require('util');
+var File = require('../../files/file'),
+ fs = require('fs'),
+ path = require('path'),
+ EventEmitter = require('events').EventEmitter,
+ util = require('util'),
+ options_merger = require('../../util/options_merger');
-var Logger = function(database, callback) {
+var default_options = {
+ max_file_size_kb: 10000
+}
+
+var Logger = function(database, options, callback) {
var self = this;
+ this.options = options_merger.merge(default_options, options);
+ this.options.max_file_size = this.options.max_file_size_kb * 1024;
this.database = database;
this.current_log_pos = 0;
this.livelog_files = [];
@@ -23,8 +30,8 @@ var Logger = function(database, callback) {
util.inherits(Logger, EventEmitter);
-module.exports.start = function(database, callback) {
- return new Logger(database, callback);
+module.exports.start = function(database, options, callback) {
+ return new Logger(database, options, callback);
};
Logger.prototype.end = function() {
@@ -158,11 +165,40 @@ Logger.prototype._startLiveLog = function(callback) {
var self = this;
var meta = this.database;
+ var written = 0;
+ var transitioning = false;
+ var transitioningQueue = [];
+
var send = function(what) {
- self.livelog_file.write(JSON.stringify(what), function(err) {
- if (err) { callback(err); return; }
- self.emit('data', what);
- });
+
+ var reallyWrite = function(what) {
+ self.livelog_file.write(JSON.stringify(what), function(err, pos, length) {
+ if (err) { callback(err); return; }
+ written += length;
+ self.current_log_pos += length;
+ self.emit('data', what);
+ });
+ };
+
+ if (written >= self.options.max_file_size) {
+ transitioningQueue.push(what);
+ if (!transitioning) {
+ transitioning = true;
+ self._openLiveLog(function(err) {
+ var record;
+ if (err) { callback(err); return; }
+ written = 0;
+ transitioning = false;
+ while (transitioningQueue.length > 0) {
+ record = transitioningQueue.splice(0, 1)[0];
+ reallyWrite(record);
+ }
+ });
+ }
+ } else {
+ reallyWrite(what);
+ }
+
};
meta.on('key_map_attached', function(key_map_name) {
4 lib/alfred/meta/replication/master.js
View
@@ -20,7 +20,7 @@ var options_merger = require('../../util/options_merger');
var default_options = {
port: 5293,
- max_temp_file_size_kb: 10000
+ max_file_size_kb: 10000
}
var Master = function(database, options, callback) {
@@ -111,7 +111,7 @@ Master.prototype.loggerStarted = function(callback) {
if (this.logger) {
callback(null);
} else {
- ReplicationLogger.start(self.database, function(err, logger) {
+ ReplicationLogger.start(self.database, {max_file_size_kb: self.options.max_file_size_kb}, function(err, logger) {
if (err) { callback(err); return; }
self.logger = logger;
callback(null);
2  package.json
View
@@ -1,6 +1,6 @@
{ "name" : "alfred"
, "description" : "In-process key-value store"
-, "version" : "0.1.0beta8"
+, "version" : "0.3.0b1"
, "homepage" : "http://pgte.github.com/alfred"
, "author" : "Pedro Teixeira <pedro.teixeira@gmail.com> (http://metaduck.com)"
, "contributors" :
26 test/replication/test_master_temp_roll.js
View
@@ -94,7 +94,7 @@ module.exports.run = function(next) {
var db_options = {
replication_master: true,
- replication_max_temp_file_size_kb: 100
+ replication_max_file_size_kb: 10
};
alfred.open(DB_PATH, db_options, function(err, db) {
@@ -106,15 +106,18 @@ module.exports.run = function(next) {
var put_count = 0;
setTimeout(function() {
- for (var i = 0; i < 10000; i++) {
- (function (i) {
- var obj = random_generator.createRandomObject();
- obj.__index = i;
- db.store.put(random_generator.createRandomString(), obj, function(err) {
- if (err) { next(err); return; }
- });
- }) (i);
- }
+ var i = 0;
+ var interval = setInterval(function() {
+ var obj = random_generator.createRandomObject();
+ obj.__index = i;
+ db.store.put(random_generator.createRandomString(), obj, function(err) {
+ if (err) { next(err); return; }
+ });
+ i ++;
+ if (i == 1000) {
+ clearInterval(interval);
+ }
+ }, 10);
}, 3000);
})
@@ -152,12 +155,11 @@ module.exports.run = function(next) {
next(new Error('Error from master: ' + obj.error));
return;
}
-
if (obj.m != 'meta') {
assert.equal(obj.m, 'store');
assert.equal(result_count, obj.v.__index);
result_count ++;
- if (result_count == 10000) {
+ if (result_count == 1000) {
clearTimeout(timeout);
setTimeout(function() {
next();
Please sign in to comment.
Something went wrong with that request. Please try again.