Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #252 from smfreegard/stream_changes

Change data_lines into a readable Stream
  • Loading branch information...
commit 92dd16d078a2f40005951fb68cba3e1c7824b5c2 2 parents ed0218d + 359b110
@baudehlo authored
View
74 chunkemitter.js
@@ -0,0 +1,74 @@
+var util = require('util');
+var EventEmitter = require('events').EventEmitter;
+
+function ChunkEmitter(buffer_size) {
+ EventEmitter.call(this);
+ this.buffer_size = parseInt(buffer_size) || (64 * 1024);
+ this.buf = null;
+ this.pos = 0;
+ this.bufs = [];
+ this.bufs_size = 0;
+}
+
+util.inherits(ChunkEmitter, EventEmitter);
+
+ChunkEmitter.prototype.fill = function (input) {
+ if (typeof input === 'string') {
+ input = new Buffer(input);
+ }
+
+ // Optimization: don't allocate a new buffer until
+ // the input we've had so far is bigger than our
+ // buffer size.
+ if (!this.buf) {
+ // We haven't allocated a buffer yet
+ this.bufs.push(input);
+ this.bufs_size += input.length;
+ if ((input.length + this.bufs_size) > this.buffer_size) {
+ this.buf = new Buffer(this.buffer_size);
+ var in_new = Buffer.concat(this.bufs, this.bufs_size);
+ input = in_new;
+ // Reset
+ this.bufs = [];
+ this.bufs_size = 0;
+ }
+ else {
+ return;
+ }
+ }
+
+ while (input.length > 0) {
+ var remaining = this.buffer_size - this.pos;
+ if (remaining === 0) {
+ this.emit('data', this.buf); //.slice(0));
+ this.buf = new Buffer(this.buffer_size);
+ this.pos = 0;
+ remaining = this.buffer_size;
+ }
+ var to_write = ((remaining > input.length) ? input.length : remaining);
+ input.copy(this.buf, this.pos, 0, to_write);
+ this.pos += to_write;
+ input = input.slice(to_write);
+ }
+}
+
+ChunkEmitter.prototype.end = function (cb) {
+ var emitted = false;
+ if (this.bufs_size > 0) {
+ this.emit('data', Buffer.concat(this.bufs, this.bufs_size));
+ emitted = true;
+ }
+ else if (this.pos > 0) {
+ this.emit('data', this.buf.slice(0, this.pos));
+ emitted = true;
+ }
+ // Reset
+ this.buf = null;
+ this.pos = 0;
+ this.bufs = [];
+ this.bufs_size = 0;
+ if (cb && typeof cb === 'function') cb();
+ return emitted;
+}
+
+module.exports = ChunkEmitter;
View
11 config/smtp.ini
@@ -19,6 +19,15 @@ port=2525
;nodes=cpus
; Daemonize
-;daemonize=0
+;daemonize=true
;daemon_log_file=/var/log/haraka.log
;daemon_pid_file=/var/run/haraka.pid
+
+; Spooling
+; Save memory by spooling large messages to disk
+;spool_dir=/var/spool/haraka
+; Specify -1 to never spool to disk
+; Specify 0 to always spool to disk
+; Otherwise specify a size in bytes, once reached the
+; message will be spooled to disk to save memory.
+;spool_after=
View
98 connection.js
@@ -13,6 +13,7 @@ var Address = require('./address').Address;
var uuid = require('./utils').uuid;
var outbound = require('./outbound');
var date_to_str = require('./utils').date_to_str;
+var indexOfLF = require('./utils').indexOfLF;
var ipaddr = require('ipaddr.js');
var version = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'))).version;
@@ -121,7 +122,7 @@ function setupClient(self) {
function Connection(client, server) {
this.client = client;
this.server = server;
- this.current_data = '';
+ this.current_data = null;
this.current_line = null;
this.state = STATE_PAUSE;
this.uuid = uuid();
@@ -165,16 +166,25 @@ exports.createConnection = function(client, server) {
Connection.prototype.process_line = function (line) {
var self = this;
- if (this.state !== STATE_DATA) {
- this.logprotocol("C: " + line + ' state=' + this.state);
+ if (this.state === STATE_DATA) {
+ if (logger.would_log(logger.LOGDATA)) {
+ this.logdata("C: " + line);
+ }
+ this.accumulate_data(line);
+ return;
+ }
+ else {
+ this.current_line = line.toString('binary').replace(/\r?\n/, '');
+ if (logger.would_log(logger.LOGPROTOCOL)) {
+ this.logprotocol("C: " + this.current_line + ' state=' + this.state);
+ }
// Check for non-ASCII characters
- if (/[^\x00-\x7F]/.test(line)) {
- return this.respond(501, 'Syntax error');
+ if (/[^\x00-\x7F]/.test(this.current_line)) {
+ return this.respond(501, 'Syntax error (8-bit characters not allowed)');
}
}
if (this.state === STATE_CMD) {
this.state = STATE_PAUSE_SMTP;
- this.current_line = line.replace(/\r?\n/, '');
var matches = /^([^ ]*)( +(.*))?$/.exec(this.current_line);
if (!matches) {
return plugins.run_hooks('unrecognized_command', this, this.current_line);
@@ -208,16 +218,15 @@ Connection.prototype.process_line = function (line) {
}
else if (this.state === STATE_LOOP) {
// Allow QUIT
- if (line.replace(/\r?\n/, '').toUpperCase() === 'QUIT') {
+ if (this.current_line.toUpperCase() === 'QUIT') {
this.cmd_quit();
}
else {
this.respond(this.loop_code, this.loop_msg);
}
}
- else if (this.state === STATE_DATA) {
- this.logdata("C: " + line);
- this.accumulate_data(line);
+ else {
+ throw new Error('unknown state ' + this.state);
}
};
@@ -226,8 +235,19 @@ Connection.prototype.process_data = function (data) {
this.logwarn("data after disconnect from " + this.remote_ip);
return;
}
-
- this.current_data += data.toString('binary'); // TODO: change all this to use Buffers
+
+ if (!this.current_data || !this.current_data.length) {
+ this.current_data = data;
+ }
+ else {
+ // Data left over in buffer
+ var buf = Buffer.concat(
+ [ this.current_data, data ],
+ (this.current_data.length + data.length)
+ );
+ this.current_data = buf;
+ }
+
this._process_data();
};
@@ -237,9 +257,10 @@ Connection.prototype._process_data = function() {
// Otherwise if multiple commands are pipelined and then the
// connection is dropped; we'll end up in the function forever.
if (this.disconnected) return;
- var results;
- while (results = line_regexp.exec(this.current_data)) {
- var this_line = results[1];
+
+ var offset;
+ while (this.current_data && ((offset = indexOfLF(this.current_data)) !== -1)) {
+ var this_line = this.current_data.slice(0, offset+1);
// Hack: bypass this code to allow HAProxy's PROXY extension
if (this.state === STATE_PAUSE && this.proxy && /^PROXY /.test(this_line)) {
if (this.proxy_timer) clearTimeout(this.proxy_timer);
@@ -250,6 +271,7 @@ Connection.prototype._process_data = function() {
// Detect early_talker but allow PIPELINING extension (ESMTP)
else if ((this.state === STATE_PAUSE || this.state === STATE_PAUSE_SMTP) && !this.esmtp) {
if (!this.early_talker) {
+ this_line = this_line.toString().replace(/\r?\n/,'');
this.logdebug('[early_talker] state=' + this.state + ' esmtp=' + this.esmtp + ' line="' + this_line + '"');
}
this.early_talker = 1;
@@ -260,7 +282,7 @@ Connection.prototype._process_data = function() {
}
else if ((this.state === STATE_PAUSE || this.state === STATE_PAUSE_SMTP) && this.esmtp) {
var valid = true;
- var cmd = this_line.slice(0,4).toUpperCase();
+ var cmd = this_line.toString('ascii').slice(0,4).toUpperCase();
switch (cmd) {
case 'RSET':
case 'MAIL':
@@ -419,11 +441,22 @@ Connection.prototype.tran_uuid = function () {
}
Connection.prototype.reset_transaction = function() {
+ if (this.transaction) {
+ this.transaction.message_stream.destroy();
+ }
delete this.transaction;
};
Connection.prototype.init_transaction = function() {
this.transaction = trans.createTransaction(this.tran_uuid());
+ // Catch any errors from the message_stream
+ var self = this;
+ this.transaction.message_stream.on('error', function (err) {
+ self.logcrit('message_stream error: ' + err.message);
+ self.respond('421', 'Internal Server Error', function () {
+ self.disconnect();
+ });
+ });
}
Connection.prototype.loop_respond = function (code, msg) {
@@ -616,8 +649,8 @@ Connection.prototype.ehlo_respond = function(retval, msg) {
"8BITMIME",
];
- var databytes = config.get('databytes');
- response.push("SIZE " + databytes || 0);
+ var databytes = parseInt(config.get('databytes')) || 0;
+ response.push("SIZE " + databytes);
this.capabilities = response;
@@ -1118,13 +1151,28 @@ Connection.prototype.data_respond = function(retval, msg) {
Connection.prototype.accumulate_data = function(line) {
var self = this;
- if (line === ".\r\n")
- return this.data_done();
- // Bare LF checks
- if (line === ".\r" || line === ".\n") {
- this.logerror("Client sent bare line-feed - .\\r or .\\n rather than .\\r\\n");
- this.respond(451, "See http://haraka.github.com/barelf.html", function() {
+ this.transaction.data_bytes += line.length;
+
+ // Look for .\r\n
+ if (line.length === 3 &&
+ line[0] === 0x2e &&
+ line[1] === 0x0d &&
+ line[2] === 0x0a)
+ {
+ this.transaction.message_stream.add_line_end(function () {
+ self.data_done();
+ });
+ return;
+ }
+
+ // Look for .\n
+ if (line.length === 2 &&
+ line[0] === 0x2e &&
+ line[1] === 0x0a)
+ {
+ this.logerror('Client sent bare line-feed - .\\n rather than .\\r\\n');
+ this.respond(451, "Bare line-feed; see http://haraka.github.com/barelf.html", function() {
self.reset_transaction();
});
return;
@@ -1135,7 +1183,7 @@ Connection.prototype.accumulate_data = function(line) {
return;
}
- this.transaction.add_data(line.replace(/^\.\./, '.').replace(/\r\n$/, "\n"));
+ this.transaction.add_data(line);
};
Connection.prototype.data_done = function() {
View
3  docs/CoreConfig.md
@@ -34,6 +34,9 @@ different levels available.
* daemonize - enable this to cause Haraka to fork into the background on start-up (default: 0)
* daemon_log_file - (default: /var/log/haraka.log) where to redirect stdout/stderr when daemonized
* daemon_pid_file - (default: /var/run/haraka.pid) where to write a PID file to
+ * spool_dir - (default: none) directory to create temporary spool files in
+ * spool_after - (default: -1) if message exceeds this size in bytes, then spool the message to disk
+ specify -1 to disable spooling completely or 0 to force all messages to be spooled to disk.
[1]: http://learnboost.github.com/cluster/ or node version >= 0.8
View
31 docs/Transaction.md
@@ -19,9 +19,30 @@ The value of the MAIL FROM command as an `Address` object.
An Array of `Address` objects of recipients from the RCPT TO command.
-* transaction.data\_lines
+* transaction.message_stream
-An Array of the lines of the email after DATA.
+A node.js Readable Stream object for the message.
+
+You use it like this:
+
+ transaction.message_stream.pipe(WritableStream, options)
+
+Where WritableStream is a node.js Writable Stream object such as a
+net.socket, fs.writableStream, process.stdout/stderr or custom stream.
+
+The options argument should be an object that overrides the following
+properties:
+
+ * line_endings (default: "\r\n")
+ * dot_stuffing (default: false)
+ * ending_dot (default: false)
+ * end (default: true)
+ * buffer_size (default: 65535)
+ * clamd_style (default: false)
+
+e.g.
+
+ transaction.message_stream.pipe(socket, { dot_stuffing: true, ending_dot: true });
* transaction.data\_bytes
@@ -32,6 +53,12 @@ The number of bytes in the email after DATA.
Adds a line of data to the email. Note this is RAW email - it isn't useful
for adding banners to the email.
+* transaction.add_line_end(cb)
+
+Notifies the message_stream that all the data has been received.
+Supply an optional callback function that will be run once any inflight data
+is finished being written.
+
* transaction.notes
A safe place to store transaction specific values.
View
391 messagestream.js
@@ -0,0 +1,391 @@
+// MessageStream class
+
+var fs = require('fs');
+var util = require('util');
+var Stream = require('stream').Stream;
+var ChunkEmitter = require('./chunkemitter');
+var indexOfLF = require('./utils').indexOfLF;
+
+var STATE_HEADERS = 1;
+var STATE_BODY = 2;
+
+function MessageStream (config, id, headers) {
+ if (!id) throw new Error('id required');
+ Stream.call(this);
+ this.uuid = id;
+ this.write_ce = null;
+ this.read_ce = null;
+ this.bytes_read = 0;
+ this.state = STATE_HEADERS;
+ this.idx = {};
+ this.end_called = false;
+ this.end_callback = null;
+ this.buffered = 0;
+ this._queue = [];
+ this.max_data_inflight = 0;
+ this.buffer_max = (!isNaN(config.main.spool_after) ?
+ Number(config.main.spool_after) : -1);
+ this.spooling = false;
+ this.fd = null;
+ this.open_pending = false;
+ this.spool_dir = config.main.spool_dir || '/tmp';
+ this.filename = this.spool_dir + '/' + id + '.eml';
+ this.write_pending = false;
+
+ this.readable = true;
+ this.paused = false;
+ this.headers = headers || [];
+ this.headers_done = false;
+ this.headers_found_eoh = false;
+ this.line_endings = "\r\n";
+ this.data_buf = null;
+ this.data_pos = 0;
+ this.dot_stuffing = false;
+ this.ending_dot = false;
+ this.buffer_size = (1024 * 64);
+ this.start = 0;
+ this.write_complete = false;
+}
+
+util.inherits(MessageStream, Stream);
+
+MessageStream.prototype.add_line = function (line) {
+ var self = this;
+
+ if (typeof line === 'string') {
+ line = new Buffer(line);
+ }
+
+ // create a ChunkEmitter
+ if (!this.write_ce) {
+ this.write_ce = new ChunkEmitter();
+ this.write_ce.on('data', function (chunk) {
+ self._write(chunk);
+ });
+ }
+
+ this.bytes_read += line.length;
+
+ // Build up an index of 'interesting' data on the fly
+ if (this.state === STATE_HEADERS) {
+ // Look for end of headers line
+ if (line.length === 2 && line[0] === 0x0d && line[1] === 0x0a) {
+ this.idx['headers'] = { start: 0, end: this.bytes_read-line.length };
+ this.state = STATE_BODY;
+ this.idx['body'] = { start: this.bytes_read };
+ }
+ }
+
+ if (this.state === STATE_BODY) {
+ // Look for MIME boundaries
+ if (line.length > 4 && line[0] === 0x2d && line[1] == 0x2d) {
+ var boundary = line.slice(2).toString().replace(/\s*$/,'');
+ if (/--\s*$/.test(line)) {
+ // End of boundary?
+ boundary = boundary.slice(0, -2);
+ if (this.idx[boundary]) {
+ this.idx[boundary]['end'] = this.bytes_read;
+ }
+ }
+ else {
+ // Start of boundary?
+ if (!this.idx[boundary]) {
+ this.idx[boundary] = { start: this.bytes_read-line.length };
+ }
+ }
+ }
+ }
+
+ this.write_ce.fill(line);
+}
+
+MessageStream.prototype.add_line_end = function (cb) {
+ // Record body end position
+ if (this.idx['body']) {
+ this.idx['body']['end'] = this.bytes_read;
+ }
+ this.end_called = true;
+ if (cb && typeof cb === 'function') {
+ this.end_callback = cb;
+ }
+ // Call _write() only if no new data was emitted
+ // This might happen if the message size matches
+ // the size of the chunk buffer.
+ if (!this.write_ce.end()) {
+ this._write();
+ }
+}
+
+MessageStream.prototype._write = function (data) {
+ var self = this;
+ if (data) {
+ this.buffered += data.length;
+ this._queue.push(data);
+ }
+ // Stats
+ if (this.buffered > this.max_data_inflight) {
+ this.max_data_inflight = this.buffered;
+ }
+ // Abort if we have pending disk operations
+ if (this.open_pending || this.write_pending) return false;
+ // Do we need to spool to disk?
+ if (this.buffer_max !== -1 && this.buffered > this.buffer_max) {
+ this.spooling = true;
+ }
+ // Have we completely finished writing all data?
+ if (this.end_called && (!this.spooling || (this.spooling && !this._queue.length))) {
+ this.write_complete = true;
+ // Do we have any waiting readers?
+ if (this.listeners('data').length) {
+ process.nextTick(function () {
+ if (self.readable && !self.paused)
+ self._read();
+ });
+ }
+ if (this.end_callback) {
+ this.end_callback();
+ }
+ return true;
+ }
+ if (this.buffer_max === -1 || (this.buffered < this.buffer_max && !this.spooling)) {
+ return true;
+ }
+ else {
+ // We're spooling to disk
+ if (!this._queue.length) {
+ return false;
+ }
+ }
+
+ // Open file descriptor if needed
+ if (!this.fd && !this.open_pending) {
+ this.open_pending = true;
+ fs.open(this.filename, 'wx+', null, function (err, fd) {
+ if (err) return self.emit('error', err);
+ self.fd = fd;
+ self.open_pending = false;
+ process.nextTick(function () {
+ self._write();
+ });
+ });
+ }
+
+ if (!this.fd) return false;
+ var to_send = this._queue.shift();
+ this.buffered -= to_send.length;
+ this.write_pending = true;
+ fs.write(this.fd, to_send, 0, to_send.length, null, function (err, written, buffer) {
+ if (err) return self.emit('error', err);
+ self.write_pending = false;
+ process.nextTick(function () {
+ self._write();
+ });
+ });
+ return true;
+}
+
+/*
+** READABLE STREAM
+*/
+
+MessageStream.prototype._read = function () {
+ var self = this;
+ if (!this.end_called) {
+ throw new Error('end not called!');
+ }
+
+ if (!this.readable || this.paused) {
+ return;
+ }
+
+ // Buffer and send headers first.
+ //
+ // Headers are always stored in an array of strings
+ // as they are heavily read and modified throughout
+ // the reception of a message.
+ //
+ // Typically headers will be < 32Kb (Sendmail limit)
+ // so we do all of them in one operation before we
+ // loop around again (and check for pause).
+ if (this.headers.length && !this.headers_done) {
+ this.headers_done = true;
+ for (var i=0; i<this.headers.length; i++) {
+ this.read_ce.fill(this.headers[i].replace(/\r?\n/g,this.line_endings));
+ }
+ // Add end of headers marker
+ this.read_ce.fill(this.line_endings);
+ // Loop
+ process.nextTick(function () {
+ if (self.readable && !self.paused)
+ self._read();
+ });
+ }
+ else {
+ // Read the message body by line
+ // If we have queued entries, then we didn't
+ // create a queue file, so we read from memory.
+ if (this._queue.length > 0) {
+ // TODO: implement start/end offsets
+ for (var i=0; i<this._queue.length; i++) {
+ this.process_buf(this._queue[i].slice(0));
+ }
+ this._read_finish();
+ }
+ else {
+ // Read the message from the queue file
+ fs.read(this.fd, this.data_buf, 0, this.buffer_size, this.data_pos, function (err, bytesRead, buf) {
+ if (err) throw err;
+ if (self.paused || !self.readable) return;
+ self.data_pos = bytesRead;
+ // Have we finished reading?
+ var complete = false;
+ if (bytesRead < buf.length) {
+ buf = buf.slice(0, bytesRead);
+ complete = true;
+ }
+ self.process_buf(buf);
+ if (complete) {
+ self._read_finish();
+ }
+ else {
+ // Loop again
+ process.nextTick(function () {
+ if (self.readable && !self.paused)
+ self._read();
+ });
+ }
+ });
+ }
+ }
+}
+
+MessageStream.prototype.process_buf = function (buf) {
+ var offset = 0;
+ while ((offset = indexOfLF(buf)) !== -1) {
+ var line = buf.slice(0, offset+1);
+ buf = buf.slice(line.length);
+ // Don't output headers if they where sent already
+ if (this.headers_done && !this.headers_found_eoh) {
+ // Allow \r\n or \n here...
+ if ((line.length === 2 && line[0] === 0x0d && line[1] === 0x0a) ||
+ (line.length === 1 && line[0] === 0x0a))
+ {
+ this.headers_found_eoh = true;
+ }
+ continue;
+ }
+ // Remove dot-stuffing if required
+ if (!this.dot_stuffing && line.length >= 4 &&
+ line[0] === 0x2e && line[1] === 0x2e)
+ {
+ line = line.slice(1);
+ }
+ // We store lines in native CRLF format; so strip CR if requested
+ if (this.line_endings === '\n' && line.length >= 2 &&
+ line[line.length-1] === 0x0a && line[line.length-2] === 0x0d)
+ {
+ line[line.length-2] = 0x0a;
+ line = line.slice(0, line.length-1);
+ }
+ this.read_ce.fill(line);
+ }
+ // Check for data left in the buffer
+ if (buf.length > 0) {
+ this.read_ce.fill(buf);
+ }
+}
+
+MessageStream.prototype._read_finish = function () {
+ var self = this;
+ // End dot required?
+ if (this.ending_dot) {
+ this.read_ce.fill('.' + this.line_endings);
+ }
+ // Tell the chunk emitter to send whatever is left
+ // We don't close the fd here so we can re-use it later.
+ this.read_ce.end(function () {
+ if (self.clamd_style) {
+ // Add 0 length to notify end
+ var buf = new Buffer(4);
+ buf.writeUInt32BE(0, 0);
+ self.emit('data', buf);
+ }
+ self.emit('end');
+ });
+}
+
+MessageStream.prototype.pipe = function (destination, options) {
+ var self = this;
+ Stream.prototype.pipe.call(this, destination, options);
+ // Options
+ this.line_endings = ((options && options.line_endings) ? options.line_endings : "\r\n");
+ this.dot_stuffing = ((options && options.dot_stuffing) ? options.dot_stuffing : false);
+ this.ending_dot = ((options && options.ending_dot) ? options.ending_dot : false);
+ this.clamd_style = ((options && options.clamd_style) ? true : false);
+ this.buffer_size = ((options && options.buffer_size) ? options.buffer_size : 1024 * 64);
+ this.start = ((options && parseInt(options.start)) ? parseInt(options.start) : 0);
+ // Reset
+ this.readable = true;
+ this.paused = false;
+ this.headers_done = false;
+ this.headers_found_eoh = false;
+ this.data_buf = new Buffer(this.buffer_size);
+ this.data_pos = 0;
+ this.read_ce = new ChunkEmitter(this.buffer_size);
+ this.read_ce.on('data', function (chunk) {
+ if (self.clamd_style) {
+ // Prefix data length to the beginning of line
+ var buf = new Buffer(chunk.length+4);
+ buf.writeUInt32BE(chunk.length, 0);
+ chunk.copy(buf, 4);
+ self.emit('data', buf);
+ }
+ else {
+ self.emit('data', chunk);
+ }
+ });
+ // Stream won't be readable until we've finished writing and add_line_end() has been called.
+ // As we've registered for events above, the _write() function can now detect that we
+ // are waiting for the data and will call _read() automatically when it is finished.
+ if (!this.write_complete) return;
+ // Create this.fd only if it doesn't already exist
+ // This is so we can re-use the already open descriptor
+ if (!this.fd && !(this._queue.length > 0)) {
+ fs.open(this.filename, 'r', null, function (err, fd) {
+ if (err) throw err;
+ self.fd = fd;
+ self._read();
+ });
+ }
+ else {
+ self._read();
+ }
+}
+
+MessageStream.prototype.pause = function () {
+ this.paused = true;
+}
+
+MessageStream.prototype.resume = function () {
+ this.paused = false;
+ this._read();
+}
+
+MessageStream.prototype.destroy = function () {
+ var self = this;
+ try {
+ if (this.fd) {
+ fs.close(this.fd, function (err) {
+ fs.unlink(self.filename);
+ });
+ }
+ else {
+ fs.unlink(this.filename);
+ }
+ }
+ catch (err) {
+ // Ignore any errors
+ }
+}
+
+module.exports = MessageStream;
View
81 outbound.js
@@ -84,15 +84,19 @@ exports.send_email = function () {
return this.send_trans_email(arguments[0], arguments[1]);
}
+ var self = this;
+
var from = arguments[0],
to = arguments[1],
- contents = arguments[2],
- next = arguments[3];
-
+ contents = arguments[2];
+ var next = arguments[3];
+
this.loginfo("Sending email via params");
var transaction = trans.createTransaction();
+ this.loginfo("Created transaction: " + transaction.uuid);
+
// set MAIL FROM address, and parse if it's not an Address object
if (from instanceof Address) {
transaction.mail_from = from;
@@ -131,19 +135,20 @@ exports.send_email = function () {
transaction.rcpt_to = to;
+
// Set data_lines to lines in contents
var match;
var re = /^([^\n]*\n?)/;
while (match = re.exec(contents)) {
var line = match[1];
- line = line.replace(/\n?$/, '\n'); // make sure it ends in \n
+ line = line.replace(/\n?$/, '\r\n'); // make sure it ends in \r\n
transaction.add_data(line);
contents = contents.substr(match[1].length);
if (contents.length === 0) {
break;
}
}
-
+ transaction.message_stream.add_line_end();
this.send_trans_email(transaction, next);
}
@@ -160,7 +165,7 @@ exports.send_trans_email = function (transaction, next) {
transaction.add_header('Date', date_to_str(new Date()));
}
- transaction.add_header('Received', 'via haraka outbound.js at ' + date_to_str(new Date()));
+ transaction.add_leading_header('Received', 'via haraka outbound.js at ' + date_to_str(new Date()));
// First get each domain
var recips = {};
@@ -222,56 +227,38 @@ exports.process_domain = function (todo, hmails, cb) {
var fname = _fname();
var tmp_path = path.join(queue_dir, '.' + fname);
var ws = fs.createWriteStream(tmp_path, { flags: WRITE_EXCL });
- var data_pos = 0;
- var write_more = function () {
- if (data_pos === todo.data_lines.length) {
- ws.on('close', function () {
- var dest_path = path.join(queue_dir, fname);
- fs.rename(tmp_path, dest_path, function (err) {
- if (err) {
- plugin.logerror("Unable to rename tmp file!: " + err);
- cb(tmp_path, DENY, "Queue error");
- }
- else {
- hmails.push(new HMailItem (fname, dest_path, todo.notes));
- cb(tmp_path, OK, "Queued!");
- }
- });
- });
- ws.destroySoon();
- return;
- }
-
- // write, but fixup "." at the beginning of the line to be ".."
- // and fixup \n to be \r\n
- var buf = new Buffer(todo.data_lines[data_pos++].replace(/^\./m, '..').replace(/\r?\n/g, "\r\n"), 'binary');
- if (ws.write(buf)) {
- write_more();
- }
- };
-
+ ws.on('close', function () {
+ var dest_path = path.join(queue_dir, fname);
+ fs.rename(tmp_path, dest_path, function (err) {
+ if (err) {
+ plugin.logerror("Unable to rename tmp file!: " + err);
+ cb(tmp_path, DENY, "Queue error");
+ }
+ else {
+ hmails.push(new HMailItem (fname, dest_path, todo.notes));
+ cb(tmp_path, OK, "Queued!");
+ }
+ });
+ });
ws.on('error', function (err) {
plugin.logerror("Unable to write queue file (" + fname + "): " + err);
ws.destroy();
cb(tmp_path, DENY, "Queueing failed");
});
-
- ws.on('drain', write_more);
-
- plugin.build_todo(todo, ws, write_more);
+ plugin.build_todo(todo, ws);
}
-exports.build_todo = function (todo, ws, write_more) {
+exports.build_todo = function (todo, ws) {
// Replacer function to exclude items from the queue file header
function exclude_from_json(key, value) {
switch (key) {
- case 'data_lines':
+ case 'message_stream':
return undefined;
default:
return value;
}
}
- var todo_str = JSON.stringify(todo, exclude_from_json);
+ var todo_str = new Buffer(JSON.stringify(todo, exclude_from_json), 'binary');
// since JS has no pack() we have to manually write the bytes of a long
var todo_length = new Buffer(4);
@@ -281,12 +268,10 @@ exports.build_todo = function (todo, ws, write_more) {
todo_length[1] = (todo_str.length >> 16) & 0xff;
todo_length[0] = (todo_str.length >> 24) & 0xff;
- ws.write(todo_length);
- if (ws.write(todo_str)) {
- // if write() above returned false, the 'drain' event will be called
- // later anyway to call write_more_data()
- write_more();
- }
+ var buf = Buffer.concat([todo_length, todo_str], todo_str.length + 4);
+
+ ws.write(buf);
+ todo.message_stream.pipe(ws, { line_endings: '\r\n', dot_stuffing: true, ending_dot: false });
}
exports.split_to_new_recipients = function (hmail, recipients) {
@@ -417,7 +402,7 @@ function TODOItem (domain, recipients, transaction) {
this.domain = domain;
this.rcpt_to = recipients;
this.mail_from = transaction.mail_from;
- this.data_lines = transaction.data_lines;
+ this.message_stream = transaction.message_stream;
this.notes = transaction.notes;
this.uuid = transaction.uuid;
return this;
View
38 plugins/clamd.js
@@ -64,42 +64,6 @@ exports.hook_data_post = function (next, connection) {
socket.setTimeout(config.main.timeout * 1000);
- var pack_len = function(length) {
- var len = new Buffer(4);
- len[3] = length & 0xFF;
- len[2] = (length >> 8) & 0xFF;
- len[1] = (length >> 16) & 0xFF;
- len[0] = (length >> 24) & 0xFF;
- return len;
- }
-
- var data_marker = 0;
- var in_data = false;
-
- var send_data = function () {
- in_data = true;
- var wrote_all = true;
- while (wrote_all && (data_marker < transaction.data_lines.length)) {
- var data_line = transaction.data_lines[data_marker];
- var len = Buffer.byteLength(data_line);
- var buf = new Buffer(parseInt(len + 4));
- pack_len(len).copy(buf);
- buf.write(data_line, 4);
- data_marker++;
- wrote_all = socket.write(buf);
- }
- if (wrote_all) {
- // We're at the end of the data_lines - send a zero length line
- in_data = false; // We don't need to be called by socket.on('drain' ...
- socket.end(pack_len(0));
- }
- };
-
- socket.on('drain', function () {
- if (in_data) {
- process.nextTick(function () { send_data() });
- }
- });
socket.on('timeout', function () {
connection.logerror(plugin, "connection timed out");
socket.destroy();
@@ -115,7 +79,7 @@ exports.hook_data_post = function (next, connection) {
addressInfo = hp === null ? '' : ' ' + hp.address + ':' + hp.port;
connection.logdebug(plugin, 'connected to host' + addressInfo);
socket.write("zINSTREAM\0", function () {
- send_data();
+ transaction.message_stream.pipe(socket, { clamd_style: true });
});
});
View
203 plugins/dkim_sign.js
@@ -2,8 +2,135 @@
// Implements DKIM core as per www.dkimcore.org
var crypto = require('crypto');
+var Stream = require('stream').Stream;
+var indexOfLF = require('./utils').indexOfLF;
+var util = require('util');
+
+function DKIMSignStream(selector, domain, private_key, headers_to_sign, header, end_callback) {
+ Stream.call(this);
+ this.selector = selector;
+ this.domain = domain;
+ this.private_key = private_key;
+ this.headers_to_sign = headers_to_sign;
+ this.header = header;
+ this.end_callback = end_callback;
+ this.writable = true;
+ this.found_eoh = false;
+ this.buffer = { ar: [], len: 0 };
+ this.hash = crypto.createHash('SHA256');
+ this.line_buffer = { ar: [], len: 0 };
+ this.signer = crypto.createSign('RSA-SHA256');
+}
+
+util.inherits(DKIMSignStream, Stream);
+
+DKIMSignStream.prototype.write = function (buf) {
+ /*
+ ** BODY (simple canonicalization)
+ */
+
+ // Merge in any partial data from last iteration
+ if (this.buffer.ar.length) {
+ this.buffer.ar.push(buf);
+ this.buffer.len += buf.length;
+ var nb = Buffer.concat(this.buffer.ar, this.buffer.len);
+ buf = nb;
+ this.buffer = { ar: [], len: 0 };
+ }
+ // Process input buffer into lines
+ var offset = 0;
+ while ((offset = indexOfLF(buf)) !== -1) {
+ var line = buf.slice(0, offset+1);
+ if (buf.length > offset) {
+ buf = buf.slice(offset+1);
+ }
+ // Look for CRLF
+ if (line.length === 2 && line[0] === 0x0d && line[1] === 0x0a) {
+ // Look for end of headers marker
+ if (!this.found_eoh) {
+ this.found_eoh = true;
+ }
+ else {
+ // Store any empty lines so that we can discard
+ // any trailing CRLFs at the end of the message
+ this.line_buffer.ar.push(line);
+ this.line_buffer.len += line.length;
+ }
+ }
+ else {
+ if (!this.found_eoh) continue; // Skip headers
+ if (this.line_buffer.ar.length) {
+ // We need to process the buffered CRLFs
+ var lb = Buffer.concat(this.line_buffer.ar, this.line_buffer.len);
+ this.line_buffer = { ar: [], len: 0 };
+ this.hash.update(lb);
+ }
+ this.hash.update(line);
+ }
+ }
+ if (buf.length) {
+ // We have partial data...
+ this.buffer.ar.push(buf);
+ this.buffer.len += buf.length;
+ }
+}
+
+DKIMSignStream.prototype.end = function (buf) {
+ this.writable = false;
+
+ // Add trailing CRLF if we have data left over
+ if (this.buffer.ar.length) {
+ this.buffer.ar.push("\r\n");
+ this.buffer.len += 2;
+ var le = Buffer.concat(this.buffer.ar, this.buffer.len);
+ this.hash.update(le);
+ this.buffer = { ar: [], len: 0 };
+ }
+
+ var bodyhash = this.hash.digest('base64');
+
+ /*
+ ** HEADERS (relaxed canonicaliztion)
+ */
+
+ var headers = [];
+ for (var i=0; i < this.headers_to_sign.length; i++ ) {
+ var head = this.header.get(this.headers_to_sign[i]);
+ if (head) {
+ head = head.replace(/\r?\n/gm, '');
+ head = head.replace(/\s+/gm, ' ');
+ head = head.replace(/\s+$/gm, '');
+ this.signer.update(this.headers_to_sign[i] + ':' + head + "\r\n");
+ headers.push(this.headers_to_sign[i]);
+ }
+ };
+
+ // Create DKIM header
+ var dkim_header = 'v=1;a=rsa-sha256;bh=' + bodyhash +
+ ';c=relaxed/simple;d=' + this.domain +
+ ';h=' + headers.join(':') +
+ ';s=' + this.selector +
+ ';b=';
+ this.signer.update('dkim-signature:' + dkim_header);
+ var signature = this.signer.sign(this.private_key, 'base64');
+ dkim_header += signature;
+
+ if (this.end_callback) this.end_callback(null, dkim_header);
+ this.end_callback = null;
+}
+
+DKIMSignStream.prototype.destroy = function () {
+ this.writable = false;
+ // Stream destroyed before the callback ran
+ if (this.end_callback) {
+ this.end_callback(new Error('Stream destroyed'));
+ }
+}
+
+exports.DKIMSignStream = DKIMSignStream;
exports.hook_queue_outbound = function (next, connection) {
+ var self = this;
var transaction = connection.transaction;
var config = this.config.get('dkim_sign.ini');
var private_key = this.config.get('dkim.private.key','data').join("\n");
@@ -37,69 +164,21 @@ exports.hook_queue_outbound = function (next, connection) {
headers_to_sign.push('from');
}
- /*
- ** BODY (simple canonicalization)
- */
- var data_marker = 0;
- var found_body = false;
- var buffer = "";
- var hash = crypto.createHash('SHA256');
- while (data_marker < transaction.data_lines.length) {
- var line = transaction.data_lines[data_marker];
- line = line.replace(/\r?\n/g, "\r\n");
- // Skip until we find the end-of-headers
- if (!found_body) {
- if (line === "\r\n") {
- found_body = true;
- }
- data_marker++;
- continue;
- }
- if (line === "\r\n") {
- // Buffer any empty lines so we can discard
- // and trailing CRLFs at the end of the message.
- buffer += line;
+ var dkim_sign = new DKIMSignStream(config.main.selector,
+ config.main.domain,
+ private_key,
+ headers_to_sign,
+ transaction.header,
+ function (err, dkim_header)
+ {
+ if (err) {
+ connection.logerror(self, err.message);
}
else {
- if (buffer) {
- hash.update(buffer);
- buffer = "";
- }
- hash.update(line, 'ascii');
+ connection.loginfo(self, dkim_header);
+ transaction.add_header('DKIM-Signature', dkim_header);
}
- data_marker++;
- }
- // Add trailing CRLF if it was missing from the last line
- if (line.slice(-2) !== "\r\n") {
- hash.update("\r\n", 'ascii');
- }
- var bodyhash = hash.digest('base64');
-
- /*
- ** HEADERS (relaxed canonicaliztion)
- */
- var headers = [];
- var signer = crypto.createSign('RSA-SHA256');
- for (var i=0; i < headers_to_sign.length; i++ ) {
- var head = transaction.header.get(headers_to_sign[i]);
- if (head) {
- head = head.replace(/\r?\n/gm, '');
- head = head.replace(/\s+/gm, ' ');
- head = head.replace(/\s+$/gm, '');
- signer.update(headers_to_sign[i] + ':' + head + "\r\n");
- headers.push(headers_to_sign[i]);
- }
- };
- var dkim_header = 'v=1;a=rsa-sha256;bh=' + bodyhash +
- ';c=relaxed/simple;d=' + config.main.domain +
- ';h=' + headers.join(':') +
- ';s=' + config.main.selector +
- ';b=';
- signer.update('dkim-signature:' + dkim_header);
- var signature = signer.sign(private_key, 'base64');
- dkim_header += signature;
- transaction.add_header('DKIM-Signature', dkim_header);
- connection.loginfo(this, 'added DKIM signature');
-
- return next();
+ return next();
+ });
+ transaction.message_stream.pipe(dkim_sign);
}
View
52 plugins/queue/quarantine.js
@@ -7,6 +7,7 @@ var existsSync = require('./utils').existsSync;
exports.register = function () {
this.register_hook('queue','quarantine');
+ this.register_hook('queue_outbound','quarantine');
}
// http://unknownerror.net/2011-05/16260-nodejs-mkdirs-recursion-create-directory.html
@@ -48,12 +49,8 @@ exports.hook_init_master = function (next) {
exports.quarantine = function (next, connection) {
var transaction = connection.transaction;
+transaction.notes.quarantine = true;
if ((connection.notes.quarantine || transaction.notes.quarantine)) {
- var lines = transaction.data_lines;
- // Skip unless we have some data
- if (lines.length === 0) {
- return next();
- }
// Calculate date in YYYYMMDD format
var d = new Date();
var yyyymmdd = d.getFullYear() + zeroPad(d.getMonth()+1, 2)
@@ -91,31 +88,32 @@ exports.quarantine = function (next, connection) {
// final destination.
mkdirs([ base_dir, 'tmp' ].join('/'), parseInt('0770', 8), function () {
mkdirs([ base_dir, dir ].join('/'), parseInt('0770', 8), function () {
- fs.writeFile([ base_dir, 'tmp', transaction.uuid ].join('/'), lines.join(''),
- function(err) {
- if (err) {
- connection.logerror(plugin, 'Error writing quarantine file: ' + err);
- }
- else {
- fs.link([ base_dir, 'tmp', transaction.uuid ].join('/'),
- [ base_dir, dir, transaction.uuid ].join('/'),
- function (err) {
- if (err) {
- connection.logerror(plugin, 'Error writing quarantine file: ' + err);
- }
- else {
- connection.loginfo(plugin, 'Stored copy of message in quarantine: ' +
- [ base_dir, dir, transaction.uuid ].join('/'));
- // Now delete the temporary file
- fs.unlink([ base_dir, 'tmp', transaction.uuid ].join('/'));
- }
- return next();
- }
- );
- }
+ var ws = fs.createWriteStream([ base_dir, 'tmp', transaction.uuid ].join('/'));
+ ws.on('error', function (err) {
+ connection.logerror(plugin, 'Error writing quarantine file: ' + err.message);
+ return next();
+ });
+ ws.on('close', function () {
+ fs.link([ base_dir, 'tmp', transaction.uuid ].join('/'),
+ [ base_dir, dir, transaction.uuid ].join('/'),
+ function (err) {
+ if (err) {
+ connection.logerror(plugin, 'Error writing quarantine file: ' + err);
+ }
+ else {
+ connection.loginfo(plugin, 'Stored copy of message in quarantine: ' +
+ [ base_dir, dir, transaction.uuid ].join('/'));
+ // Now delete the temporary file
+ fs.unlink([ base_dir, 'tmp', transaction.uuid ].join('/'));
+ }
+ return next();
+ }
+ );
});
+ transaction.message_stream.pipe(ws, { line_endings: '\n' });
});
});
+
}
else {
return next();
View
4 plugins/queue/smtp_forward.js
@@ -30,7 +30,7 @@ exports.hook_queue = function (next, connection) {
}
smtp_client.on('data', function () {
- smtp_client.start_data(connection.transaction.data_lines);
+ smtp_client.start_data(connection.transaction.message_stream);
});
smtp_client.on('dot', function () {
@@ -54,3 +54,5 @@ exports.hook_queue = function (next, connection) {
});
});
};
+
+exports.hook_queue_outbound = exports.hook_queue;
View
2  plugins/queue/smtp_proxy.js
@@ -61,7 +61,7 @@ exports.hook_queue = function (next, connection) {
var smtp_client = connection.notes.smtp_client;
if (!smtp_client) return next();
smtp_client.next = next;
- smtp_client.start_data(connection.transaction.data_lines);
+ smtp_client.start_data(connection.transaction.message_stream);
};
exports.hook_rset = function (next, connection) {
View
38 plugins/spamassassin.js
@@ -50,35 +50,6 @@ exports.hook_data_post = function (next, connection) {
connection.transaction.notes.spamd_user ||
'default';
- var data_marker = 0;
- var in_data = false;
- var end_pending = true;
-
- // TODO: buffer writes into 64K chunks
- var send_data = function () {
- in_data = true;
- var wrote_all = true;
- while (wrote_all && (data_marker < connection.transaction.data_lines.length)) {
- var line = connection.transaction.data_lines[data_marker];
- data_marker++;
- // dot-stuffing not necessary for spamd
- wrote_all = socket.write(new Buffer(line, 'binary'));
- if (!wrote_all) return;
- }
- // we get here if wrote_all still true, and we got to end of data_lines
- if (end_pending) {
- end_pending = false;
- socket.end("\r\n");
- }
- };
-
- socket.on('drain', function () {
- connection.logdebug(plugin, 'drain');
- if (end_pending && in_data) {
- process.nextTick(function () { send_data() });
- }
- });
-
socket.on('timeout', function () {
connection.logerror(plugin, "spamd connection timed out");
socket.end();
@@ -100,9 +71,9 @@ exports.hook_data_post = function (next, connection) {
if (connection.relaying) {
headers.push('X-Haraka-Relay: true');
}
- socket.write(headers.join("\r\n"), function () {
- send_data();
- });
+
+ socket.write(headers.join("\r\n"));
+ connection.transaction.message_stream.pipe(socket);
});
var spamd_response = {};
@@ -136,6 +107,9 @@ exports.hook_data_post = function (next, connection) {
});
socket.on('end', function () {
+ // Abort if the connection or transaction are gone
+ if (!connection || (connection && !connection.transaction)) return next();
+
// Now we do stuff with the results...
plugin.fixup_old_headers(config.main.old_headers_action, connection.transaction);
View
15 plugins/test_queue.js
@@ -2,16 +2,9 @@
var fs = require('fs');
exports.hook_queue = function(next, connection) {
- var lines = connection.transaction.data_lines;
- if (lines.length === 0) {
- return next(DENY);
- }
-
- fs.writeFile('/tmp/mail.eml', lines.join(''), function(err) {
- if (err) {
- return next(DENY, "Saving failed");
- }
-
+ var ws = fs.createWriteStream('/tmp/mail.eml');
+ ws.once('end', function () {
return next(OK);
- });
+ }
+ connection.transaction.message_stream.pipe(ws);
};
View
45 smtp_client.js
@@ -104,12 +104,6 @@ function SMTPClient(port, host, connect_timeout) {
self.socket.setTimeout(0);
});
- this.socket.on('drain', function () {
- if (self.command === 'mailbody') {
- process.nextTick(function () { self.continue_data() });
- }
- });
-
var closed = function (msg) {
return function (error) {
if (!error) {
@@ -149,42 +143,9 @@ SMTPClient.prototype.send_command = function (command, data) {
};
SMTPClient.prototype.start_data = function (data) {
- this.command = 'mailbody';
- if (data instanceof Function) {
- this.send_data = data;
- }
- else if (Array.isArray(data)) {
- var data_marker = 0;
- this.send_data = function () {
- while (data_marker < data.length) {
- var line = data[data_marker];
- data_marker++;
- if (!this.send_data_line(line)) {
- return false;
- }
- }
- return true;
- };
- }
- else {
- this.send_data = function () {
- this.socket.write(data);
- return true;
- };
- }
- this.continue_data();
-};
-
-SMTPClient.prototype.continue_data = function () {
- if (!this.send_data()) {
- return;
- }
- this.send_command('dot');
-};
-
-SMTPClient.prototype.send_data_line = function (line) {
- var buf = new Buffer(line.replace(/^\./, '..').replace(/\r?\n/g, '\r\n'), 'binary');
- return this.socket.write(buf);
+ this.response = [];
+ this.command = 'dot';
+ data.pipe(this.socket, { dot_stuffing: true, ending_dot: true, end: false });
};
SMTPClient.prototype.release = function () {
View
28 transaction.js
@@ -6,12 +6,15 @@ var logger = require('./logger');
var Header = require('./mailheader').Header;
var body = require('./mailbody');
var utils = require('./utils');
+var MessageStream = require('./messagestream');
var trans = exports;
function Transaction() {
+ this.uuid = null;
this.mail_from = null;
this.rcpt_to = [];
+ this.header_lines = [];
this.data_lines = [];
this.banner = null;
this.data_bytes = 0;
@@ -19,6 +22,7 @@ function Transaction() {
this.parse_body = false;
this.notes = {};
this.header = new Header();
+ this.message_stream = null;
this.rcpt_count = {
accept: 0,
tempfail: 0,
@@ -31,35 +35,38 @@ exports.Transaction = Transaction;
exports.createTransaction = function(uuid) {
var t = new Transaction();
t.uuid = uuid || utils.uuid();
+ // Initialize MessageStream here to pass in the UUID
+ t.message_stream = new MessageStream(config.get('smtp.ini'), t.uuid, t.header.header_list);
return t;
};
Transaction.prototype.add_data = function(line) {
- this.data_bytes += line.length;
+ this.message_stream.add_line(line);
+ if (typeof line !== 'string') {
+ line = line.toString('binary').replace(/^\.\./, '.').replace(/\r\n$/, '\n');
+ }
// check if this is the end of headers line (note the regexp isn't as strong
// as it should be - it accepts whitespace in a blank line - we've found this
// to be a good heuristic rule though).
if (this.header_pos === 0 && line.match(/^\s*$/)) {
- this.header.parse(this.data_lines);
- this.header_pos = this.data_lines.length;
+ this.header.parse(this.header_lines);
+ this.header_pos = this.header_lines.length;
if (this.parse_body) {
this.body = this.body || new body.Body(this.header, {"banner": this.banner});
}
}
- else if (this.header_pos && this.parse_body) {
- line = this.body.parse_more(line);
+ else if (this.header_pos === 0) {
+ // Build up headers
+ this.header_lines.push(line);
}
- if (line.length) {
- this.data_lines.push(line);
+ else if (this.header_pos && this.parse_body) {
+ this.body.parse_more(line);
}
};
Transaction.prototype.end_data = function() {
if (this.header_pos && this.parse_body) {
var data = this.body.parse_end();
- if (data.length) {
- this.data_lines.push(data);
- }
}
}
@@ -75,7 +82,6 @@ Transaction.prototype.add_leading_header = function(key, value) {
Transaction.prototype.reset_headers = function () {
var header_lines = this.header.lines();
- this.data_lines = header_lines.concat(this.data_lines.slice(this.header_pos));
this.header_pos = header_lines.length;
};
View
7 utils.js
@@ -119,3 +119,10 @@ var versions = process.version.split('.'),
subversion = Number(versions[1]);
exports.existsSync = require((version > 0 || subversion >= 8) ? 'fs' : 'path').existsSync;
+
+exports.indexOfLF = function (buf) {
+ for (var i=0; i<buf.length; i++) {
+ if (buf[i] === 0x0a) return i;
+ }
+ return -1;
+}
Please sign in to comment.
Something went wrong with that request. Please try again.