From ef21d712d31ac49f0b1e90a9099e7ed9577a8b68 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Thu, 28 Nov 2013 17:24:00 +0000 Subject: [PATCH 001/160] Prevent original message buffer from being modified --- messagestream.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/messagestream.js b/messagestream.js index cf8091efb..b903eee5e 100644 --- a/messagestream.js +++ b/messagestream.js @@ -286,6 +286,8 @@ MessageStream.prototype.process_buf = function (buf) { if (this.line_endings === '\n' && line.length >= 2 && line[line.length-1] === 0x0a && line[line.length-2] === 0x0d) { + // We copy the line to a new buffer before modifying the copy + line = new Buffer(line); line[line.length-2] = 0x0a; line = line.slice(0, line.length-1); } From 8e735d39a5f6dc275fef6d4a431878cdf1fbf8d7 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Fri, 29 Nov 2013 11:18:11 +0000 Subject: [PATCH 002/160] Send full path of plugin to vm. Fixes #383 --- plugins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins.js b/plugins.js index b6472148f..e4b15d19d 100644 --- a/plugins.js +++ b/plugins.js @@ -148,7 +148,7 @@ plugins._load_and_compile_plugin = function(name) { }; constants.import(sandbox); try { - vm.runInNewContext(code, sandbox, name); + vm.runInNewContext(code, sandbox, fp[i]); } catch (err) { logger.logcrit("Compiling plugin: " + name + " failed"); From b2fb0e989f5e76838766b4a36572155d9c1b0847 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 3 Dec 2013 14:50:52 -0500 Subject: [PATCH 003/160] Change around so config is loaded when changed --- plugins/relay_force_routing.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/plugins/relay_force_routing.js b/plugins/relay_force_routing.js index db9092dae..b31b1590a 100644 --- a/plugins/relay_force_routing.js +++ b/plugins/relay_force_routing.js @@ -2,30 +2,25 @@ // documentation via: haraka -h plugins/relay_force_routing -exports.register = function() { - this.register_hook('get_mx', 'force_routing'); - this.domain_ini = this.config.get('relay_dest_domains.ini', 'ini'); -}; - -exports.force_routing = function (next, hmail, domain) { - var force_route = lookup_routing(this, this.domain_ini['domains'], domain); +exports.hook_get_mx = function (next, hmail, domain) { + var domain_ini = this.config.get('relay_dest_domains.ini', 'ini'); + var force_route = lookup_routing(domain_ini['domains'], domain); if (force_route != "NOTFOUND" ){ - this.logdebug(this, 'using ' + force_route + ' for ' + domain); + this.logdebug('using ' + force_route + ' for: ' + domain); next(OK, force_route); } else { - this.logdebug(this, 'using normal MX lookup' + ' for ' + domain); + this.logdebug('using normal MX lookup for: ' + domain); next(CONT); } -}; +} /** * @return {string} */ -function lookup_routing (plugin, domains_ini, domain) { +function lookup_routing (domains_ini, domain) { if (domain in domains_ini) { var config = JSON.parse(domains_ini[domain]); - plugin.logdebug(plugin, 'found config for' + domain + ': ' + domains_ini['nexthop']); return config['nexthop']; } return "NOTFOUND"; From 954a36ebae47be91a657a23e970def23bc6fc650 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 3 Dec 2013 15:40:22 -0500 Subject: [PATCH 004/160] Don't re-secure if STARTTLS advertised at 2nd EHLO --- smtp_client.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smtp_client.js b/smtp_client.js index 1c5a8f8a8..ba4576ab1 100644 --- a/smtp_client.js +++ b/smtp_client.js @@ -279,6 +279,9 @@ exports.get_client_plugin = function (plugin, connection, config, callback) { config.main.host, config.main.connect_timeout, config.main.timeout, config.main.max_connections); pool.acquire(function (err, smtp_client) { connection.logdebug(plugin, 'Got smtp_client: ' + smtp_client.uuid); + + var secured = false; + smtp_client.call_next = function (retval, msg) { if (this.next) { var next = this.next; @@ -315,11 +318,12 @@ exports.get_client_plugin = function (plugin, connection, config, callback) { return; } } - if (smtp_client.response[line].match(/^STARTTLS/)) { + if (smtp_client.response[line].match(/^STARTTLS/) && !secured) { tls_key = plugin.config.get('tls_key.pem', 'binary'); tls_cert = plugin.config.get('tls_cert.pem', 'binary'); if (tls_key && tls_cert && enable_tls) { smtp_client.socket.on('secure', function () { + secured = true; smtp_client.emit('greeting', 'EHLO'); }); smtp_client.send_command('STARTTLS'); From f188605f8ed0386c130b8fda18ba3170fde4c019 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 3 Dec 2013 16:10:44 -0500 Subject: [PATCH 005/160] Finally fix RCPT parsing --- rfc1869.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rfc1869.js b/rfc1869.js index b155db50a..df1543a9b 100644 --- a/rfc1869.js +++ b/rfc1869.js @@ -76,7 +76,7 @@ exports.parse = function(type, line, strict) { } } else { - console.log("Looking at " + line); + // console.log("Looking at " + line); if (line.match(/\@.*\s/)) { throw new Error("Syntax error in parameters"); } @@ -84,8 +84,10 @@ exports.parse = function(type, line, strict) { if (line.match(/\s/)) { throw new Error("Syntax error in parameters"); } - else if (line.match(/\@/) && !line.match(/^<.*>$/)) { - line = '<' + line + '>'; + else if (line.match(/\@/)) { + if (!line.match(/^<.*>$/)) { + line = '<' + line + '>'; + } } else if (!line.match(/^(postmaster|abuse)$/i)) { throw new Error("Syntax error in address"); From 257eb8cafb4f3f43639e5182d17cd18dd3c674e4 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 3 Dec 2013 16:10:55 -0500 Subject: [PATCH 006/160] More tests for rcpt parsing --- tests/rfc1869.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/rfc1869.js b/tests/rfc1869.js index e4d927c9f..3c0a1cdde 100644 --- a/tests/rfc1869.js +++ b/tests/rfc1869.js @@ -42,5 +42,13 @@ exports.basic = { 'MAIL FROM: somekey other=foo': function (test) { _check(test, 'MAIL FROM: somekey other=foo', ['', 'somekey', 'other=foo']); - } + }, + 'RCPT TO: state=1': function (test) { + _check(test, 'RCPT TO: state=1', + ['', 'state=1']); + }, + 'RCPT TO: foo=bar': function (test) { + _check(test, 'RCPT TO: foo=bar', + ['', 'foo=bar']); + } }; From 963803bb1308c8b92a94379eab560232340d4e09 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Tue, 17 Dec 2013 22:45:14 +0000 Subject: [PATCH 007/160] Improve outbound logging, pass failed recipients as extra parameters --- docs/Outbound.md | 4 ++++ outbound.js | 35 ++++++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/Outbound.md b/docs/Outbound.md index a8dff56c8..c4424afce 100644 --- a/docs/Outbound.md +++ b/docs/Outbound.md @@ -86,6 +86,10 @@ parameter is the error message received from the remote end. If you do not wish to have a bounce message sent to the originating sender of the email then you can return `OK` from this hook to stop it from sending a bounce message. +The variable hmail.bounce_extra can be accessed from this hook. This is an +Object which contains each recipient as the key and the value is the code +and response received from the upstream server for that recipient. + ### The delivered hook When mails are successfully delivered to the remote end then the `delivered` diff --git a/outbound.js b/outbound.js index c2b0cba92..065d6a30a 100644 --- a/outbound.js +++ b/outbound.js @@ -877,7 +877,11 @@ HMailItem.prototype.try_deliver = function () { }); } -var smtp_regexp = /^([0-9]{3})([ -])(.*)/; +var smtp_regexp = /^(\d{3})([ -])(?:(\d\.\d\.\d)\s)?(.*)/; + +function map_recips(item) { + return Object.keys(item)[0]; +} HMailItem.prototype.try_deliver_host = function (mx) { if (this.hostlist.length === 0) { @@ -999,19 +1003,26 @@ HMailItem.prototype.try_deliver_host = function (mx) { if (matches = smtp_regexp.exec(line)) { var code = matches[1], cont = matches[2], - rest = matches[3]; + extc = matches[3], + rest = matches[4]; response.push(rest); if (cont === ' ') { if (code.match(/^4/)) { if (/^rcpt/.test(command)) { // this recipient was rejected - fail_recips.push(last_recip); + self.lognotice('recipient ' + last_recip + ' deferred: ' + + code + ' ' + ((extc) ? extc + ' ' : '') + response.join(' ')); + (function () { + var o = {}; + o[last_recip] = code + ' ' + ((extc) ? extc + ' ' : '') + response.join(' '); + fail_recips.push(o); + })(); } else { var reason = response.join(' '); socket.send_command('QUIT'); processing_mail = false; - return self.temp_fail("Upstream error: " + code + " " + reason); + return self.temp_fail("Upstream error: " + code + " " + ((extc) ? extc + ' ' : '') + reason); } } else if (code.match(/^5/)) { @@ -1020,7 +1031,13 @@ HMailItem.prototype.try_deliver_host = function (mx) { return socket.send_command('HELO', config.get('me')); } if (/^rcpt/.test(command)) { - bounce_recips.push(last_recip); + self.lognotice('recipient ' + last_recip + ' rejected: ' + + code + ' ' + ((extc) ? extc + ' ' : '') + response.join(' ')); + (function() { + var o = {}; + o[last_recip] = code + ' ' + ((extc) ? extc + ' ' : '') + response.join(' '); + bounce_recips.push(o); + })(); } else { var reason = response.join(' '); @@ -1056,16 +1073,16 @@ HMailItem.prototype.try_deliver_host = function (mx) { if (!recipients.length) { if (fail_recips.length) { self.refcount++; - exports.split_to_new_recipients(self, fail_recips, "Some recipients temporarily failed", function (hmail) { + exports.split_to_new_recipients(self, fail_recips.map(map_recips), "Some recipients temporarily failed", function (hmail) { self.discard(); - hmail.temp_fail("Some recipients temp failed: " + fail_recips.join(', ')); + hmail.temp_fail("Some recipients temp failed: " + fail_recips.map(map_recips).join(', '), fail_recips); }); } if (bounce_recips.length) { self.refcount++; - exports.split_to_new_recipients(self, bounce_recips, "Some recipients rejected", function (hmail) { + exports.split_to_new_recipients(self, bounce_recips.map(map_recips), "Some recipients rejected", function (hmail) { self.discard(); - hmail.bounce("Some recipients failed: " + bounce_recips.join(', ')); + hmail.bounce("Some recipients failed: " + bounce_recips.map(map_recips).join(', '), bounce_recips); }); } if (ok_recips) { From 7bf6920b919cf7574810e77581e6a2bab188a9b6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 14:52:14 -0500 Subject: [PATCH 008/160] README.md, smtp_forward is default queue plugin make documentation match what's configured --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19974b595..ddf3b7cbb 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,8 @@ And it will run. However the big thing you want to do next is to edit the `config/plugins` file. This determines what plugins run in Haraka, and controls the overall behaviour of the server. By default the server is setup to receive mails for -domains in `host_list` and deliver them via `qmail-queue`. Queueing to -qmail is likely not what you need unless you have qmail installed, so this is -likely the first thing you want to change. +domains in `host_list` and deliver them via `smtp-forward`. Configure the +destination in `config/smtp_forward.ini`. Each plugin has documentation available via `haraka -h plugins/`. Look there for information about how each plugin is configured, edit your From bd8f2acf1d6072f2af804ca7525286c3c238e49a Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 14:53:31 -0500 Subject: [PATCH 009/160] checks for haraka -c, config dir & smtp.ini exist check that the provided path exists, a config dir exists within it, and a smtp.ini exists within the config dir. --- bin/haraka | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bin/haraka b/bin/haraka index cbe65ef5a..fbe29c16c 100755 --- a/bin/haraka +++ b/bin/haraka @@ -344,6 +344,26 @@ else if (parsed.qempty) { } else if (parsed.configs) { var haraka_path = path.join(base, 'haraka.js'); + + var base_dir = process.argv[3]; + var err_msg = "Did you install a Haraka config? (haraka -i " + base_dir +")"; + if ( !fs.existsSync(base_dir) ) + fail( "No such directory: " + base_dir + "\n" + err_msg ); + if ( ! fs.statSync(base_dir).isDirectory() ) + fail( "Not a directory at: " + base_dir + "\n" + err_msg ); + + var config_dir = path.join(base_dir,'config'); + if ( !fs.existsSync( path.join(config_dir) ) ) + fail( "No config directory in: " + base_dir ); + if ( !fs.statSync( config_dir ).isDirectory() ) + fail( "Not a directory at: " + config_dir ); + + var smtp_ini = path.join(config_dir, 'smtp.ini'); + if ( !fs.existsSync( smtp_ini ) ) + fail( "No smtp.ini in: " + config_dir ); + if ( !fs.statSync( smtp_ini ).isFile() ) + fail( "Not a file at: " + smtp_ini ); + process.argv[1] = haraka_path; process.env.HARAKA = parsed.configs; require(haraka_path); From 368f91c80e423b0c28adce84d087e0fa0ce66a51 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 18 Dec 2013 16:18:53 -0500 Subject: [PATCH 010/160] bin/haraka: simplify startup checks --- bin/haraka | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/bin/haraka b/bin/haraka index fbe29c16c..adcb39a2c 100755 --- a/bin/haraka +++ b/bin/haraka @@ -349,20 +349,10 @@ else if (parsed.configs) { var err_msg = "Did you install a Haraka config? (haraka -i " + base_dir +")"; if ( !fs.existsSync(base_dir) ) fail( "No such directory: " + base_dir + "\n" + err_msg ); - if ( ! fs.statSync(base_dir).isDirectory() ) - fail( "Not a directory at: " + base_dir + "\n" + err_msg ); - var config_dir = path.join(base_dir,'config'); - if ( !fs.existsSync( path.join(config_dir) ) ) - fail( "No config directory in: " + base_dir ); - if ( !fs.statSync( config_dir ).isDirectory() ) - fail( "Not a directory at: " + config_dir ); - - var smtp_ini = path.join(config_dir, 'smtp.ini'); + var smtp_ini = path.join(base_dir,'config','smtp.ini'); if ( !fs.existsSync( smtp_ini ) ) - fail( "No smtp.ini in: " + config_dir ); - if ( !fs.statSync( smtp_ini ).isFile() ) - fail( "Not a file at: " + smtp_ini ); + fail( "No smtp.ini at: " + smtp_ini + "\n" + err_msg ); process.argv[1] = haraka_path; process.env.HARAKA = parsed.configs; From dc6754161f8e76a22b231d284a8b052fef7a1045 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Thu, 19 Dec 2013 00:50:13 -0500 Subject: [PATCH 011/160] mail_from.is_resolvable: moved re_bogus_ip into config was hard coded in the plugin. This allows for local site modification. This is useful for entities (like me) that have firewalls with all their public IPs. The VMs and services all run on private or loopback IPs, as routed by the firewall. Being behind the firewall, Haraka sees the loopback IPs and rejects them. --- config/mail_from.is_resolvable.ini | 1 + plugins/mail_from.is_resolvable.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/mail_from.is_resolvable.ini b/config/mail_from.is_resolvable.ini index cdb20468c..6128f847a 100644 --- a/config/mail_from.is_resolvable.ini +++ b/config/mail_from.is_resolvable.ini @@ -1,3 +1,4 @@ timeout=30 allow_mx_ip=0 reject_no_mx=1 +re_bogus_ip=/^(?:0\.0\.0\.0|255\.255\.255\.255|127\.)/ diff --git a/plugins/mail_from.is_resolvable.js b/plugins/mail_from.is_resolvable.js index 10c2fa862..189719c8c 100644 --- a/plugins/mail_from.is_resolvable.js +++ b/plugins/mail_from.is_resolvable.js @@ -1,6 +1,5 @@ // Check MAIL FROM domain is resolvable to an MX var dns = require('dns'); -var re_bogus_ip = /^(?:0\.0\.0\.0|255\.255\.255\.255|127\.)/; exports.hook_mail = function(next, connection, params) { var mail_from = params[0]; @@ -14,6 +13,7 @@ exports.hook_mail = function(next, connection, params) { var plugin = this; var domain = mail_from.host; var config = this.config.get('mail_from.is_resolvable.ini'); + var re_bogus_ip = new RegExp(config.main.re_bogus_ip); // Just in case DNS never comes back (UDP), we should DENYSOFT. var timeout_id = setTimeout(function () { From 46c1d494f0acb76e99aecf0550071a67bb9b6f54 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Thu, 19 Dec 2013 12:22:43 +0000 Subject: [PATCH 012/160] Make sure that 4xx responses are sent on error or if ENOSPC --- connection.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/connection.js b/connection.js index 487f077b3..83b35e120 100644 --- a/connection.js +++ b/connection.js @@ -240,7 +240,7 @@ Connection.prototype.process_line = function (line) { else { this.logerror(method + " failed: " + err); } - this.respond(500, "Internal Server Error", function() { + this.respond(421, "Internal Server Error", function() { self.disconnect(); }); } @@ -884,6 +884,7 @@ Connection.prototype.rcpt_ok_respond = function (retval, msg) { this.lognotice(dmsg + ' ' + [ 'code=' + constants.translate(retval), 'msg="' + (msg || '') + '"', + 'sender="' + this.transaction.mail_from.address() + '"', ].join(' ')); switch (retval) { case constants.deny: @@ -934,6 +935,7 @@ Connection.prototype.rcpt_respond = function(retval, msg) { this.lognotice(dmsg + ' ' + [ 'code=' + constants.translate(retval), 'msg="' + (msg || '') + '"', + 'sender="' + this.transaction.mail_from.address() + '"', ].join(' ')); } switch (retval) { @@ -1120,7 +1122,13 @@ Connection.prototype.cmd_mail = function(line) { else { this.logerror(err); } - return this.respond(501, ["Command parsing failed", err]); + // Explicitly handle out-of-disk space errors + if (err.code === 'ENOSPC') { + return this.respond(452, 'Internal Server Error'); + } + else { + return this.respond(501, ["Command parsing failed", err]); + } } // Get rest of key=value pairs @@ -1169,7 +1177,13 @@ Connection.prototype.cmd_rcpt = function(line) { else { this.logerror(err); } - return this.respond(501, ["Command parsing failed", err]); + // Explicitly handle out-of-disk space errors + if (err.code === 'ENOSPC') { + return this.respond(452, 'Internal Server Error'); + } + else { + return this.respond(501, ["Command parsing failed", err]); + } } // Get rest of key=value pairs From 26dac04ac60ff610724e91e6d8937dcbb6116284 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Thu, 19 Dec 2013 13:29:15 -0500 Subject: [PATCH 013/160] mf.is_resolvable: added default re_bogus_ip in case it's missing from the config file --- plugins/mail_from.is_resolvable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mail_from.is_resolvable.js b/plugins/mail_from.is_resolvable.js index 189719c8c..7ab5d676e 100644 --- a/plugins/mail_from.is_resolvable.js +++ b/plugins/mail_from.is_resolvable.js @@ -13,7 +13,7 @@ exports.hook_mail = function(next, connection, params) { var plugin = this; var domain = mail_from.host; var config = this.config.get('mail_from.is_resolvable.ini'); - var re_bogus_ip = new RegExp(config.main.re_bogus_ip); + var re_bogus_ip = new RegExp(config.main.re_bogus_ip || /^(?:0\.0\.0\.0|255\.255\.255\.255|127\.)/ ); // Just in case DNS never comes back (UDP), we should DENYSOFT. var timeout_id = setTimeout(function () { From 8e9ec023e1d630e75194c07192c0a8da028ad839 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Thu, 19 Dec 2013 13:43:54 -0500 Subject: [PATCH 014/160] mf.is_resolvable: convert bogus re to string --- plugins/mail_from.is_resolvable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mail_from.is_resolvable.js b/plugins/mail_from.is_resolvable.js index 7ab5d676e..6ca86c124 100644 --- a/plugins/mail_from.is_resolvable.js +++ b/plugins/mail_from.is_resolvable.js @@ -13,7 +13,7 @@ exports.hook_mail = function(next, connection, params) { var plugin = this; var domain = mail_from.host; var config = this.config.get('mail_from.is_resolvable.ini'); - var re_bogus_ip = new RegExp(config.main.re_bogus_ip || /^(?:0\.0\.0\.0|255\.255\.255\.255|127\.)/ ); + var re_bogus_ip = new RegExp(config.main.re_bogus_ip || '^(?:0\.0\.0\.0|255\.255\.255\.255|127\.)' ); // Just in case DNS never comes back (UDP), we should DENYSOFT. var timeout_id = setTimeout(function () { From f0838c096e955f1f17024e77327f732cff8694b0 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 20 Dec 2013 04:04:38 -0500 Subject: [PATCH 015/160] new plugin: auth_vpopmaild --- config/auth_vpopmaild.ini | 2 ++ plugins/auth/auth_vpopmaild.js | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 config/auth_vpopmaild.ini create mode 100644 plugins/auth/auth_vpopmaild.js diff --git a/config/auth_vpopmaild.ini b/config/auth_vpopmaild.ini new file mode 100644 index 000000000..5ac0a43fe --- /dev/null +++ b/config/auth_vpopmaild.ini @@ -0,0 +1,2 @@ +host=127.0.0.6 +port=89 diff --git a/plugins/auth/auth_vpopmaild.js b/plugins/auth/auth_vpopmaild.js new file mode 100644 index 000000000..0366c7a85 --- /dev/null +++ b/plugins/auth/auth_vpopmaild.js @@ -0,0 +1,61 @@ +// Auth against vpopmaild + +var sock = require('./line_socket'); + +exports.register = function () { + this.inherits('auth/auth_base'); +} + +exports.hook_capabilities = function (next, connection) { + var config = this.config.get('auth_vpopmaild.ini'); + if (connection.using_tls) { + var methods = [ 'PLAIN', 'LOGIN' ]; + connection.capabilities.push('AUTH ' + methods.join(' ')); + connection.notes.allowed_auth_methods = methods; + } + next(); +}; + +exports.check_plain_passwd = function (connection, user, passwd, cb) { + this.try_auth_vpopmaild(connection, user, passwd, cb); +} + +exports.try_auth_vpopmaild = function (connection, user, passwd, cb) { + + var plugin = this; + var config = this.config.get('auth_vpopmaild.ini'); + + var auth_success = false; + var result = ""; + + var socket = new sock.Socket(); + socket.connect( ( config.main.port || 89), (config.main.host || '127.0.0.1') ); + socket.setTimeout(300 * 1000); + + socket.on('timeout', function () { + connection.logerror(plugin, "vpopmaild connection timed out"); + socket.end(); + }); + socket.on('error', function (err) { + connection.logerror(plugin, "vpopmaild connection failed: " + err); + }); + socket.on('connect', function () { + socket.write("login " + user + ' ' + passwd + "\n\r"); + }); + socket.on('line', function (line) { + connection.logprotocol(plugin, 'C:' + line); + if (line.match(/^\+OK/)) { + auth_success = true; + } + if ( line.match(/^\./) ) + socket.end(); + }); + socket.on('close', function () { + connection.loginfo(plugin, 'AUTH user="' + user + '" success=' + auth_success); + return cb(auth_success); + }); + socket.on('end', function () { +// return cb(auth_success); + }); +}; + From 395b7229a8ee9dc2c9419d19b6c6737a18a42080 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 20 Dec 2013 16:53:03 -0500 Subject: [PATCH 016/160] auth_vpopmaild: s/\t/ /g & added doc file --- docs/plugins/auth/auth_vpopmaild.js | 18 ++++++++++++++++++ plugins/auth/auth_vpopmaild.js | 9 +++++---- 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 docs/plugins/auth/auth_vpopmaild.js diff --git a/docs/plugins/auth/auth_vpopmaild.js b/docs/plugins/auth/auth_vpopmaild.js new file mode 100644 index 000000000..ccfeb373e --- /dev/null +++ b/docs/plugins/auth/auth_vpopmaild.js @@ -0,0 +1,18 @@ +auth/auth_vpopmaild +============== + +The `auth/vpopmaild` plugin allows you to authenticate against a vpopmaild +daemon. + +Configuration +------------- + +Configuration is stored in `config/auth_vpopmaild.ini` and uses the INI +style formatting. + +There are two configuration settings: + +host: The host/IP that vpopmaild is listening on (default: localhost). + +port: The TCP port that vpopmaild is listening on (default: 89). + diff --git a/plugins/auth/auth_vpopmaild.js b/plugins/auth/auth_vpopmaild.js index 0366c7a85..97b98bbfc 100644 --- a/plugins/auth/auth_vpopmaild.js +++ b/plugins/auth/auth_vpopmaild.js @@ -40,22 +40,23 @@ exports.try_auth_vpopmaild = function (connection, user, passwd, cb) { connection.logerror(plugin, "vpopmaild connection failed: " + err); }); socket.on('connect', function () { - socket.write("login " + user + ' ' + passwd + "\n\r"); + socket.write("login " + user + ' ' + passwd + "\n\r"); }); socket.on('line', function (line) { connection.logprotocol(plugin, 'C:' + line); if (line.match(/^\+OK/)) { - auth_success = true; + auth_success = true; } - if ( line.match(/^\./) ) + if ( line.match(/^\./) ) { socket.end(); + } }); socket.on('close', function () { connection.loginfo(plugin, 'AUTH user="' + user + '" success=' + auth_success); return cb(auth_success); }); socket.on('end', function () { -// return cb(auth_success); + // return cb(auth_success); }); }; From f2602aa68438555020e6fd0b294851380c5c3d44 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 20 Dec 2013 17:55:10 -0500 Subject: [PATCH 017/160] flat_file: typo & grammar fixes in docs --- docs/plugins/auth/flat_file.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/plugins/auth/flat_file.md b/docs/plugins/auth/flat_file.md index f6c9760e9..b1b140f21 100644 --- a/docs/plugins/auth/flat_file.md +++ b/docs/plugins/auth/flat_file.md @@ -16,10 +16,10 @@ Configuration is stored in `config/auth_flat_file.ini` and uses the INI style formatting. Authentification methods are listed in the `[core]` section under `methods` -parameter. You can list few authentification methods comma separated. Currently -are only two methods supported : `CRAM-MD5` and `LOGIN`. Be aware, the LOGIN -method is highly unsecure and can be used normaly only for local communication. -We stronly recommend only `CRAM-MD5` to be used. +parameter. You can list a few authentification methods comma separated. Currently +only two methods are supported : `CRAM-MD5` and `LOGIN`. Be aware, the LOGIN +method is highly unsecure and should be used only for local communication. +We strongly recommend only `CRAM-MD5`. Example: [core] From 0117e97c17c29468f59250fdfd1ca285859fbcdb Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 23 Dec 2013 04:13:19 -0500 Subject: [PATCH 018/160] graph: updated to work with sqlite3, issue #369 the original node-sqlite plugin is deprecated and doesn't build. resolves issue #369 --- plugins/graph.js | 62 +++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/plugins/graph.js b/plugins/graph.js index 8d1631e27..fa3a1ddb3 100644 --- a/plugins/graph.js +++ b/plugins/graph.js @@ -1,35 +1,16 @@ // log our denys -var sqlite = require('sqlite'); -var db = new sqlite.Database(); +var sqlite3 = require('sqlite3').verbose(); +// var db = new sqlite3.Database(':memory:', createTable); +var db = new sqlite3.Database('graphlog.db', createTable); -var insert; var select = "SELECT COUNT(*) AS hits, plugin FROM graphdata WHERE timestamp >= ? AND timestamp < ? GROUP BY plugin"; +var insert = db.prepare( "INSERT INTO graphdata VALUES (?,?)" ); -db.open('graphlog.db', function (err) { - if (err) { - throw err; - } - db.execute("PRAGMA synchronous=NORMAL", // make faster, rawwwrr - function (err, rows) { - if (err) { - throw err; - } - db.execute("CREATE TABLE graphdata (timestamp INTEGER NOT NULL, plugin TEXT NOT NULL)", - function (err, rows) { - if (!err) { - db.execute("CREATE INDEX graphdata_idx ON graphdata (timestamp)", function(){}); - } - db.prepare("INSERT INTO graphdata VALUES (?,?)", - function (err, stmt) { - if (err) { - throw err; - } - insert = stmt; - }); - }); - }); -}); +function createTable() { + db.exec( "CREATE TABLE IF NOT EXISTS graphdata (timestamp INTEGER NOT NULL, plugin TEXT NOT NULL)") + .exec( "CREATE INDEX IF NOT EXISTS graphdata_idx ON graphdata (timestamp)"); +} var plugins = {}; @@ -61,12 +42,12 @@ exports.hook_init_master = function (next) { function (req, res) { plugin.handle_http_request(req, res); }); - + server.on('error', function (err) { plugin.logerror("http server failed to start. Maybe running elsewhere?" + err); next(DENY); }); - + server.listen(port, "127.0.0.1", function () { plugin.loginfo("http server running on port " + port); next(); @@ -83,12 +64,12 @@ exports.hook_disconnect = function (next, connection) { exports.hook_deny = function (next, connection, params) { var plugin = this; - insert.bindArray([new Date().getTime(), params[2]], function (err) { + insert.bind([new Date().getTime(), params[2]], function (err) { if (err) { plugin.logerror("Insert DENY failed: " + err); return next(); } - insert.fetchAll(function (err, rows) { + insert.run(function (err, rows) { if (err) { plugin.logerror("Insert failed: " + err); } @@ -100,12 +81,12 @@ exports.hook_deny = function (next, connection, params) { exports.hook_queue_ok = function (next, connection, params) { var plugin = this; - insert.bindArray([new Date().getTime(), 'accepted'], function (err) { + insert.bind([new Date().getTime(), 'accepted'], function (err) { if (err) { plugin.logerror("Insert DENY failed: " + err); return next(); } - insert.fetchAll(function (err, rows) { + insert.run(function (err, rows) { if (err) { plugin.logerror("Insert failed: " + err); } @@ -117,7 +98,7 @@ exports.hook_queue_ok = function (next, connection, params) { exports.handle_http_request = function (req, res) { var parsed = urlp.parse(req.url, true); - this.loginfo("Handling URL: " + parsed.href); + // this.loginfo("Handling URL: " + parsed.href); switch (parsed.pathname) { case '/': this.handle_root(res, parsed); @@ -229,12 +210,16 @@ exports.get_data = function (res, earliest, today, group_by) { res.write(data + "\n"); } - db.query(select, [earliest, next_stop], function (err, row) { + db.each(select, [earliest, next_stop], function (err, row) { if (err) { res.end(); return plugin.logerror("SELECT failed: " + err); } - if (!row) { + plugin.loginfo("got: " + row.hits + ", " + row.plugin + " next_stop: " + next_stop); + + aggregate[row.plugin] = row.hits; + }, + function (err, rows ) { write_to(utils.ISODate(new Date(next_stop)) + ',' + utils.sort_keys(plugins).map(function(i){ return 1000 * 60 * (aggregate[i]/group_by) }).join(',') ); @@ -247,10 +232,7 @@ exports.get_data = function (res, earliest, today, group_by) { }); } } - // plugin.loginfo("got: " + row.hits + ", " + row.plugin + " next_stop: " + next_stop); - - aggregate[row.plugin] = row.hits; - }); + ); }; var reset_agg = function () { From 7d71e01a279fff0adb65b09977f26f8bcf10d5fe Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 23 Dec 2013 15:25:40 -0500 Subject: [PATCH 019/160] added graph.ini, consolidate graph settings into .ini removing the need to edit graph.js to alter the DB file/path, or listening IP will use old .ini files if newer config.ini not present, and retains defaults --- config/graph.ini | 12 ++++++++++++ docs/plugins/graph.md | 18 ++++++++++++++---- plugins/graph.js | 35 ++++++++++++++++++++--------------- 3 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 config/graph.ini diff --git a/config/graph.ini b/config/graph.ini new file mode 100644 index 000000000..43755dacc --- /dev/null +++ b/config/graph.ini @@ -0,0 +1,12 @@ +; the filename the SQLite database is stored in +; can be a file name or :memory: +;db_file=:memory: +db_file=graphlog.db + +; The port to listen on for http. Default: `8080`. +http_addr=127.0.0.1 +http_port=8080 + +; Regular expression to match plugins to ignore for logging. +; Default: `queue|graph|relay` +ignore_re=`queue|graph|relay` diff --git a/docs/plugins/graph.md b/docs/plugins/graph.md index d9e3c4b3d..63bcebe07 100644 --- a/docs/plugins/graph.md +++ b/docs/plugins/graph.md @@ -5,17 +5,27 @@ This plugin logs accepted and rejected emails into a database and provides a web server which you can browse to and view graphs over time of the plugins which rejected connections. -In order for this to work you need to install the `sqlite` module via -`npm install sqlite` in your Haraka directory. +In order for this to work you need to install the `sqlite3` module via +`npm install sqlite3` in your Haraka directory. Configuration ------------- -* grapher.http_port +config settings are stored in config/graph.ini + +* db_file + + The file name (or :memory:), where data is stored + +* http_addr + + The IP address to listen on for http. Default: `127.0.0.1`. + +* http_port The port to listen on for http. Default: `8080`. -* grapher.ignore_re +* ignore_re Regular expression to match plugins to ignore for logging. Default: `queue|graph|relay` diff --git a/plugins/graph.js b/plugins/graph.js index fa3a1ddb3..562739e13 100644 --- a/plugins/graph.js +++ b/plugins/graph.js @@ -1,27 +1,27 @@ // log our denys +var http = require('http'); +var urlp = require('url'); +var utils = require('./utils'); + var sqlite3 = require('sqlite3').verbose(); -// var db = new sqlite3.Database(':memory:', createTable); -var db = new sqlite3.Database('graphlog.db', createTable); +var db; var select = "SELECT COUNT(*) AS hits, plugin FROM graphdata WHERE timestamp >= ? AND timestamp < ? GROUP BY plugin"; -var insert = db.prepare( "INSERT INTO graphdata VALUES (?,?)" ); +var insert; +var plugins = {}; +var config; + +var width = 800; function createTable() { db.exec( "CREATE TABLE IF NOT EXISTS graphdata (timestamp INTEGER NOT NULL, plugin TEXT NOT NULL)") .exec( "CREATE INDEX IF NOT EXISTS graphdata_idx ON graphdata (timestamp)"); } -var plugins = {}; - -var http = require('http'); -var urlp = require('url'); -var utils = require('./utils'); -var width = 800; - exports.register = function () { - var plugin = this; - var ignore_re = this.config.get('grapher.ignore_re') || 'queue|graph|relay'; + config = this.config.get('graph.ini'); + var ignore_re = config.main.ignore_re || this.config.get('grapher.ignore_re') || 'queue|graph|relay'; ignore_re = new RegExp(ignore_re); plugins = {accepted: 0, disconnect_early: 0}; @@ -33,11 +33,16 @@ exports.register = function () { } } ); + + var db_name = config.main.db_file || 'graphlog.db'; + db = new sqlite3.Database(db_name, createTable); + insert = db.prepare( "INSERT INTO graphdata VALUES (?,?)" ); }; exports.hook_init_master = function (next) { var plugin = this; - var port = this.config.get('grapher.http_port') || 8080; + var port = config.main.http_port || this.config.get('grapher.http_port') || 8080; + var addr = config.main.http_addr || '127.0.0.1'; var server = http.createServer( function (req, res) { plugin.handle_http_request(req, res); @@ -48,8 +53,8 @@ exports.hook_init_master = function (next) { next(DENY); }); - server.listen(port, "127.0.0.1", function () { - plugin.loginfo("http server running on port " + port); + server.listen(port, addr, function () { + plugin.loginfo("http server running on " + addr + ':' port); next(); }); } From 23098a4cada81515234b20b00e03e8f8bde6f608 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 23 Dec 2013 15:38:46 -0500 Subject: [PATCH 020/160] graph: added a missing + in the log message --- plugins/graph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/graph.js b/plugins/graph.js index 562739e13..c9a55321d 100644 --- a/plugins/graph.js +++ b/plugins/graph.js @@ -54,7 +54,7 @@ exports.hook_init_master = function (next) { }); server.listen(port, addr, function () { - plugin.loginfo("http server running on " + addr + ':' port); + plugin.loginfo("http server running on " + addr + ':' + port); next(); }); } From d80962e7e051c9a5cec352bd983d061aef0ea981 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 15:56:49 -0500 Subject: [PATCH 021/160] new plugin: rcpt_to.qmail_deliverable --- config/rcpt_to.qmail_deliverable.ini | 8 ++ plugins/rcpt_to.qmail_deliverable.js | 122 +++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 config/rcpt_to.qmail_deliverable.ini create mode 100644 plugins/rcpt_to.qmail_deliverable.js diff --git a/config/rcpt_to.qmail_deliverable.ini b/config/rcpt_to.qmail_deliverable.ini new file mode 100644 index 000000000..28dd54d4c --- /dev/null +++ b/config/rcpt_to.qmail_deliverable.ini @@ -0,0 +1,8 @@ +; the IP address of the host running qmail_deliverable +; default: host=127.0.0.1 +host=127.0.0.1 + +; the TCP port qmail_deliverabled is listening on, default 8998 +port=8998 + + diff --git a/plugins/rcpt_to.qmail_deliverable.js b/plugins/rcpt_to.qmail_deliverable.js new file mode 100644 index 000000000..529dfe582 --- /dev/null +++ b/plugins/rcpt_to.qmail_deliverable.js @@ -0,0 +1,122 @@ +// validate an email address is local, using qmail-deliverabled + +var http = require('http'); + +var options = { + method: 'get', +} + +exports.register = function() { + var config = this.config.get('rcpt_to.qmail_deliverable.ini'); + + options.host = config.main.host || '127.0.0.1'; + options.port = config.main.port || 8998; + + this.logdebug(this, "host: " + options.host ); + this.logdebug(this, "port: " + options.port ); + + this.register_hook('rcpt', 'rcpt_to_qmd'); +} + +exports.rcpt_to_qmd = function(next, connection, params) { + var rcpt = params[0]; + var email = rcpt.address(); + + // TODO: this is a good place to validate email + // the perl Qmail::Deliverable client does a rfc2822 "atext" test + // but Haraka might have done this for us, by this point + + this.logdebug("checking " + email ); + return get_qmd_response(next,connection,email); +} + +function get_qmd_response(next,conn,email) { + options.path = '/qd1/deliverable?' + email; + conn.logprotocol(conn, 'PATH: ' + options.path); + var req = http.get(options, function(res) { + conn.logprotocol(conn, 'STATUS: ' + res.statusCode); + conn.logprotocol(conn, 'HEADERS: ' + JSON.stringify(res.headers)); + res.on('data', function (chunk) { + res.setEncoding('utf8'); + conn.logprotocol(conn, 'BODY: ' + chunk); + var hexnum = new Number(chunk).toString(16); + return check_qmd_reponse( next, conn, hexnum ); + }); + }).on('error', function(e) { + conn.loginfo(conn,"Got error: " + e.message); + }); +}; + +function check_qmd_reponse(next,connection,hexnum) { + connection.logprotocol( "HEXRV: " + hexnum ); + + switch(hexnum) { + case '11': + connection.loginfo("qmd error, permission failure"); + return next(); + break; + case '12': + connection.loginfo("qmd pass, qmail-command in dot-qmail"); + return next(OK); + break; + case '13': + connection.loginfo("qmd pass, bouncesaying with program"); + return next(OK); + break; + case '14': + var from = connection.transaction.mail_from.address(); + if ( ! from || from === '<>') { + return next(DENY, "fail, mailing lists do not accept null senders"); + } + connection.loginfo("qmd pass, ezmlm list"); + return next(OK); + break; + case '21': + connection.loginfo("qmd Temporarily undeliverable: group/world writable"); + return next(); + break; + case '22': + connection.loginfo("qmd Temporarily undeliverable: sticky home directory"); + return next(); + break; + case '2f': + connection.loginfo("qmd error, Qmail::Deliverable::Client::ERROR"); + return next(); + break; + case 'f1': + connection.loginfo("qmd pass, normal delivery"); + return next(OK); + break; + case 'f2': + connection.loginfo("qmd pass, vpopmail dir"); + return next(OK); + break; + case 'f3': + connection.loginfo("qmd pass, vpopmail alias"); + return next(OK); + break; + case 'f4': + connection.loginfo("qmd pass, vpopmail catchall"); + return next(OK); + break; + case 'f5': + connection.loginfo("qmd pass, vpopmail vuser"); + return next(OK); + break; + case 'f6': + connection.loginfo("qmd pass, vpopmail qmail-ext"); + return next(OK); + break; + case 'fe': + connection.loginfo("qmd error, SHOULD NOT HAPPEN"); + return next(); + break; + case 'ff': + connection.loginfo("qmd fail, address not local"); + return next(); + break; + default: + connection.loginfo("qmd error, unknown rv: " + hexnum); + return next(); + } +}; From 8d35d366b8548cb23aa23f9aae4e4f2005bfba28 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 19:06:09 -0500 Subject: [PATCH 022/160] qmd: code quality improvements removed no-longer-needed breaks changed error message for unknown QMD::Client error pass this, so error messages log with plugin name --- plugins/rcpt_to.qmail_deliverable.js | 67 +++++++++++----------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/plugins/rcpt_to.qmail_deliverable.js b/plugins/rcpt_to.qmail_deliverable.js index 529dfe582..8d2209ea7 100644 --- a/plugins/rcpt_to.qmail_deliverable.js +++ b/plugins/rcpt_to.qmail_deliverable.js @@ -27,96 +27,81 @@ exports.rcpt_to_qmd = function(next, connection, params) { // but Haraka might have done this for us, by this point this.logdebug("checking " + email ); - return get_qmd_response(next,connection,email); + return get_qmd_response(next,this,connection,email); } -function get_qmd_response(next,conn,email) { +function get_qmd_response(next,plugin,connection,email) { options.path = '/qd1/deliverable?' + email; - conn.logprotocol(conn, 'PATH: ' + options.path); + plugin.logprotocol('PATH: ' + options.path); var req = http.get(options, function(res) { - conn.logprotocol(conn, 'STATUS: ' + res.statusCode); - conn.logprotocol(conn, 'HEADERS: ' + JSON.stringify(res.headers)); + plugin.logprotocol('STATUS: ' + res.statusCode); + plugin.logprotocol('HEADERS: ' + JSON.stringify(res.headers)); res.on('data', function (chunk) { res.setEncoding('utf8'); - conn.logprotocol(conn, 'BODY: ' + chunk); + plugin.logprotocol('BODY: ' + chunk); var hexnum = new Number(chunk).toString(16); - return check_qmd_reponse( next, conn, hexnum ); + return check_qmd_reponse(next,plugin,connection,hexnum); }); }).on('error', function(e) { - conn.loginfo(conn,"Got error: " + e.message); + plugin.loginfo("Got error: " + e.message); }); }; -function check_qmd_reponse(next,connection,hexnum) { - connection.logprotocol( "HEXRV: " + hexnum ); +function check_qmd_reponse(next,plugin,connection,hexnum) { + plugin.logprotocol("HEXRV: " + hexnum ); switch(hexnum) { case '11': - connection.loginfo("qmd error, permission failure"); + plugin.loginfo("error, permission failure"); return next(); - break; case '12': - connection.loginfo("qmd pass, qmail-command in dot-qmail"); + plugin.loginfo("pass, qmail-command in dot-qmail"); return next(OK); - break; case '13': - connection.loginfo("qmd pass, bouncesaying with program"); + plugin.loginfo("pass, bouncesaying with program"); return next(OK); - break; case '14': var from = connection.transaction.mail_from.address(); if ( ! from || from === '<>') { return next(DENY, "fail, mailing lists do not accept null senders"); } - connection.loginfo("qmd pass, ezmlm list"); + plugin.loginfo("pass, ezmlm list"); return next(OK); - break; case '21': - connection.loginfo("qmd Temporarily undeliverable: group/world writable"); + plugin.loginfo("Temporarily undeliverable: group/world writable"); return next(); - break; case '22': - connection.loginfo("qmd Temporarily undeliverable: sticky home directory"); + plugin.loginfo("Temporarily undeliverable: sticky home directory"); return next(); - break; case '2f': - connection.loginfo("qmd error, Qmail::Deliverable::Client::ERROR"); + plugin.loginfo("error communicating with qmail-deliverabled."); return next(); - break; case 'f1': - connection.loginfo("qmd pass, normal delivery"); + plugin.loginfo("pass, normal delivery"); return next(OK); - break; case 'f2': - connection.loginfo("qmd pass, vpopmail dir"); + plugin.loginfo("pass, vpopmail dir"); return next(OK); - break; case 'f3': - connection.loginfo("qmd pass, vpopmail alias"); + plugin.loginfo("pass, vpopmail alias"); return next(OK); - break; case 'f4': - connection.loginfo("qmd pass, vpopmail catchall"); + plugin.loginfo("pass, vpopmail catchall"); return next(OK); - break; case 'f5': - connection.loginfo("qmd pass, vpopmail vuser"); + plugin.loginfo("pass, vpopmail vuser"); return next(OK); - break; case 'f6': - connection.loginfo("qmd pass, vpopmail qmail-ext"); + plugin.loginfo("pass, vpopmail qmail-ext"); return next(OK); - break; case 'fe': - connection.loginfo("qmd error, SHOULD NOT HAPPEN"); + plugin.loginfo("error, SHOULD NOT HAPPEN"); return next(); - break; case 'ff': - connection.loginfo("qmd fail, address not local"); + plugin.loginfo("fail, address not local"); return next(); - break; default: - connection.loginfo("qmd error, unknown rv: " + hexnum); + plugin.loginfo("error, unknown rv: " + hexnum); return next(); } }; From 2d7d76f0bff35a6ceba0e7ee8ffa63b779f59bd1 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 19:17:17 -0500 Subject: [PATCH 023/160] qmd: url escape email address --- plugins/rcpt_to.qmail_deliverable.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/rcpt_to.qmail_deliverable.js b/plugins/rcpt_to.qmail_deliverable.js index 8d2209ea7..ec2198e6e 100644 --- a/plugins/rcpt_to.qmail_deliverable.js +++ b/plugins/rcpt_to.qmail_deliverable.js @@ -1,6 +1,7 @@ // validate an email address is local, using qmail-deliverabled var http = require('http'); +var querystring = require('querystring'); var options = { method: 'get', @@ -31,7 +32,7 @@ exports.rcpt_to_qmd = function(next, connection, params) { } function get_qmd_response(next,plugin,connection,email) { - options.path = '/qd1/deliverable?' + email; + options.path = '/qd1/deliverable?' + querystring.escape(email); plugin.logprotocol('PATH: ' + options.path); var req = http.get(options, function(res) { plugin.logprotocol('STATUS: ' + res.statusCode); From 09bb1e3fdf3ba195d5a95e2cc9347053ba88cba8 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 19:20:57 -0500 Subject: [PATCH 024/160] qmd: move res.setEncoding before data --- plugins/rcpt_to.qmail_deliverable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/rcpt_to.qmail_deliverable.js b/plugins/rcpt_to.qmail_deliverable.js index ec2198e6e..90a57e5b0 100644 --- a/plugins/rcpt_to.qmail_deliverable.js +++ b/plugins/rcpt_to.qmail_deliverable.js @@ -37,8 +37,8 @@ function get_qmd_response(next,plugin,connection,email) { var req = http.get(options, function(res) { plugin.logprotocol('STATUS: ' + res.statusCode); plugin.logprotocol('HEADERS: ' + JSON.stringify(res.headers)); + res.setEncoding('utf8'); res.on('data', function (chunk) { - res.setEncoding('utf8'); plugin.logprotocol('BODY: ' + chunk); var hexnum = new Number(chunk).toString(16); return check_qmd_reponse(next,plugin,connection,hexnum); From f1d25b9e85df1b0811a5de5d521b5549ca728fb2 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 19:25:25 -0500 Subject: [PATCH 025/160] added docs for qmail_deliverable plugin --- docs/plugins/rcpt_to.qmail_deliverable.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/plugins/rcpt_to.qmail_deliverable.md diff --git a/docs/plugins/rcpt_to.qmail_deliverable.md b/docs/plugins/rcpt_to.qmail_deliverable.md new file mode 100644 index 000000000..c523d42c1 --- /dev/null +++ b/docs/plugins/rcpt_to.qmail_deliverable.md @@ -0,0 +1,15 @@ + +qmail_deliverable +============ + +This plugin implements a client for checking the deliverability of an email +address against the qmail-deliverabled daemon. +See http://search.cpan.org/dist/Qmail-Deliverable/ + + +Configuration +------------- + +You can modify the host/port that qmail-deliverabled is listening on by +altering the contents of config/rcpt_to.qmail_deliverable.ini + From 040258390638bf86cfff0145641e8f40637496c5 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 22:26:32 -0500 Subject: [PATCH 026/160] qmd: log entries have connection UUID and config fetching moved inside the rcpt_to hook --- plugins/rcpt_to.qmail_deliverable.js | 73 ++++++++++++++-------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/plugins/rcpt_to.qmail_deliverable.js b/plugins/rcpt_to.qmail_deliverable.js index 90a57e5b0..0529fbaa1 100644 --- a/plugins/rcpt_to.qmail_deliverable.js +++ b/plugins/rcpt_to.qmail_deliverable.js @@ -8,101 +8,100 @@ var options = { } exports.register = function() { - var config = this.config.get('rcpt_to.qmail_deliverable.ini'); + this.register_hook('rcpt', 'rcpt_to_qmd'); +}; + +exports.rcpt_to_qmd = function(next, connection, params) { + var config = this.config.get('rcpt_to.qmail_deliverable.ini'); options.host = config.main.host || '127.0.0.1'; options.port = config.main.port || 8998; + this.logdebug(connection, "host: " + options.host ); + this.logdebug(connection, "port: " + options.port ); - this.logdebug(this, "host: " + options.host ); - this.logdebug(this, "port: " + options.port ); - - this.register_hook('rcpt', 'rcpt_to_qmd'); -} - -exports.rcpt_to_qmd = function(next, connection, params) { var rcpt = params[0]; var email = rcpt.address(); // TODO: this is a good place to validate email - // the perl Qmail::Deliverable client does a rfc2822 "atext" test - // but Haraka might have done this for us, by this point + // Qmail::Deliverable::Client does a rfc2822 "atext" test + // but Haraka might have done this for us, at this point? - this.logdebug("checking " + email ); + this.logdebug(connection, "checking " + email ); return get_qmd_response(next,this,connection,email); } -function get_qmd_response(next,plugin,connection,email) { +function get_qmd_response(next,plugin,conn,email) { options.path = '/qd1/deliverable?' + querystring.escape(email); - plugin.logprotocol('PATH: ' + options.path); + plugin.logprotocol(conn, 'PATH: ' + options.path); var req = http.get(options, function(res) { - plugin.logprotocol('STATUS: ' + res.statusCode); - plugin.logprotocol('HEADERS: ' + JSON.stringify(res.headers)); + plugin.logprotocol(conn, 'STATUS: ' + res.statusCode); + plugin.logprotocol(conn, 'HEADERS: ' + JSON.stringify(res.headers)); res.setEncoding('utf8'); res.on('data', function (chunk) { - plugin.logprotocol('BODY: ' + chunk); + plugin.logprotocol(conn, 'BODY: ' + chunk); var hexnum = new Number(chunk).toString(16); - return check_qmd_reponse(next,plugin,connection,hexnum); + return check_qmd_reponse(next,plugin,conn,hexnum); }); }).on('error', function(e) { - plugin.loginfo("Got error: " + e.message); + plugin.loginfo(conn, "Got error: " + e.message); }); }; -function check_qmd_reponse(next,plugin,connection,hexnum) { - plugin.logprotocol("HEXRV: " + hexnum ); +function check_qmd_reponse(next,plugin,conn,hexnum) { + plugin.logprotocol(conn,"HEXRV: " + hexnum ); switch(hexnum) { case '11': - plugin.loginfo("error, permission failure"); + plugin.loginfo(conn, "error, permission failure"); return next(); case '12': - plugin.loginfo("pass, qmail-command in dot-qmail"); + plugin.loginfo(conn, "pass, qmail-command in dot-qmail"); return next(OK); case '13': - plugin.loginfo("pass, bouncesaying with program"); + plugin.loginfo(conn, "pass, bouncesaying with program"); return next(OK); case '14': - var from = connection.transaction.mail_from.address(); + var from = conn.transaction.mail_from.address(); if ( ! from || from === '<>') { return next(DENY, "fail, mailing lists do not accept null senders"); } - plugin.loginfo("pass, ezmlm list"); + plugin.loginfo(conn, "pass, ezmlm list"); return next(OK); case '21': - plugin.loginfo("Temporarily undeliverable: group/world writable"); + plugin.loginfo(conn, "Temporarily undeliverable: group/world writable"); return next(); case '22': - plugin.loginfo("Temporarily undeliverable: sticky home directory"); + plugin.loginfo(conn, "Temporarily undeliverable: sticky home directory"); return next(); case '2f': - plugin.loginfo("error communicating with qmail-deliverabled."); + plugin.loginfo(conn, "error communicating with qmail-deliverabled."); return next(); case 'f1': - plugin.loginfo("pass, normal delivery"); + plugin.loginfo(conn, "pass, normal delivery"); return next(OK); case 'f2': - plugin.loginfo("pass, vpopmail dir"); + plugin.loginfo(conn, "pass, vpopmail dir"); return next(OK); case 'f3': - plugin.loginfo("pass, vpopmail alias"); + plugin.loginfo(conn, "pass, vpopmail alias"); return next(OK); case 'f4': - plugin.loginfo("pass, vpopmail catchall"); + plugin.loginfo(conn, "pass, vpopmail catchall"); return next(OK); case 'f5': - plugin.loginfo("pass, vpopmail vuser"); + plugin.loginfo(conn, "pass, vpopmail vuser"); return next(OK); case 'f6': - plugin.loginfo("pass, vpopmail qmail-ext"); + plugin.loginfo(conn, "pass, vpopmail qmail-ext"); return next(OK); case 'fe': - plugin.loginfo("error, SHOULD NOT HAPPEN"); + plugin.loginfo(conn, "error, SHOULD NOT HAPPEN"); return next(); case 'ff': - plugin.loginfo("fail, address not local"); + plugin.loginfo(conn, "fail, address not local"); return next(); default: - plugin.loginfo("error, unknown rv: " + hexnum); + plugin.loginfo(conn, "error, unknown rv: " + hexnum); return next(); } }; From ac7cd62ca6a0917c44074681230e52bc41ec337d Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 20:22:49 -0500 Subject: [PATCH 027/160] added docs/Logging.md --- docs/Logging.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/Logging.md diff --git a/docs/Logging.md b/docs/Logging.md new file mode 100644 index 000000000..5fc85db02 --- /dev/null +++ b/docs/Logging.md @@ -0,0 +1,46 @@ + +Logging +================== + +Logging conventions within Haraka + +See also +------------------ +https://github.com/baudehlo/Haraka/pull/119 + +logline will always always be in the form: + + [level] [connection uuid] [origin] message + +where origin is "haraka\_core" or the name of the plugin which +triggered the message, and "connection uuid" is the ID of the +connection associated with the message. + +when calling a log method on logger, you should provide the +plugin object and the connection object anywhere in the arguments +to the log method. + + logger.logdebug("i like turtles", plugin, connection); + +will yield, for example, + + [DEBUG] [7F1C820F-DC79-4192-9AA6-5307354B20A6] [dnsbl] i like turtles + +if you call the log method on the connection object, you can +forego the connection as argument: + + connection.logdebug("turtles all the way down", plugin); + +and similarly for the log methods on the plugin object: + + plugin.logdebug("he just really likes turtles", connection); + +failing to provide a connection and/or plugin object will leave +the default values in the log (currently "core" and +"no\_connection"). + +this is implemented by testing for argument type in +the logger.js log\* method. objects-as-arguments are then sniffed +to try to determine if they're a connection or plugin instance. + + From 80bdac976d0b59a03fb4caec3128ff1295aefb6d Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 15:46:04 -0500 Subject: [PATCH 028/160] renamed Logging.md -> Logging_API.md --- docs/{Logging.md => Logging_API.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{Logging.md => Logging_API.md} (100%) diff --git a/docs/Logging.md b/docs/Logging_API.md similarity index 100% rename from docs/Logging.md rename to docs/Logging_API.md From 2b220c214971db515e07c63755ab9088cac02da6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 15:46:44 -0500 Subject: [PATCH 029/160] Logging_API: addded API suffix to H1 --- docs/Logging_API.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Logging_API.md b/docs/Logging_API.md index 5fc85db02..4d7876690 100644 --- a/docs/Logging_API.md +++ b/docs/Logging_API.md @@ -1,5 +1,5 @@ -Logging +Logging API ================== Logging conventions within Haraka From 6cd520166b6472680ad4ecf26cb86fd030b2cd87 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 18:02:28 -0500 Subject: [PATCH 030/160] rfc5322_header: allow config file overrides --- plugins/data.rfc5322_header_checks.js | 31 ++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/plugins/data.rfc5322_header_checks.js b/plugins/data.rfc5322_header_checks.js index bc052ec0f..1443251e1 100644 --- a/plugins/data.rfc5322_header_checks.js +++ b/plugins/data.rfc5322_header_checks.js @@ -1,8 +1,29 @@ -// Enforce RFC 5322 Section 3.6 + +// Enforce RFC 5322 Section 3.6 var required_headers = ['Date', 'From']; -var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', +var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', 'Bcc', 'Message-Id', 'In-Reply-To', 'References', 'Subject']; +var date_future_days = 2; +var date_past_days = 15; + +exports.register = function() { + var config = this.config.get('data.headers.ini'); + + if ( config.main.required ) { + required_headers = config.main.required.split(','); + }; + if ( config.main.singular ) { + singular_headers = config.main.singular.split(','); + }; + + if ( config.main.date_future_days ) { + date_future_days = config.main.date_future_days; + } + if ( config.main.date_past_days ) { + date_past_days = config.main.date_past_days; + } +} exports.hook_data_post = function (next, connection) { var header = connection.transaction.header; @@ -10,7 +31,7 @@ exports.hook_data_post = function (next, connection) { for (var i=0,l=required_headers.length; i < l; i++) { if (header.get_all(required_headers[i]).length === 0) { - return next(DENY, "Required header '" + required_headers[i] + + return next(DENY, "Required header '" + required_headers[i] + "' missing"); } } @@ -18,8 +39,8 @@ exports.hook_data_post = function (next, connection) { // Headers that MUST be unique if present for (var i=0,l=singular_headers.length; i < l; i++) { if (header.get_all(singular_headers[i]).length > 1) { - return next(DENY, "Message contains non-unique '" + - singular_headers[i] + "' header"); + return next(DENY, "Only one " + singular_headers[i] + + " header allowed. See RFC 5322, Section 3.6"); } } From 6f578a820a182a34ae96dfcbf27795dc71eb7a6e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 18:06:56 -0500 Subject: [PATCH 031/160] added data.headers.ini --- config/data.headers | 25 +++++++++++++++++++++++++ config/data.headers.ini | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 config/data.headers create mode 100644 config/data.headers.ini diff --git a/config/data.headers b/config/data.headers new file mode 100644 index 000000000..7d2c880b0 --- /dev/null +++ b/config/data.headers @@ -0,0 +1,25 @@ +; configuration for data.headers plugin + +; Requiring a date header will cause the loss of valid mail. The JavaMail +; sender used by some banks, photo processing services, health insurance +; companies, bounce senders, and others send messages without a Date header. +; +; If you can afford to reject some valid mail, please do enforce this, and +; encourage mailers toward RFC adherence. Otherwise, do not require Date. + +; Headers that MUST be present (RFC 5322) +; required=From,Date ; <-- RFC 5322 compliant +required=From,Date + + +; If the date header is present, and future and/or I are +; defined, it will be validated. +date_future_days=2 +date_past_days=15 + + +; Headers that MUST be unique if present (RFC 5322) +; singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject (RFC 5322) +singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject + + diff --git a/config/data.headers.ini b/config/data.headers.ini new file mode 100644 index 000000000..7d2c880b0 --- /dev/null +++ b/config/data.headers.ini @@ -0,0 +1,25 @@ +; configuration for data.headers plugin + +; Requiring a date header will cause the loss of valid mail. The JavaMail +; sender used by some banks, photo processing services, health insurance +; companies, bounce senders, and others send messages without a Date header. +; +; If you can afford to reject some valid mail, please do enforce this, and +; encourage mailers toward RFC adherence. Otherwise, do not require Date. + +; Headers that MUST be present (RFC 5322) +; required=From,Date ; <-- RFC 5322 compliant +required=From,Date + + +; If the date header is present, and future and/or I are +; defined, it will be validated. +date_future_days=2 +date_past_days=15 + + +; Headers that MUST be unique if present (RFC 5322) +; singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject (RFC 5322) +singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject + + From 3b116dcb5a134efe52924c2f9de030b0fed300b0 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 18:07:34 -0500 Subject: [PATCH 032/160] removed inadvertent commit of config/data.headers (missing .ini) --- config/data.headers | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 config/data.headers diff --git a/config/data.headers b/config/data.headers deleted file mode 100644 index 7d2c880b0..000000000 --- a/config/data.headers +++ /dev/null @@ -1,25 +0,0 @@ -; configuration for data.headers plugin - -; Requiring a date header will cause the loss of valid mail. The JavaMail -; sender used by some banks, photo processing services, health insurance -; companies, bounce senders, and others send messages without a Date header. -; -; If you can afford to reject some valid mail, please do enforce this, and -; encourage mailers toward RFC adherence. Otherwise, do not require Date. - -; Headers that MUST be present (RFC 5322) -; required=From,Date ; <-- RFC 5322 compliant -required=From,Date - - -; If the date header is present, and future and/or I are -; defined, it will be validated. -date_future_days=2 -date_past_days=15 - - -; Headers that MUST be unique if present (RFC 5322) -; singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject (RFC 5322) -singular=Date,From,Sender,Reply-To,To,Cc,Bcc,Message-Id,In-Reply-To,References,Subject - - From c320541a171da53fdebf26e7caea4c08838e665e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 24 Dec 2013 18:10:06 -0500 Subject: [PATCH 033/160] renamed rfc5322_header_checks -> headers does more than RFC 5322 checks now, and is now configurable to do less as well (in case you want to use the plugin, but allow common misconfigurations, like the lack of a Date header) --- plugins/{data.rfc5322_header_checks.js => data.headers.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/{data.rfc5322_header_checks.js => data.headers.js} (100%) diff --git a/plugins/data.rfc5322_header_checks.js b/plugins/data.headers.js similarity index 100% rename from plugins/data.rfc5322_header_checks.js rename to plugins/data.headers.js From 6fa0e7f6d498db168df0648e181017450e9f9665 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 16:51:54 -0500 Subject: [PATCH 034/160] headers: added Date validity tests, updated UPGRADE with deprecation notes data.headers.ini, added more setting descriptions updated docs/plugins/data.headers.md Conflicts: UPGRADE config/data.headers.ini --- config/data.headers.ini | 11 ++-- docs/plugins/data.headers.md | 24 +++++++++ docs/plugins/data.rfc5322_header_checks.md | 10 ---- plugins/data.headers.js | 58 +++++++++++++++------- 4 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 docs/plugins/data.headers.md delete mode 100644 docs/plugins/data.rfc5322_header_checks.md diff --git a/config/data.headers.ini b/config/data.headers.ini index 7d2c880b0..ba89e337d 100644 --- a/config/data.headers.ini +++ b/config/data.headers.ini @@ -11,9 +11,14 @@ ; required=From,Date ; <-- RFC 5322 compliant required=From,Date - -; If the date header is present, and future and/or I are -; defined, it will be validated. +; Received +; If you have no outbound, add 'Received' to the required list for an +; aggressive anti-spam measure. It works because all real mail relays will +; add a `Received` header. It may false positive on some bulk mail that +; uses a custom tool to send, but this appears to be fairly rare. + +; If the date header is present, and future and/or past days are +; defined, it will be validated. 0 = disabled date_future_days=2 date_past_days=15 diff --git a/docs/plugins/data.headers.md b/docs/plugins/data.headers.md new file mode 100644 index 000000000..8ae0f844e --- /dev/null +++ b/docs/plugins/data.headers.md @@ -0,0 +1,24 @@ +data.headers +========================== + +This plugin performs sanity checks on mail headers. + +RFC 5322 Section 3.6: +--------------------- + +All messages MUST have a 'Date' and 'From' header and a message may not contain +more than one 'Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', 'Bcc', +'Message-Id', 'In-Reply-To', 'References' or 'Subject' header. + +The default list of required and singular headers can be customized in +config/data.headers.ini. + +Any message that does not meet the configured requirements will be rejected. + + +Date Validity +------------------- + +This plugin also tests the contents of the Date field, assuring that the +timestamps in the Date field are neither too old (default: 15 days), nor +too far in the future (default: 2 days). diff --git a/docs/plugins/data.rfc5322_header_checks.md b/docs/plugins/data.rfc5322_header_checks.md deleted file mode 100644 index 9d5ef2829..000000000 --- a/docs/plugins/data.rfc5322_header_checks.md +++ /dev/null @@ -1,10 +0,0 @@ -data.rfc5322_header_checks -========================== - -This plugin enforces RFC 5322 Section 3.6 which states that: - -All messages MUST have a 'Date' and 'From' header and a message may not contain -more than one 'Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', 'Bcc', -'Message-Id', 'In-Reply-To', 'References' or 'Subject' header. - -Any message that does not meet these requirements will be rejected. diff --git a/plugins/data.headers.js b/plugins/data.headers.js index 1443251e1..b81db51a5 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -7,26 +7,11 @@ var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', var date_future_days = 2; var date_past_days = 15; -exports.register = function() { - var config = this.config.get('data.headers.ini'); - - if ( config.main.required ) { - required_headers = config.main.required.split(','); - }; - if ( config.main.singular ) { - singular_headers = config.main.singular.split(','); - }; - - if ( config.main.date_future_days ) { - date_future_days = config.main.date_future_days; - } - if ( config.main.date_past_days ) { - date_past_days = config.main.date_past_days; - } -} - exports.hook_data_post = function (next, connection) { + refreshConfig(this); + var header = connection.transaction.header; + // Headers that MUST be present for (var i=0,l=required_headers.length; i < l; i++) { if (header.get_all(required_headers[i]).length === 0) @@ -44,5 +29,42 @@ exports.hook_data_post = function (next, connection) { } } + var msg_date = header.get_all('Date'); + if ( msg_date.length > 0 ) { + this.logdebug(connection, "message date: " + msg_date); + var msg_secs = Date.parse(msg_date); + this.logdebug(connection, "parsed date: " + msg_secs); + var now_secs = Date.now(); + this.logdebug(connection, "now seconds: " + now_secs); + + if ( date_future_days > 0 && msg_secs > (now_secs + (date_future_days * 24 * 3600)) ) { + this.loginfo(connection, "date too far in the future: " + msg_date ); + return next(DENY, "The Date header is too far in the future"); + } + if ( date_past_days > 0 && msg_secs < (now_secs - ( date_past_days * 24 * 3600 )) ) { + this.loginfo(connection, "date too old: " + msg_date ); + return next(DENY, "The Date header is too old"); + }; + }; + return next(); } + +function refreshConfig(plugin) { + var config = plugin.config.get('data.headers.ini'); + + if ( config.main.required !== 'undefined' ) { + required_headers = config.main.required.split(','); + }; + if ( config.main.singular !== 'undefined' ) { + singular_headers = config.main.singular.split(','); + }; + + if ( config.main.date_future_days !== 'undefined' ) { + date_future_days = config.main.date_future_days; + } + if ( config.main.date_past_days !== 'undefined' ) { + date_past_days = config.main.date_past_days; + } +} + From 4b53400e1f93336ce049a8c87e979570e8fe686b Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 25 Dec 2013 01:28:24 -0500 Subject: [PATCH 035/160] updated config/plugins for new data.headers --- config/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins b/config/plugins index 5405990d1..248871f27 100644 --- a/config/plugins +++ b/config/plugins @@ -7,7 +7,7 @@ dnsbl # Check mail headers are valid -data.rfc5322_header_checks +data.headers # block mail from some known bad HELOs - see config/helo.checks.ini for configuration helo.checks From a604da0b58fded6a228ccf19d6ace09e1311813a Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 17:07:15 -0500 Subject: [PATCH 036/160] restored rfc5322_header_checks --- plugins/data.rfc5322_header_checks.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 plugins/data.rfc5322_header_checks.js diff --git a/plugins/data.rfc5322_header_checks.js b/plugins/data.rfc5322_header_checks.js new file mode 100644 index 000000000..bc052ec0f --- /dev/null +++ b/plugins/data.rfc5322_header_checks.js @@ -0,0 +1,27 @@ +// Enforce RFC 5322 Section 3.6 +var required_headers = ['Date', 'From']; +var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', + 'Bcc', 'Message-Id', 'In-Reply-To', 'References', + 'Subject']; + +exports.hook_data_post = function (next, connection) { + var header = connection.transaction.header; + // Headers that MUST be present + for (var i=0,l=required_headers.length; i < l; i++) { + if (header.get_all(required_headers[i]).length === 0) + { + return next(DENY, "Required header '" + required_headers[i] + + "' missing"); + } + } + + // Headers that MUST be unique if present + for (var i=0,l=singular_headers.length; i < l; i++) { + if (header.get_all(singular_headers[i]).length > 1) { + return next(DENY, "Message contains non-unique '" + + singular_headers[i] + "' header"); + } + } + + return next(); +} From 7c08e0fba831623bdb760f89334e4907baf316af Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 17:16:04 -0500 Subject: [PATCH 037/160] add deprecated notice within legacy plugins --- UPGRADE | 9 +++++++++ plugins/data.nomsgid.js | 6 +++++- plugins/data.noreceived.js | 6 +++++- plugins/data.rfc5322_header_checks.js | 8 ++++++-- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 UPGRADE diff --git a/UPGRADE b/UPGRADE new file mode 100644 index 000000000..0b523cccc --- /dev/null +++ b/UPGRADE @@ -0,0 +1,9 @@ + +2013.12.27 + +new plugin: data.headers + + deprecates data.rfc5322_header_checks.js + deprecates data.noreceived.js + deprecates data.nomsgid.js + diff --git a/plugins/data.nomsgid.js b/plugins/data.nomsgid.js index 6acb83e58..8655e5d92 100644 --- a/plugins/data.nomsgid.js +++ b/plugins/data.nomsgid.js @@ -1,5 +1,9 @@ // Check whether an email has a Message-Id header or not, and reject if not +exports.register = function () { + this.logwarn("NOTICE: plugin deprecated, use 'headers' instead!"); +} + exports.hook_data_post = function (next, connection) { if (connection.transaction.header.get_all('Message-Id').length === 0) { next(DENY, "Mails here must have a Message-Id header"); @@ -7,4 +11,4 @@ exports.hook_data_post = function (next, connection) { else { next(); } -} \ No newline at end of file +} diff --git a/plugins/data.noreceived.js b/plugins/data.noreceived.js index a746f2d64..174656db2 100644 --- a/plugins/data.noreceived.js +++ b/plugins/data.noreceived.js @@ -3,6 +3,10 @@ // NB: Don't check this on your outbounds. It's also a pretty strict check // for inbounds too, so use with caution. +exports.register = function () { + this.logwarn("NOTICE: plugin deprecated, use 'headers' instead!"); +} + exports.hook_data_post = function (next, connection) { // We always have the received header that Haraka added, so check for 1 if (connection.transaction.header.get_all('Received').length === 1) { @@ -11,4 +15,4 @@ exports.hook_data_post = function (next, connection) { else { next(); } -} \ No newline at end of file +} diff --git a/plugins/data.rfc5322_header_checks.js b/plugins/data.rfc5322_header_checks.js index bc052ec0f..3263a07d7 100644 --- a/plugins/data.rfc5322_header_checks.js +++ b/plugins/data.rfc5322_header_checks.js @@ -1,16 +1,20 @@ -// Enforce RFC 5322 Section 3.6 +// Enforce RFC 5322 Section 3.6 var required_headers = ['Date', 'From']; var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', 'Bcc', 'Message-Id', 'In-Reply-To', 'References', 'Subject']; +exports.register = function () { + this.logwarn("NOTICE: plugin deprecated, use 'data.headers' instead!"); +} + exports.hook_data_post = function (next, connection) { var header = connection.transaction.header; // Headers that MUST be present for (var i=0,l=required_headers.length; i < l; i++) { if (header.get_all(required_headers[i]).length === 0) { - return next(DENY, "Required header '" + required_headers[i] + + return next(DENY, "Required header '" + required_headers[i] + "' missing"); } } From d261ae90187ff4a8a37f7f73f1cbed0fe87ca0df Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 17:20:39 -0500 Subject: [PATCH 038/160] rdns.regexp: added startup deprecation warning --- plugins/rdns.regexp.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/rdns.regexp.js b/plugins/rdns.regexp.js index 283a78cec..1268eaceb 100644 --- a/plugins/rdns.regexp.js +++ b/plugins/rdns.regexp.js @@ -5,6 +5,9 @@ // to using the new connect.rdns_access plugin, as this plugin is now deprecated // and may be removed in a future version of Haraka. +exports.register = function () { + this.logwarn("NOTICE: deprecated, use 'connect.rdns_access' instead!"); +} exports.hook_connect = function (next, connection) { var deny_list = this.config.get('rdns.deny_regexps', 'list'); From b6be6662e97f4300fa889216134afb03d1643975 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 17:23:10 -0500 Subject: [PATCH 039/160] data.no*, added data. prefix to warning message --- plugins/data.nomsgid.js | 2 +- plugins/data.noreceived.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/data.nomsgid.js b/plugins/data.nomsgid.js index 8655e5d92..d1062ba30 100644 --- a/plugins/data.nomsgid.js +++ b/plugins/data.nomsgid.js @@ -1,7 +1,7 @@ // Check whether an email has a Message-Id header or not, and reject if not exports.register = function () { - this.logwarn("NOTICE: plugin deprecated, use 'headers' instead!"); + this.logwarn("NOTICE: plugin deprecated, use 'data.headers' instead!"); } exports.hook_data_post = function (next, connection) { diff --git a/plugins/data.noreceived.js b/plugins/data.noreceived.js index 174656db2..a01cff584 100644 --- a/plugins/data.noreceived.js +++ b/plugins/data.noreceived.js @@ -4,7 +4,7 @@ // for inbounds too, so use with caution. exports.register = function () { - this.logwarn("NOTICE: plugin deprecated, use 'headers' instead!"); + this.logwarn("NOTICE: plugin deprecated, use 'data.headers' instead!"); } exports.hook_data_post = function (next, connection) { From 90126a0982a1a6a2067ccc85eac66aeba817898e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 18:02:04 -0500 Subject: [PATCH 040/160] restored data.rfc5322_header_checks.md docs/plugins/data.rfc5322_header_checks.md --- docs/plugins/data.rfc5322_header_checks.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/plugins/data.rfc5322_header_checks.md diff --git a/docs/plugins/data.rfc5322_header_checks.md b/docs/plugins/data.rfc5322_header_checks.md new file mode 100644 index 000000000..9d5ef2829 --- /dev/null +++ b/docs/plugins/data.rfc5322_header_checks.md @@ -0,0 +1,10 @@ +data.rfc5322_header_checks +========================== + +This plugin enforces RFC 5322 Section 3.6 which states that: + +All messages MUST have a 'Date' and 'From' header and a message may not contain +more than one 'Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', 'Bcc', +'Message-Id', 'In-Reply-To', 'References' or 'Subject' header. + +Any message that does not meet these requirements will be rejected. From 117459da712da63385ff94ce6129fc6d453f4ba3 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 18:04:46 -0500 Subject: [PATCH 041/160] added deprecated notice in plugin docs --- docs/plugins/data.nomsgid.md | 2 ++ docs/plugins/data.noreceived.md | 2 ++ docs/plugins/data.rfc5322_header_checks.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docs/plugins/data.nomsgid.md b/docs/plugins/data.nomsgid.md index 426e14ac7..05ff1447e 100644 --- a/docs/plugins/data.nomsgid.md +++ b/docs/plugins/data.nomsgid.md @@ -1,6 +1,8 @@ data.nomsgid ============ +NOTICE: this plugin is deprecated. Use data.headers instead. + Quite simply enabling this plugin blocks all mails lacking a Message-Id header. This is an aggressive anti-spam measure, but since most mail systems will add a Message-Id header, it tends to block a good chunk of abusive mail. diff --git a/docs/plugins/data.noreceived.md b/docs/plugins/data.noreceived.md index 6d347314b..e0946fee8 100644 --- a/docs/plugins/data.noreceived.md +++ b/docs/plugins/data.noreceived.md @@ -1,6 +1,8 @@ data.noreceived =============== +NOTICE: this plugin is deprecated. Use data.headers instead. + This plugin very simply blocks any mail arriving at your system that has no `Received` headers. diff --git a/docs/plugins/data.rfc5322_header_checks.md b/docs/plugins/data.rfc5322_header_checks.md index 9d5ef2829..76d33261f 100644 --- a/docs/plugins/data.rfc5322_header_checks.md +++ b/docs/plugins/data.rfc5322_header_checks.md @@ -1,6 +1,8 @@ data.rfc5322_header_checks ========================== +NOTICE: this plugin is deprecated. Use data.headers instead. + This plugin enforces RFC 5322 Section 3.6 which states that: All messages MUST have a 'Date' and 'From' header and a message may not contain From b25292257dd5507b5e0dd908c2e8a8717a8f3b91 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 18:14:26 -0500 Subject: [PATCH 042/160] docs/CoreConfig: added rfc_1869 note add discussed in issue #404 --- docs/CoreConfig.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/CoreConfig.md b/docs/CoreConfig.md index b4e60685b..1c6a76260 100644 --- a/docs/CoreConfig.md +++ b/docs/CoreConfig.md @@ -135,3 +135,13 @@ different levels available. A list of HAProxy hosts that Haraka should enable the PROXY protocol from. See HAProxy.md + +* strict_rfc1869 + + When enabled, this setting requires senders to conform to RFC 1869 and + RFC 821 when sending the MAIL FROM and RCPT TO commands. In particular, + the inclusion of spurious spaces or missing angle brackets will be rejected. + + to enable: echo '1' > /path/to/haraka/config/strict_rfc1869 + to disable: echo '0' > /path/to/haraka/config/strict_rfc1869 + From 3a7e96d29946dfefcce7ed9934dc1628d8e53107 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 25 Dec 2013 02:33:37 -0500 Subject: [PATCH 043/160] dkim_sign: added config/dkim/+domain support --- plugins/dkim_sign.js | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index d389663e0..c9719b51a 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -87,7 +87,7 @@ DKIMSignStream.prototype.end = function (buf) { this.buffer = { ar: [], len: 0 }; } - var bodyhash = this.hash.digest('base64'); + var bodyhash = this.hash.digest('base64'); /* ** HEADERS (relaxed canonicaliztion) @@ -132,20 +132,34 @@ exports.DKIMSignStream = DKIMSignStream; exports.hook_queue_outbound = function (next, connection) { var self = this; var transaction = connection.transaction; - var config = this.config.get('dkim_sign.ini'); - var private_key = this.config.get('dkim.private.key','data').join("\n"); + + var private_key; var headers_to_sign = []; + var domain; + var selector; + + var keydir = get_keydir( "config/dkim/" + domain ); + if ( -d keydir ) { + domain = connection.transaction.mail_from.host(); + private_key = this.config.get(keydir+'private', 'data').join("\n"); + selector = this.config.get(keydir+'selector','data').join("\n"); + } + else { + var config = this.config.get('dkim_sign.ini'); + private_key = this.config.get('dkim.private.key','data').join("\n"); + selector = config.main.selector; + }; // Make sure we have all the relevant configuration if (!private_key) { - connection.logerror(this, 'skipped: missing dkim.private.key'); + connection.logerror(this, 'skipped: missing dkim private key'); return next(); } if (config.main.disabled && /(?:1|true|y[es])/i.test(config.main.disabled)) { connection.logerror(this, 'skipped: disabled'); return next(); } - if (!config.main.selector) { + if (!selector) { connection.logerror(this, 'skipped: missing selector'); return next(); } @@ -164,12 +178,12 @@ exports.hook_queue_outbound = function (next, connection) { headers_to_sign.push('from'); } - var dkim_sign = new DKIMSignStream(config.main.selector, - config.main.domain, - private_key, - headers_to_sign, - transaction.header, - function (err, dkim_header) + var dkim_sign = new DKIMSignStream(selector, + domain, + private_key, + headers_to_sign, + transaction.header, + function (err, dkim_header) { if (err) { connection.logerror(self, err.message); @@ -179,6 +193,6 @@ exports.hook_queue_outbound = function (next, connection) { transaction.add_header('DKIM-Signature', dkim_header); } return next(); - }); + }); transaction.message_stream.pipe(dkim_sign); } From 7f21d847f0903cc04ccf33f961ca39500e932c75 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 25 Dec 2013 04:07:47 -0500 Subject: [PATCH 044/160] dkim: completed per-domain dkim key use --- config/dkim/dkim_key_gen.sh | 64 +++++++++++++++++++++++++++++++++++++ plugins/dkim_sign.js | 30 ++++++++++++----- 2 files changed, 86 insertions(+), 8 deletions(-) create mode 100755 config/dkim/dkim_key_gen.sh diff --git a/config/dkim/dkim_key_gen.sh b/config/dkim/dkim_key_gen.sh new file mode 100755 index 000000000..9fcecb8b7 --- /dev/null +++ b/config/dkim/dkim_key_gen.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +usage() { + echo " usage: $0 [haraka username]" + echo " " + exit +} + +if [ -z $1 ]; +then + usage +fi + +DOMAIN=$1 +SMTPD=$2 +if [ -z $SMTPD ]; +then + SMTPD="www" +fi + +# create a directory for each DKIM signing domain +mkdir -p $DOMAIN +cd $DOMAIN + +# create a selector in the format mmmYYYY (apr2014) +date '+%h%Y' | tr "[:upper:]" "[:lower:]" > selector + +# generate private and public keys +openssl genrsa -out private 2048 +chmod 400 private +openssl rsa -in private -out public -pubout + +# make it really easy to publish the public key in DNS +cat > dns < Date: Thu, 26 Dec 2013 19:47:28 -0500 Subject: [PATCH 045/160] update DKIM doc.md, add comments in 'dns' file --- config/dkim/dkim_key_gen.sh | 10 +++- docs/plugins/dkim_sign.md | 91 +++++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 35 deletions(-) diff --git a/config/dkim/dkim_key_gen.sh b/config/dkim/dkim_key_gen.sh index 9fcecb8b7..55a114500 100755 --- a/config/dkim/dkim_key_gen.sh +++ b/config/dkim/dkim_key_gen.sh @@ -22,15 +22,21 @@ fi mkdir -p $DOMAIN cd $DOMAIN -# create a selector in the format mmmYYYY (apr2014) +# The selector can be any value that is a valid DNS label +# create in the common format: mmmYYYY (apr2014) date '+%h%Y' | tr "[:upper:]" "[:lower:]" > selector # generate private and public keys +# key length considerations +# The minimum recommended key length for short duration keys (ones that +# will be replaced within a few months) is 1024. If you are unlikely to +# rotate your keys frequently, choose 2048, at the expense of more CPU. openssl genrsa -out private 2048 chmod 400 private openssl rsa -in private -out public -pubout # make it really easy to publish the public key in DNS +# by creating a file named 'dns', with instructions cat > dns < dkim.public.key + cd /path/to/haraka/config/dkim + ./dkim_key_gen.sh example.org -A selector is used to identify the keys used to attach a token to a -piece of email. It does appear in the header of the email sent, but -isn’t otherwise visible or meaningful to the final recipient. Any time -you generate a new key pair you need to choose a new selector. +Peek into the dkim_key_gen.sh shell script to see the commands used to +create and format the DKIM public key. Within the config/dkim/example.org + directory will be 4 files: -A selector is a string of no more than 63 lower-case alphanumeric -characters (a-z or 0-9) followed by a period “.”, followed by another -string of no more than 63 lower-case alphanumeric characters. + % ls config/dkim/example.org/ + dns private public selector -Next you have to publish the public key as a DNS TXT record for your -domain by concatenating the selector, the literal string ._domainkey. -and your domain name. e.g. mail._domainkey.example.com +The`_private` and `public` files contain the DKIM keys, the selector is +in the `selector` file and the `dns` file contains a formatted record of +the public key, as well as suggestions for DKIM, SPF, and DMARC policy +records. The records in `dns` are ready to be copy/pasted into the DNS +zone for example.org. -The content of the TXT record can be created by concatenating the -literal string “v=DKIM1;t=s;n=core;p=” and the public key excluding -the ---BEGIN and ---END lines and wrapping the key into a single line. +The DKIM DNS record will look like this: -See the key wizard at http://dkimtools.org/tools + may2013._domainkey TXT "v=DKIM1;p=[public key stripped of whitespace];" -Configuation +And the values in the address have the following meaning: + + hash: h=[ sha1 | sha256 ] + test; t=[ s | s:y ] + granularity: g=[ ] + notes: n=[ ] + services: s=[email] + keytypes: [ rsa ] + + +What to sign ------------ +The DKIM signing key for messages from example.org _should_ be signed with + a DKIM key for example.org. Failing to do so will result in messages not +having an *aligned* DKIM signature. For DMARC enabled domains, this will +likely result in deliverability problems. + +For correct alignment, Haraka signs each message with that domains DKIM key. +For an alternative, see the legacy Single Domain Configuration below. + + +Configuration +------------- + This plugin uses the configuration dkim_sign.ini in INI format. All configuration should appear within the 'main' block and is checked for updates on every run. - disabled = [ 1 | true | yes ] (OPTIONAL) - + Set this to disable DKIM signing -- selector = name (REQUIRED) +- headers_to_sign = list, of; headers (REQUIRED) + + Set this to the list of headers that should be signed + separated by either a comma, colon or semi-colon. + This is to prevent any tampering of the specified headers. + The 'From' header is required to be present by the RFC and + will be added if it is missing. + + +Single Domain Configuration +-------------------- + +To sign all messages with a single DKIM key, use these config settings. + +- selector = name Set this to the selector name published in DNS under the _domainkey sub-domain of the domain referenced below. -- domain = name (REQUIRED) +- domain = name (OPTIONAL) + + Set this to the domain name that will be used to sign messages + which don't match a per-domain DKIM key. The DNS TXT entry for: - Set this to the domain name that will be used to sign the - message. The DNS TXT entry for: - ._domainkey. MUST be present, otherwise remote systems will not be able to validate the signature applied to the message. -- headers_to_sign = list, of; headers (REQUIRED) - - Set this to the list of headers that should be signed - separated by either a comma, colon or semi-colon. - This is to prevent any tampering of the specified headers. - The 'From' header is required to be present by the RFC and - will be added if it is missing. From cbb58c4b190b169e4f8ce5cc70e48d894a6e29d4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 15:27:48 -0500 Subject: [PATCH 046/160] dkim: check for higher level DKIM keys when signing for example, when a message is being sent from mail.example.com, also check for an example.com signing key. --- plugins/dkim_sign.js | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index ce1578e9a..ac832f214 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -136,10 +136,11 @@ exports.hook_queue_outbound = function (next, connection) { var private_key; var headers_to_sign = []; - var domain = transaction.mail_from.host; var selector; var config = this.config.get('dkim_sign.ini'); - var keydir = get_keydir( domain ); + var stuff = get_keydir(self, connection); + var domain = stuff[0]; + var keydir = stuff[1]; connection.logdebug(this, 'dkim_keydir: '+keydir); @@ -204,9 +205,36 @@ exports.hook_queue_outbound = function (next, connection) { transaction.message_stream.pipe(dkim_sign); } -function get_keydir(domain) { - // TODO: check for existence of keydir - // If it doesn't exist, break the domain into labels and check for - // a higher level match (ie, match example.com for mail.example.com). - return "config/dkim/" + domain; +function get_keydir(plugin, conn) { + + // is there a better way to find this? + var haraka_dir = process.argv[3]; + + // TODO: the DKIM signing key should be aligned with the domain + // in the From header, so we *should* parse the domain from there. + // However, the From header can contain multiple addresses and should be + // parsed as described in RFC 2822 3.6.2. If From has multiple-addresses, + // then we must parse and use the domain in the Sender header. + // var domain = self.header.get('from').host; + + // In all cases I have seen, but likely not all cases, this suffices + var domain = conn.transaction.mail_from.host; + + // split the domain name into labels + var labels = domain.split('.'); + + // find the most specific match (ex: mail.example.com, example.com, com) + for ( var i=0; i Date: Fri, 27 Dec 2013 15:29:14 -0500 Subject: [PATCH 047/160] dkim docs: formatting fixes --- docs/plugins/dkim_sign.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/dkim_sign.md b/docs/plugins/dkim_sign.md index 51ca9c868..3bdf0023c 100644 --- a/docs/plugins/dkim_sign.md +++ b/docs/plugins/dkim_sign.md @@ -24,7 +24,7 @@ create and format the DKIM public key. Within the config/dkim/example.org % ls config/dkim/example.org/ dns private public selector -The`_private` and `public` files contain the DKIM keys, the selector is +The`private` and `public` files contain the DKIM keys, the selector is in the `selector` file and the `dns` file contains a formatted record of the public key, as well as suggestions for DKIM, SPF, and DMARC policy records. The records in `dns` are ready to be copy/pasted into the DNS @@ -47,7 +47,7 @@ And the values in the address have the following meaning: What to sign ------------ -The DKIM signing key for messages from example.org _should_ be signed with +The DKIM signing key for messages from example.org *should* be signed with a DKIM key for example.org. Failing to do so will result in messages not having an *aligned* DKIM signature. For DMARC enabled domains, this will likely result in deliverability problems. From f4ea302749f8cb03662424fe1264c658418431a9 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 27 Dec 2013 15:34:44 -0500 Subject: [PATCH 048/160] dkim docs: clarify single use config --- docs/plugins/dkim_sign.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/plugins/dkim_sign.md b/docs/plugins/dkim_sign.md index 3bdf0023c..2435f10c8 100644 --- a/docs/plugins/dkim_sign.md +++ b/docs/plugins/dkim_sign.md @@ -67,7 +67,7 @@ checked for updates on every run. Set this to disable DKIM signing -- headers_to_sign = list, of; headers (REQUIRED) +- headers_to_sign = list, of; headers (REQUIRED) Set this to the list of headers that should be signed separated by either a comma, colon or semi-colon. @@ -79,20 +79,18 @@ checked for updates on every run. Single Domain Configuration -------------------- -To sign all messages with a single DKIM key, use these config settings. +To sign all messages with a single DKIM key, these two config settings +are required. - selector = name Set this to the selector name published in DNS under the _domainkey sub-domain of the domain referenced below. -- domain = name (OPTIONAL) +- domain = name Set this to the domain name that will be used to sign messages which don't match a per-domain DKIM key. The DNS TXT entry for: ._domainkey. - MUST be present, otherwise remote systems will not be able - to validate the signature applied to the message. - From 66c68aec9c0caa32fa5ee44495a34d591f382858 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 28 Dec 2013 03:25:10 -0500 Subject: [PATCH 049/160] dkim: removed sync fs calls --- plugins/dkim_sign.js | 196 +++++++++++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 54 deletions(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index ac832f214..132909eab 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -5,7 +5,8 @@ var crypto = require('crypto'); var Stream = require('stream').Stream; var fs = require('fs'); var indexOfLF = require('./utils').indexOfLF; -var util = require('util'); +var util = require('util'); +var async = require('async'); function DKIMSignStream(selector, domain, private_key, headers_to_sign, header, end_callback) { Stream.call(this); @@ -131,61 +132,78 @@ DKIMSignStream.prototype.destroy = function () { exports.DKIMSignStream = DKIMSignStream; exports.hook_queue_outbound = function (next, connection) { - var self = this; - var transaction = connection.transaction; + var plugin = this; + if ( !isEnabled(plugin) ) return next(); + + getKeyDirAsync(plugin,connection,function(keydir) { + var domain, selector, private_key; + var dkconf = plugin.config.get('dkim_sign.ini'); + if ( ! keydir ) { + domain = dkconf.main.domain; + private_key = plugin.config.get('dkim.private.key','data').join("\n"); + selector = dkconf.main.selector; + } + else { + domain = keydir.split('/').pop(); + connection.logdebug(plugin, 'dkim_domain: '+domain); + private_key = plugin.config.get('dkim/'+domain+'/private', 'data').join("\n"); + selector = plugin.config.get('dkim/'+domain+'/selector','data').join("\n"); + } - var private_key; - var headers_to_sign = []; - var selector; - var config = this.config.get('dkim_sign.ini'); - var stuff = get_keydir(self, connection); - var domain = stuff[0]; - var keydir = stuff[1]; + if ( ! hasKeyData(plugin,connection,domain,selector,private_key) ) { + return next(); + }; + + var headers_to_sign = getHeadersToSign(dkconf); + var transaction = connection.transaction; + var dkim_sign = new DKIMSignStream(selector, + domain, + private_key, + headers_to_sign, + transaction.header, + function (err, dkim_header) + { + if (err) { + connection.logerror(plugin, err.message); + } + else { + connection.loginfo(plugin, dkim_header); + transaction.add_header('DKIM-Signature', dkim_header); + } + return next(); + }); + transaction.message_stream.pipe(dkim_sign); + }); +}; +/* +exports.hook_queue_outbound = function (next, connection) { + var plugin = this; + if ( !isEnabled(plugin) ) return next(); + var keydir = get_keydir(plugin, connection); connection.logdebug(this, 'dkim_keydir: '+keydir); - if ( fs.existsSync(keydir) ) { - private_key = this.config.get('dkim/'+domain+'/private', 'data').join("\n"); - selector = this.config.get('dkim/'+domain+'/selector','data').join("\n"); + var domain, selector, private_key; + var dkconf = plugin.config.get('dkim_sign.ini'); + + if ( keydir === false ) { + domain = dkconf.main.domain; + private_key = this.config.get('dkim.private.key','data').join("\n"); + selector = dkconf.main.selector; } else { - domain = config.main.domain; - private_key = this.config.get('dkim.private.key','data').join("\n"); - selector = config.main.selector; + domain = keydir.split('/').pop(); + connection.logdebug(this, 'dkim_domain: '+domain); + private_key = this.config.get('dkim/'+domain+'/private', 'data').join("\n"); + selector = this.config.get('dkim/'+domain+'/selector','data').join("\n"); }; - // Make sure we have all the relevant configuration - if (!private_key) { - connection.logerror(this, 'skipped: missing dkim private key'); - return next(); - } - if (config.main.disabled && /(?:1|true|y[es])/i.test(config.main.disabled)) { - connection.logerror(this, 'skipped: disabled'); - return next(); - } - if (!selector) { - connection.logerror(this, 'skipped: missing selector'); + if ( ! hasKeyData(plugin,connection,domain,selector,private_key) ) { return next(); - } - if (!domain) { - connection.logerror(this, 'skipped: missing domain'); - return next(); - } - - connection.logprotocol(this, 'private_key: '+private_key); - connection.logprotocol(this, 'selector: '+selector); - - if (config.main.headers_to_sign) { - headers_to_sign = config.main.headers_to_sign - .toLowerCase() - .replace(/\s+/g,'') - .split(/[,;:]/); - } - // From MUST be present - if (headers_to_sign.indexOf('from') === -1) { - headers_to_sign.push('from'); - } + }; + var headers_to_sign = getHeadersToSign(dkconf); + var transaction = connection.transaction; var dkim_sign = new DKIMSignStream(selector, domain, private_key, @@ -194,21 +212,19 @@ exports.hook_queue_outbound = function (next, connection) { function (err, dkim_header) { if (err) { - connection.logerror(self, err.message); + connection.logerror(plugin, err.message); } else { - connection.loginfo(self, dkim_header); + connection.loginfo(plugin, dkim_header); transaction.add_header('DKIM-Signature', dkim_header); } return next(); }); transaction.message_stream.pipe(dkim_sign); } - +*/ function get_keydir(plugin, conn) { - - // is there a better way to find this? - var haraka_dir = process.argv[3]; + var haraka_dir = process.env.HARAKA; // TODO: the DKIM signing key should be aligned with the domain // in the From header, so we *should* parse the domain from there. @@ -230,11 +246,83 @@ function get_keydir(plugin, conn) { var keydir = haraka_dir + "/config/dkim/"+hld; if ( fs.existsSync(keydir) ) { plugin.loginfo(conn, "found key dir: "+keydir); - return [hld,keydir]; + return keydir; }; plugin.logdebug(conn, "missing key dir: "+keydir); } plugin.loginfo(conn, "no key dir for "+domain+" found"); - return [domain, haraka_dir+"/config/dkim/"+domain]; + return false; +} + +function getKeyDirAsync(plugin, conn, cb) { + + var haraka_dir = process.env.HARAKA; + var domain = conn.transaction.mail_from.host; + var labels = domain.split('.'); + + // list possible matches (ex: mail.example.com, example.com, com) + var dom_hier = new Array; + for ( var i=0; i Date: Sun, 29 Dec 2013 00:21:41 -0500 Subject: [PATCH 050/160] dkim: replaced Array() with [] --- plugins/dkim_sign.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index 132909eab..81631ecff 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -262,7 +262,7 @@ function getKeyDirAsync(plugin, conn, cb) { var labels = domain.split('.'); // list possible matches (ex: mail.example.com, example.com, com) - var dom_hier = new Array; + var dom_hier = []; for ( var i=0; i Date: Sun, 29 Dec 2013 00:28:19 -0500 Subject: [PATCH 051/160] removed fs.sync methods --- plugins/dkim_sign.js | 76 +++----------------------------------------- 1 file changed, 4 insertions(+), 72 deletions(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index 81631ecff..f15e915cd 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -135,7 +135,7 @@ exports.hook_queue_outbound = function (next, connection) { var plugin = this; if ( !isEnabled(plugin) ) return next(); - getKeyDirAsync(plugin,connection,function(keydir) { + getKeyDir(plugin,connection,function(keydir) { var domain, selector, private_key; var dkconf = plugin.config.get('dkim_sign.ini'); if ( ! keydir ) { @@ -175,58 +175,12 @@ exports.hook_queue_outbound = function (next, connection) { transaction.message_stream.pipe(dkim_sign); }); }; -/* -exports.hook_queue_outbound = function (next, connection) { - var plugin = this; - if ( !isEnabled(plugin) ) return next(); - var keydir = get_keydir(plugin, connection); - connection.logdebug(this, 'dkim_keydir: '+keydir); +function getKeyDir(plugin, conn, cb) { - var domain, selector, private_key; - var dkconf = plugin.config.get('dkim_sign.ini'); - - if ( keydir === false ) { - domain = dkconf.main.domain; - private_key = this.config.get('dkim.private.key','data').join("\n"); - selector = dkconf.main.selector; - } - else { - domain = keydir.split('/').pop(); - connection.logdebug(this, 'dkim_domain: '+domain); - private_key = this.config.get('dkim/'+domain+'/private', 'data').join("\n"); - selector = this.config.get('dkim/'+domain+'/selector','data').join("\n"); - }; - - if ( ! hasKeyData(plugin,connection,domain,selector,private_key) ) { - return next(); - }; - - var headers_to_sign = getHeadersToSign(dkconf); - var transaction = connection.transaction; - var dkim_sign = new DKIMSignStream(selector, - domain, - private_key, - headers_to_sign, - transaction.header, - function (err, dkim_header) - { - if (err) { - connection.logerror(plugin, err.message); - } - else { - connection.loginfo(plugin, dkim_header); - transaction.add_header('DKIM-Signature', dkim_header); - } - return next(); - }); - transaction.message_stream.pipe(dkim_sign); -} -*/ -function get_keydir(plugin, conn) { var haraka_dir = process.env.HARAKA; - // TODO: the DKIM signing key should be aligned with the domain + // the DKIM signing key should be aligned with the domain // in the From header, so we *should* parse the domain from there. // However, the From header can contain multiple addresses and should be // parsed as described in RFC 2822 3.6.2. If From has multiple-addresses, @@ -239,28 +193,6 @@ function get_keydir(plugin, conn) { // split the domain name into labels var labels = domain.split('.'); - // find the most specific match (ex: mail.example.com, example.com, com) - for ( var i=0; i Date: Tue, 7 Jan 2014 20:36:55 -0500 Subject: [PATCH 052/160] dkim: remove extra checks --- plugins/dkim_sign.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index f15e915cd..c2e07896e 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -203,12 +203,6 @@ function getKeyDir(plugin, conn, cb) { async.filter(dom_hier, fs.exists, function(results) { plugin.logdebug(conn, results); - if ( !results ) { - cb(false); - }; - if ( typeof results === 'string' ) { - cb(results); - }; cb(results[0]); }); }; From 796ba4deed63a89063ce44cd0878d52f66994356 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 01:19:33 -0500 Subject: [PATCH 053/160] add Authentication-Results header (RFC 5451) --- connection.js | 9 +++++++++ plugins/auth/auth_base.js | 2 ++ 2 files changed, 11 insertions(+) diff --git a/connection.js b/connection.js index 83b35e120..b7702461f 100644 --- a/connection.js +++ b/connection.js @@ -142,6 +142,7 @@ function Connection(client, server) { this.greeting = null; this.hello_host = null; this.using_tls = false; + this.auth = 'none'; this.state = states.STATE_PAUSE; this.prev_state = null; this.loop_code = null; @@ -1232,6 +1233,13 @@ Connection.prototype.received_line = function() { ].join(''); }; +Connection.prototype.auth_results = function() { + // Implement RFC5451 + return [ config.get('me'), 'auth=' + this.auth, + (this.notes.authentication_results ? this.notes.authentication_results : ''), + ].join('; '); +}; + Connection.prototype.cmd_data = function(args) { // RFC 5321 Section 4.3.2 // DATA does not accept arguments @@ -1246,6 +1254,7 @@ Connection.prototype.cmd_data = function(args) { } this.accumulate_data('Received: ' + this.received_line() + "\r\n"); + this.accumulate_data('Authentication-Results: ' + this.auth_results() + "\r\n"); plugins.run_hooks('data', this); }; diff --git a/plugins/auth/auth_base.js b/plugins/auth/auth_base.js index d9921bd75..4013a8fdc 100644 --- a/plugins/auth/auth_base.js +++ b/plugins/auth/auth_base.js @@ -93,6 +93,7 @@ exports.check_user = function (next, connection, credentials, method) { connection.relaying = 1; connection.respond(235, "Authentication successful", function () { connection.authheader = "(authenticated bits=0)\n"; + connection.auth = 'pass'; connection.notes.auth_user = credentials[0]; return next(OK); }); @@ -104,6 +105,7 @@ exports.check_user = function (next, connection, credentials, method) { connection.notes.auth_fails++; var delay = Math.pow(2, connection.notes.auth_fails - 1); connection.lognotice(self, 'delaying response for ' + delay + ' seconds'); + connection.auth = 'fail'; setTimeout(function () { connection.respond(535, "Authentication failed", function () { connection.reset_transaction(); From 2eb28ccf270a43acb876b91519e4abf79925eb70 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 03:35:13 -0500 Subject: [PATCH 054/160] Auth-Res: added SPF results --- connection.js | 14 +++++++++----- plugins/auth/auth_base.js | 4 ++-- plugins/spf.js | 7 +++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/connection.js b/connection.js index b7702461f..3bc1c7d10 100644 --- a/connection.js +++ b/connection.js @@ -142,7 +142,6 @@ function Connection(client, server) { this.greeting = null; this.hello_host = null; this.using_tls = false; - this.auth = 'none'; this.state = states.STATE_PAUSE; this.prev_state = null; this.loop_code = null; @@ -1233,11 +1232,16 @@ Connection.prototype.received_line = function() { ].join(''); }; -Connection.prototype.auth_results = function() { +Connection.prototype.auth_results = function(message) { // Implement RFC5451 - return [ config.get('me'), 'auth=' + this.auth, - (this.notes.authentication_results ? this.notes.authentication_results : ''), - ].join('; '); + if ( ! this.notes.authentication_results ) { + this.notes.authentication_results = [ config.get('me') ]; + }; + if ( message ) { + this.notes.authentication_results.push(message); + }; + this.logdebug(this.notes.authentication_results); + return this.notes.authentication_results.join('; '); }; Connection.prototype.cmd_data = function(args) { diff --git a/plugins/auth/auth_base.js b/plugins/auth/auth_base.js index 4013a8fdc..6212a946c 100644 --- a/plugins/auth/auth_base.js +++ b/plugins/auth/auth_base.js @@ -93,7 +93,7 @@ exports.check_user = function (next, connection, credentials, method) { connection.relaying = 1; connection.respond(235, "Authentication successful", function () { connection.authheader = "(authenticated bits=0)\n"; - connection.auth = 'pass'; + connection.auth_results('auth=pass'); connection.notes.auth_user = credentials[0]; return next(OK); }); @@ -105,7 +105,7 @@ exports.check_user = function (next, connection, credentials, method) { connection.notes.auth_fails++; var delay = Math.pow(2, connection.notes.auth_fails - 1); connection.lognotice(self, 'delaying response for ' + delay + ' seconds'); - connection.auth = 'fail'; + connection.auth_results('auth=fail'); setTimeout(function () { connection.respond(535, "Authentication failed", function () { connection.reset_transaction(); diff --git a/plugins/spf.js b/plugins/spf.js index 8b5510cec..137b7db94 100644 --- a/plugins/spf.js +++ b/plugins/spf.js @@ -52,8 +52,10 @@ exports.hook_mail = function (next, connection, params) { var mfrom = params[0].address(); var host = params[0].host; var spf = new SPF(); + var auth_result; if (connection.notes.spf_helo) { + auth_result = spf.result(connection.notes.spf_helo).toLowerCase; // Add a trace header txn.add_leading_header('Received-SPF', spf.result(connection.notes.spf_helo) + @@ -72,6 +74,7 @@ exports.hook_mail = function (next, connection, params) { case spf.SPF_NONE: case spf.SPF_NEUTRAL: case spf.SPF_PASS: + connection.auth_results( "spf="+auth_result+" smtp.helo="+connection.hello_host); return next(); case spf.SPF_SOFTFAIL: if (cfg.main.helo_softfail_reject) { @@ -100,6 +103,7 @@ exports.hook_mail = function (next, connection, params) { default: // Unknown result connection.logerror(self, 'unknown result code=' + result); + connection.auth_results( "spf="+auth_result+" smtp.helo="+connection.hello_host); return next(); } } @@ -141,12 +145,14 @@ exports.hook_mail = function (next, connection, params) { 'helo=' + connection.hello_host, 'envelope-from=<' + mfrom + '>', ].join('; ')); + auth_result = spf.result(result).toLowerCase(); txn.notes.spf_mail_result = spf.result(result); txn.notes.spf_mail_record = spf.spf_record; switch (result) { case spf.SPF_NONE: case spf.SPF_NEUTRAL: case spf.SPF_PASS: + connection.auth_results( "spf="+auth_result+" smtp.mailfrom="+host); return next(); case spf.SPF_SOFTFAIL: if (cfg.main.mail_softfail_reject) { @@ -171,6 +177,7 @@ exports.hook_mail = function (next, connection, params) { default: // Unknown result connection.logerror(self, 'unknown result code=' + result); + connection.auth_results( "spf="+auth_result+" smtp.mailfrom="+host); return next(); } }); From 543963922b131832e3fbd97cf8fdc1b65bf6fe25 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 18:10:35 -0500 Subject: [PATCH 055/160] auth: added auth method to A-R header --- plugins/auth/auth_base.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/auth/auth_base.js b/plugins/auth/auth_base.js index 6212a946c..5ca24b128 100644 --- a/plugins/auth/auth_base.js +++ b/plugins/auth/auth_base.js @@ -93,7 +93,7 @@ exports.check_user = function (next, connection, credentials, method) { connection.relaying = 1; connection.respond(235, "Authentication successful", function () { connection.authheader = "(authenticated bits=0)\n"; - connection.auth_results('auth=pass'); + connection.auth_results('auth=pass ('+method.toLowerCase()+')' ); connection.notes.auth_user = credentials[0]; return next(OK); }); @@ -105,7 +105,8 @@ exports.check_user = function (next, connection, credentials, method) { connection.notes.auth_fails++; var delay = Math.pow(2, connection.notes.auth_fails - 1); connection.lognotice(self, 'delaying response for ' + delay + ' seconds'); - connection.auth_results('auth=fail'); + // here we include the username, as shown in RFC 5451 example + connection.auth_results('auth=fail ('+method.toLowerCase()+') smtp.auth='+ credentials[0]); setTimeout(function () { connection.respond(535, "Authentication failed", function () { connection.reset_transaction(); From e1e8bbcba0b9dcd9e1dd71c0872148ef8c93ae82 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 18:12:24 -0500 Subject: [PATCH 056/160] lookup_rdns: added iprev to Auth-Results header --- plugins/lookup_rdns.strict.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/lookup_rdns.strict.js b/plugins/lookup_rdns.strict.js index 5b22717ed..e0c1fd51f 100644 --- a/plugins/lookup_rdns.strict.js +++ b/plugins/lookup_rdns.strict.js @@ -91,6 +91,7 @@ exports.hook_lookup_rdns = function (next, connection) { if (!called_next) { called_next++; clearTimeout(timeout_id); + connection.auth_results("iprev=permerror"); if (_in_whitelist(connection, plugin, connection.remote_ip)) { next(OK, connection.remote_ip); @@ -124,6 +125,7 @@ exports.hook_lookup_rdns = function (next, connection) { if (_in_whitelist(connection, plugin, rdns)) { next(OK, rdns); } else { + connection.auth_results("iprev=fail"); _dns_error(connection, next, err, rdns, plugin, fwd_nxdomain, fwd_dnserror); } @@ -135,6 +137,7 @@ exports.hook_lookup_rdns = function (next, connection) { if (!called_next) { called_next++; clearTimeout(timeout_id); + connection.auth_results("iprev=pass"); return next(OK, rdns); } } From c3271bf3ad1cb676fdb61e829b8cb56245e9374b Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 20:20:08 -0500 Subject: [PATCH 057/160] connection: store auth results in transaction when a transaction is available. Else, default to connection --- connection.js | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/connection.js b/connection.js index 3bc1c7d10..5c4da8b48 100644 --- a/connection.js +++ b/connection.js @@ -1233,15 +1233,37 @@ Connection.prototype.received_line = function() { }; Connection.prototype.auth_results = function(message) { - // Implement RFC5451 + // http://tools.ietf.org/search/rfc7001 + var in_transaction = this.transaction && this.transaction.notes ? true : false; + + // initialize connection note if ( ! this.notes.authentication_results ) { this.notes.authentication_results = [ config.get('me') ]; - }; + } + + // initialize transaction note, if possible + if ( in_transaction === true && !this.transaction.notes.authentication_results ) { + this.transaction.notes.authentication_results = []; + } + + // if message, store it in the appropriate note if ( message ) { - this.notes.authentication_results.push(message); + if ( in_transaction === true ) { + this.transaction.notes.authentication_results.push(message); + this.logdebug("ar_tran: " + this.transaction.notes.authentication_results); + } + else { + this.notes.authentication_results.push(message); + this.logdebug("ar_conn: " + this.notes.authentication_results); + } }; - this.logdebug(this.notes.authentication_results); - return this.notes.authentication_results.join('; '); + + // return the formatted header + var header = [ this.notes.authentication_results.join('; '), + (in_transaction === true ? this.transaction.notes.authentication_results.join('; ') : '') + ].join('; '); + this.logdebug("ar_header: " + header); + return header; }; Connection.prototype.cmd_data = function(args) { @@ -1258,7 +1280,7 @@ Connection.prototype.cmd_data = function(args) { } this.accumulate_data('Received: ' + this.received_line() + "\r\n"); - this.accumulate_data('Authentication-Results: ' + this.auth_results() + "\r\n"); + this.transaction.add_header('Authentication-Results', this.auth_results() ); plugins.run_hooks('data', this); }; From 90c0445423ea4991c4e8dcc8d37d87be4bac3067 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 8 Jan 2014 20:52:57 -0500 Subject: [PATCH 058/160] move old Auth-Results to Original-Auth-Results --- connection.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/connection.js b/connection.js index 5c4da8b48..442e8971c 100644 --- a/connection.js +++ b/connection.js @@ -1266,6 +1266,18 @@ Connection.prototype.auth_results = function(message) { return header; }; +Connection.prototype.auth_results_clean = function(conn) { + // http://tools.ietf.org/html/draft-kucherawy-original-authres-00.html + var ars = conn.transaction.header.get_all('Authentication-Results'); + if ( ars.length === 0 ) { return; }; + + for (var i=0; i < ars.length; i++) { + conn.transaction.header.remove_header( ars[i] ); + conn.transaction.header.add_header('Original-Authentication-Results', ars[i] ); + } + conn.loginfo("Authentication-Results moved to Original-Authentication-Results" ); +}; + Connection.prototype.cmd_data = function(args) { // RFC 5321 Section 4.3.2 // DATA does not accept arguments @@ -1280,6 +1292,7 @@ Connection.prototype.cmd_data = function(args) { } this.accumulate_data('Received: ' + this.received_line() + "\r\n"); + this.auth_results_clean(this); this.transaction.add_header('Authentication-Results', this.auth_results() ); plugins.run_hooks('data', this); }; From e25faf28b1223e9bb5e8606115f19d72eff01bec Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Fri, 10 Jan 2014 23:21:44 +0000 Subject: [PATCH 059/160] If host is allowed by acl, don't deny the recipient because the domain isn't in the allow list --- plugins/relay_acl.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/relay_acl.js b/plugins/relay_acl.js index 47e78f4b6..54e6ed30c 100644 --- a/plugins/relay_acl.js +++ b/plugins/relay_acl.js @@ -22,6 +22,9 @@ exports.check_acl = function (next, connection, params) { }; exports.check_relay_domains = function (next, connection, params) { + // Skip this if the host is already allowed to relay + if (connection.relaying) return next(); + this.dest_domains_ini = this.config.get('relay_dest_domains.ini', 'ini'); var dest_domain = params[0].host; From 1e76fc45e9f7a373f0de7a67a48cdc06fbf0cce6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 19:07:24 -0500 Subject: [PATCH 060/160] spf: moved auth_results processing out of switch so that it's always added to auth_results previously, when SPF failed, it wasn't added. When combined with a delay_deny plugin, the connection would ultimately pass without SPF results saved --- plugins/spf.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/spf.js b/plugins/spf.js index 137b7db94..2f94039d1 100644 --- a/plugins/spf.js +++ b/plugins/spf.js @@ -70,11 +70,11 @@ exports.hook_mail = function (next, connection, params) { ].join('; ')); // Use the result from HELO if the return-path is null if (!host) { + connection.auth_results( "spf="+auth_result+" smtp.helo="+connection.hello_host); switch (connection.notes.spf_helo) { case spf.SPF_NONE: case spf.SPF_NEUTRAL: case spf.SPF_PASS: - connection.auth_results( "spf="+auth_result+" smtp.helo="+connection.hello_host); return next(); case spf.SPF_SOFTFAIL: if (cfg.main.helo_softfail_reject) { @@ -103,7 +103,6 @@ exports.hook_mail = function (next, connection, params) { default: // Unknown result connection.logerror(self, 'unknown result code=' + result); - connection.auth_results( "spf="+auth_result+" smtp.helo="+connection.hello_host); return next(); } } @@ -146,13 +145,13 @@ exports.hook_mail = function (next, connection, params) { 'envelope-from=<' + mfrom + '>', ].join('; ')); auth_result = spf.result(result).toLowerCase(); + connection.auth_results( "spf="+auth_result+" smtp.mailfrom="+host); txn.notes.spf_mail_result = spf.result(result); txn.notes.spf_mail_record = spf.spf_record; switch (result) { case spf.SPF_NONE: case spf.SPF_NEUTRAL: case spf.SPF_PASS: - connection.auth_results( "spf="+auth_result+" smtp.mailfrom="+host); return next(); case spf.SPF_SOFTFAIL: if (cfg.main.mail_softfail_reject) { @@ -177,7 +176,6 @@ exports.hook_mail = function (next, connection, params) { default: // Unknown result connection.logerror(self, 'unknown result code=' + result); - connection.auth_results( "spf="+auth_result+" smtp.mailfrom="+host); return next(); } }); From b14be3565420fd5094108e193168b5ec6db5ae8d Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 19:49:47 -0500 Subject: [PATCH 061/160] connection: made code more linear make it easier to read and grok --- connection.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/connection.js b/connection.js index 442e8971c..aa61c6563 100644 --- a/connection.js +++ b/connection.js @@ -1234,39 +1234,38 @@ Connection.prototype.received_line = function() { Connection.prototype.auth_results = function(message) { // http://tools.ietf.org/search/rfc7001 - var in_transaction = this.transaction && this.transaction.notes ? true : false; + var has_conn = this.notes.authentication_results ? true : false; + var has_tran = (this.transaction && this.transaction.notes) ? true : false; // initialize connection note - if ( ! this.notes.authentication_results ) { - this.notes.authentication_results = [ config.get('me') ]; - } + if ( has_conn === false) { this.notes.authentication_results = []; }; // initialize transaction note, if possible - if ( in_transaction === true && !this.transaction.notes.authentication_results ) { + if ( has_tran === true && !this.transaction.notes.authentication_results ) { this.transaction.notes.authentication_results = []; } // if message, store it in the appropriate note if ( message ) { - if ( in_transaction === true ) { + if ( has_tran === true ) { this.transaction.notes.authentication_results.push(message); - this.logdebug("ar_tran: " + this.transaction.notes.authentication_results); } else { this.notes.authentication_results.push(message); - this.logdebug("ar_conn: " + this.notes.authentication_results); } }; - // return the formatted header - var header = [ this.notes.authentication_results.join('; '), - (in_transaction === true ? this.transaction.notes.authentication_results.join('; ') : '') + // format the new header + var header = [ + config.get('me'), + (has_conn === true ? this.notes.authentication_results.join('; ') : ''), + (has_tran === true ? this.transaction.notes.authentication_results.join('; ') : '') ].join('; '); - this.logdebug("ar_header: " + header); return header; }; Connection.prototype.auth_results_clean = function(conn) { + // move any existing Auth-Res headers to Original-Auth-Res headers // http://tools.ietf.org/html/draft-kucherawy-original-authres-00.html var ars = conn.transaction.header.get_all('Authentication-Results'); if ( ars.length === 0 ) { return; }; From e5309344ee624520d84bb11b70f730c3493cd54f Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 18:11:41 -0500 Subject: [PATCH 062/160] import Steve's p0f plugin --- config/connect.p0f.ini | 2 + plugins/connect.p0f.js | 236 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 config/connect.p0f.ini create mode 100644 plugins/connect.p0f.js diff --git a/config/connect.p0f.ini b/config/connect.p0f.ini new file mode 100644 index 000000000..cf40ae197 --- /dev/null +++ b/config/connect.p0f.ini @@ -0,0 +1,2 @@ + +socket_path=/tmp/.p0f_socket diff --git a/plugins/connect.p0f.js b/plugins/connect.p0f.js new file mode 100644 index 000000000..26a95eda3 --- /dev/null +++ b/plugins/connect.p0f.js @@ -0,0 +1,236 @@ +// Javascript p0f v3 client + +var net = require('net'); +var ipaddr = require('ipaddr.js'); + +function p0f_client(path) { + var self = this; + + this.sock = null; + this.send_queue = []; + this.receive_queue = []; + this.connected = false; + this.ready = false; + this.socket_has_error = false; + this.restart_interval = false; + + var connect = function () { + self.sock = net.createConnection(path); + + self.sock.setTimeout(5 * 1000); + + self.sock.on('connect', function () { + self.sock.setTimeout(30 * 1000); + self.connected = true; + self.socket_has_error = false; + self.ready = true; + if (self.restart_interval) clearInterval(self.restart_interval); + self.process_send_queue(); + }); + + self.sock.on('data', function (data) { + for (var i=0; i 0) { + throw new Error('unexpected data received'); + } + var item = this.receive_queue.shift(); + + /////////////////// + // Decode packet // + /////////////////// + + // Response magic dword (0x50304602), native endian. + if (data.readUInt32LE(0) !== 0x50304602) { + return item.cb(new Error('bad response magic!')); + } + // Status dword: 0x00 for 'bad query', 0x10 for 'OK', and 0x20 for 'no match' + var st = data.readUInt32LE(4); + switch (st) { + case (0x00): + return item.cb(new Error('bad query')); + break; + case (0x10): + var p0f = { + query: item.ip, + first_seen: data.readUInt32LE(8), + last_seen: data.readUInt32LE(12), + total_conn: data.readUInt32LE(16), + uptime_min: data.readUInt32LE(20), + up_mod_days: data.readUInt32LE(24), + last_nat: data.readUInt32LE(28), + last_chg: data.readUInt32LE(32), + distance: data.readInt16LE(36), + bad_sw: data.readUInt8(38), + os_match_q: data.readUInt8(39), + os_name: decode_string(data, 40, 72), + os_flavor: decode_string(data, 72, 104), + http_name: decode_string(data, 104, 136), + http_flavor: decode_string(data, 136, 168), + link_type: decode_string(data, 168, 200), + language: decode_string(data, 200, 232), + } + return item.cb(null, p0f); + break; + case (0x20): + return item.cb(null, null); + break; + default: + throw new Error('unknown status: ' + st); + } +} + +p0f_client.prototype.query = function (ip, cb) { + if (this.socket_has_error) { + return cb(this.socket_has_error); + } + if (!this.connected) { + return cb(new Error('socket not connected')); + } + var addr = ipaddr.parse(ip); + var bytes = addr.toByteArray(); + var buf = new Buffer(21); + buf.writeUInt32LE(0x50304601, 0); // query magic + buf.writeUInt8(((addr.kind() === 'ipv6') ? 0x6 : 0x4), 4); + for (var i=0; i < bytes.length; i++) { + buf.writeUInt8(bytes[i], 5 + i); + } + if (!this.ready) { + this.send_queue.push({ip: ip, cb: cb, buf: buf}); + } + else { + this.receive_queue.push({ip: ip, cb: cb}); + if (!this.sock.write(buf)) this.ready = false; + } +} + +p0f_client.prototype.process_send_queue = function () { + if (this.send_queue.length > 0) { + for (var i=0; i Date: Fri, 10 Jan 2014 18:22:59 -0500 Subject: [PATCH 063/160] my first whack at p0f renamed p0f -> connect.p0f (as other plugins do) whitespace/indention cleanups added config option for adding the X-Haraka-p0f header (enable/disable/rename) refactored several methods to remove unnecessary indentions changed passing of 'this' in log* methods to (var plugin = this) and passing plugin, so log entries have the correct plugin name (no, I don't know why that fixes it) consolidated 3 instances of repetition into format_results function added a couple function names to anonymous functions (useful debug info) --- config/connect.p0f.ini | 6 + plugins/connect.p0f.js | 271 +++++++++++++++++++++-------------------- 2 files changed, 145 insertions(+), 132 deletions(-) diff --git a/config/connect.p0f.ini b/config/connect.p0f.ini index cf40ae197..3ec8c1901 100644 --- a/config/connect.p0f.ini +++ b/config/connect.p0f.ini @@ -1,2 +1,8 @@ +; where the p0f socket is found +; default: socket_path=/tmp/.p0f_socket socket_path=/tmp/.p0f_socket + +; add_header, add a message header with a p0f summary +; default: X-Haraka-p0f +add_header=X-Haraka-p0f diff --git a/plugins/connect.p0f.js b/plugins/connect.p0f.js index 26a95eda3..34746c796 100644 --- a/plugins/connect.p0f.js +++ b/plugins/connect.p0f.js @@ -1,4 +1,4 @@ -// Javascript p0f v3 client +// p0f v3 client - http://lcamtuf.coredump.cx/p0f3/ var net = require('net'); var ipaddr = require('ipaddr.js'); @@ -14,54 +14,49 @@ function p0f_client(path) { this.socket_has_error = false; this.restart_interval = false; - var connect = function () { - self.sock = net.createConnection(path); + self.sock = net.createConnection(path); - self.sock.setTimeout(5 * 1000); + self.sock.setTimeout(5 * 1000); - self.sock.on('connect', function () { - self.sock.setTimeout(30 * 1000); - self.connected = true; - self.socket_has_error = false; - self.ready = true; - if (self.restart_interval) clearInterval(self.restart_interval); - self.process_send_queue(); - }); + self.sock.on('connect', function () { + self.sock.setTimeout(30 * 1000); + self.connected = true; + self.socket_has_error = false; + self.ready = true; + if (self.restart_interval) clearInterval(self.restart_interval); + self.process_send_queue(); + }); - self.sock.on('data', function (data) { - for (var i=0; i 0) { - throw new Error('unexpected data received'); + throw new Error('unexpected data received'); } var item = this.receive_queue.shift(); @@ -86,41 +81,41 @@ p0f_client.prototype.decode_response = function (data) { // Response magic dword (0x50304602), native endian. if (data.readUInt32LE(0) !== 0x50304602) { - return item.cb(new Error('bad response magic!')); + return item.cb(new Error('bad response magic!')); } // Status dword: 0x00 for 'bad query', 0x10 for 'OK', and 0x20 for 'no match' var st = data.readUInt32LE(4); switch (st) { - case (0x00): - return item.cb(new Error('bad query')); - break; - case (0x10): - var p0f = { + case (0x00): + return item.cb(new Error('bad query')); + break; + case (0x10): + var p0f = { query: item.ip, - first_seen: data.readUInt32LE(8), - last_seen: data.readUInt32LE(12), - total_conn: data.readUInt32LE(16), - uptime_min: data.readUInt32LE(20), - up_mod_days: data.readUInt32LE(24), - last_nat: data.readUInt32LE(28), - last_chg: data.readUInt32LE(32), - distance: data.readInt16LE(36), - bad_sw: data.readUInt8(38), - os_match_q: data.readUInt8(39), - os_name: decode_string(data, 40, 72), - os_flavor: decode_string(data, 72, 104), - http_name: decode_string(data, 104, 136), - http_flavor: decode_string(data, 136, 168), - link_type: decode_string(data, 168, 200), - language: decode_string(data, 200, 232), - } - return item.cb(null, p0f); - break; - case (0x20): - return item.cb(null, null); - break; - default: - throw new Error('unknown status: ' + st); + first_seen: data.readUInt32LE(8), + last_seen: data.readUInt32LE(12), + total_conn: data.readUInt32LE(16), + uptime_min: data.readUInt32LE(20), + up_mod_days: data.readUInt32LE(24), + last_nat: data.readUInt32LE(28), + last_chg: data.readUInt32LE(32), + distance: data.readInt16LE(36), + bad_sw: data.readUInt8(38), + os_match_q: data.readUInt8(39), + os_name: decode_string(data, 40, 72), + os_flavor: decode_string(data, 72, 104), + http_name: decode_string(data, 104, 136), + http_flavor: decode_string(data, 136, 168), + link_type: decode_string(data, 168, 200), + language: decode_string(data, 200, 232), + } + return item.cb(null, p0f); + break; + case (0x20): + return item.cb(null, null); + break; + default: + throw new Error('unknown status: ' + st); } } @@ -149,19 +144,19 @@ p0f_client.prototype.query = function (ip, cb) { } p0f_client.prototype.process_send_queue = function () { - if (this.send_queue.length > 0) { - for (var i=0; i Date: Fri, 10 Jan 2014 18:33:21 -0500 Subject: [PATCH 064/160] p0f: added docs --- docs/plugins/connect.p0f.md | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/plugins/connect.p0f.md diff --git a/docs/plugins/connect.p0f.md b/docs/plugins/connect.p0f.md new file mode 100644 index 000000000..a5bdeff00 --- /dev/null +++ b/docs/plugins/connect.p0f.md @@ -0,0 +1,48 @@ +connect.p0f - A TCP Fingerprinting Plugin +======================== + +Use TCP fingerprint info (remote computer OS, network distance, etc) to +implement more sophisticated anti-spam policies. + +This plugin inserts a _p0f_ connection note with information deduced +from the TCP fingerprint. The note typically includes at least the link, +detail, distance, uptime, genre. Here's an example: + + genre => FreeBSD + detail => 8.x (1) + uptime => 1390 + link => ethernet/modem + distance => 17 + +Which was parsed from this p0f fingerprint: + + 24.18.227.2:39435 - FreeBSD 8.x (1) (up: 1390 hrs) + -> 208.75.177.101:25 (distance 17, link: ethernet/modem) + +The following additional values may also be available in +the _p0f_ connection note: + + magic, status, first_seen, last_seen, total_conn, uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q, os_name, os_flavor, http_name, http_flavor, link_type, and language. + + +Configuration +----------------- + +1. start p0f + +Create a startup script for p0f that creates a communication socket when your +server starts up. + + /usr/local/bin/p0f -u smtpd -d -s /tmp/.p0f_socket 'dst port 25 or dst port 587' + chown smtpd /tmp/.p0f_socket + +2. configure p0f plugin + +add an entry to config/plugins to enable p0f: + + connect.p0f + + +3. review settings in config/connect.p0f.ini + + From de8d999bf30aa36c27bfaf07b520ffd246f5faa6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 19:02:32 -0500 Subject: [PATCH 065/160] TODO: added plugin renames --- TODO | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TODO b/TODO index 7e664f0d5..276fc3b0c 100644 --- a/TODO +++ b/TODO @@ -17,3 +17,19 @@ Outbound improvements - Limit concurrency by domain - Disable deliveries for a domain - Pool connections by domain/MX + +Remove the following deprecated plugins + - rdns.regexp + - data.nomsgid + - data.noreceived + - data.rfc5322_header_checks + - daemonize + +Rename the following plugins + - avg -> data.avg + - clamd -> data.clamd + - spamassassin -> data.spamassassin + - spf -> mail_from.spf + +Move the following plugins: + - test_queue -> queue/test_queue From d17f7b04e2b4f31171ad2b23f22b687dd6189835 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 11 Jan 2014 01:49:55 -0500 Subject: [PATCH 066/160] TODO: added rename attachment -> data.attachment --- TODO | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO b/TODO index 276fc3b0c..4dcf38181 100644 --- a/TODO +++ b/TODO @@ -26,6 +26,7 @@ Remove the following deprecated plugins - daemonize Rename the following plugins + - attachment -> data.attachment - avg -> data.avg - clamd -> data.clamd - spamassassin -> data.spamassassin From 5a03b8241138050e69be04ff812010416c8771de Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 12 Jan 2014 00:58:13 -0500 Subject: [PATCH 067/160] TODO: added rename toobusy -> connect.toobusy --- TODO | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO b/TODO index 4dcf38181..6f025ea23 100644 --- a/TODO +++ b/TODO @@ -26,6 +26,7 @@ Remove the following deprecated plugins - daemonize Rename the following plugins + - toobusy -> connect.toobusy - attachment -> data.attachment - avg -> data.avg - clamd -> data.clamd From 4407c074052d1db89a63af9fdc9254998b0342a1 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 12 Jan 2014 01:00:07 -0500 Subject: [PATCH 068/160] headers: fixed Date header bug I was calculating the seconds myself, and didn't notice that JS returns milliseconds by default this caused the "Too old" and "Too future" date validation to be much too strict. --- plugins/data.headers.js | 55 ++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/plugins/data.headers.js b/plugins/data.headers.js index b81db51a5..387e78dc4 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -8,7 +8,8 @@ var date_future_days = 2; var date_past_days = 15; exports.hook_data_post = function (next, connection) { - refreshConfig(this); + var plugin = this; + refreshConfig(plugin); var header = connection.transaction.header; @@ -27,31 +28,45 @@ exports.hook_data_post = function (next, connection) { return next(DENY, "Only one " + singular_headers[i] + " header allowed. See RFC 5322, Section 3.6"); } - } - - var msg_date = header.get_all('Date'); - if ( msg_date.length > 0 ) { - this.logdebug(connection, "message date: " + msg_date); - var msg_secs = Date.parse(msg_date); - this.logdebug(connection, "parsed date: " + msg_secs); - var now_secs = Date.now(); - this.logdebug(connection, "now seconds: " + now_secs); - - if ( date_future_days > 0 && msg_secs > (now_secs + (date_future_days * 24 * 3600)) ) { - this.loginfo(connection, "date too far in the future: " + msg_date ); - return next(DENY, "The Date header is too far in the future"); - } - if ( date_past_days > 0 && msg_secs < (now_secs - ( date_past_days * 24 * 3600 )) ) { - this.loginfo(connection, "date too old: " + msg_date ); - return next(DENY, "The Date header is too old"); - }; }; + var errmsg = checkDateValid(plugin,connection); + if (errmsg) return next(DENY, errmsg); + return next(); } +function checkDateValid (plugin,connection) { + + var msg_date = connection.transaction.header.get_all('Date'); + if (!msg_date || msg_date.length === 0) return; + + connection.logdebug(plugin, "message date: " + msg_date); + msg_date = Date.parse(msg_date); + + if ( date_future_days > 0 ) { + var too_future = new Date; + too_future.setHours(too_future.getHours() + 24 * date_future_days); + // connection.logdebug(plugin, "too future: " + too_future); + if ( msg_date > too_future ) { + connection.loginfo(plugin, "date is newer than: " + too_future ); + return "The Date header is too far in the future"; + }; + } + if ( date_past_days > 0 ) { + var too_old = new Date; + too_old.setHours(too_old.getHours() - 24 * date_past_days); + // connection.logdebug(plugin, "too old: " + too_old); + if ( msg_date < too_old ) { + connection.loginfo(plugin, "date is older than: " + too_old); + return "The Date header is too old"; + }; + }; + return; +}; + function refreshConfig(plugin) { - var config = plugin.config.get('data.headers.ini'); + var config = plugin.config.get('data.headers.ini'); if ( config.main.required !== 'undefined' ) { required_headers = config.main.required.split(','); From 366e3bd4e0cdbb54ba0164954bebdaea997b960c Mon Sep 17 00:00:00 2001 From: armored Date: Wed, 15 Jan 2014 20:08:48 -0700 Subject: [PATCH 069/160] Added trailing space to 'found config for' --- plugins/relay_acl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/relay_acl.js b/plugins/relay_acl.js index 54e6ed30c..faeb6e461 100644 --- a/plugins/relay_acl.js +++ b/plugins/relay_acl.js @@ -53,7 +53,7 @@ exports.check_relay_domains = function (next, connection, params) { function dest_domain_action(connection, plugin, domains_ini, dest_domain) { if (dest_domain in domains_ini) { var config = JSON.parse(domains_ini[dest_domain]); - connection.logdebug(plugin, 'found config for' + dest_domain + ': ' + domains_ini['action']); + connection.logdebug(plugin, 'found config for ' + dest_domain + ': ' + domains_ini['action']); return config['action']; } return 'deny'; From d992b1d04f6616bbf822b64b8f440d55e51d0e2e Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Thu, 16 Jan 2014 15:51:23 -0500 Subject: [PATCH 070/160] Fix usage of 'drain' event on fs.write() The 'drain' event only fires if fs.write() returns false, not every time something is drained to the filesystem. Also switch to doing it 'once' not every time. Fixes #395 --- outbound.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/outbound.js b/outbound.js index 065d6a30a..0da0edb32 100644 --- a/outbound.js +++ b/outbound.js @@ -447,11 +447,12 @@ exports.process_domain = function (ok_paths, todo, hmails, cb) { fs.unlink(tmp_path, function () {}); cb("Queueing failed"); }); - self.build_todo(todo, ws); - todo.message_stream.pipe(ws, { line_endings: '\r\n', dot_stuffing: true, ending_dot: false }); + self.build_todo(todo, ws, function () { + todo.message_stream.pipe(ws, { line_endings: '\r\n', dot_stuffing: true, ending_dot: false }); + }); } -exports.build_todo = function (todo, ws) { +exports.build_todo = function (todo, ws, write_more) { // Replacer function to exclude items from the queue file header function exclude_from_json(key, value) { switch (key) { @@ -473,16 +474,18 @@ exports.build_todo = function (todo, ws) { var buf = Buffer.concat([todo_length, todo_str], todo_str.length + 4); - ws.write(buf); + var continue_writing = ws.write(buf); + if (continue_writing) return write_more(); + ws.once('drain', write_more); } exports.split_to_new_recipients = function (hmail, recipients, response, cb) { + var self = this; if (recipients.length === hmail.todo.rcpt_to.length) { // Split to new for no reason - increase refcount and return self hmail.refcount++; return cb(hmail); } - var self = this; var fname = _fname(); var tmp_path = path.join(queue_dir, '.' + fname); var ws = new FsyncWriteStream(tmp_path, { flags: WRITE_EXCL }); @@ -530,8 +533,6 @@ exports.split_to_new_recipients = function (hmail, recipients, response, cb) { hmail.bounce("Error re-queueing some recipients", err); }); - ws.on('drain', write_more); - var new_todo = JSON.parse(JSON.stringify(hmail.todo)); new_todo.rcpt_to = recipients; new_todo.uuid = utils.uuid(); From fe1f4a90fe59a7fa8eb99b08c493956b1547baf2 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 14 Jan 2014 04:08:24 -0500 Subject: [PATCH 071/160] karma: initial authoring --- config/connect.karma.ini | 47 ++++++++ connection.js | 4 +- docs/plugins/karma.md | 205 +++++++++++++++++++++++++++++++++++ plugins/connect.karma.js | 229 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 config/connect.karma.ini create mode 100644 docs/plugins/karma.md create mode 100644 plugins/connect.karma.js diff --git a/config/connect.karma.ini b/config/connect.karma.ini new file mode 100644 index 000000000..2434f2584 --- /dev/null +++ b/config/connect.karma.ini @@ -0,0 +1,47 @@ + +; nice: get a karma score greater than the positive connection limit +; naughty: achieve a karma score lower than the negative connection limit +; +; how many days to penalize naughty senders +penalty_days = 1 + + +; concurrency limits. Comment out this block of settings to disable +[concurrency] +naughty=1 +neutral=2 +nice=10 + +[connection_limit] +; Be conservative to avoid false positives! +; the threshhold below which a connection is considered naughty +negative=-3 +; score at which a connection is considered nice +positive=2 + +; karma history = nice - naughty connections. To achieve a negative score, +; senders must send more naughty than nice messages. Is it worth getting +; 5 spam and 2 ham? Adjust this knob accordingly. +history_negative=-3 + + +[redis] +server_ip = 127.0.0.1 +server_port = 6379 + + +[spammy_tlds] +; award negative karma to spammy TLDs +; careful, awarding karma > msg_negative_limit may blacklist that TLD +info=-4 +pw=-4 +tw=-3 +biz=-3 +cl=-2 +br=-2 +fr=-2 +be=-2 +jp=-2 +no=-2 +se=-2 +sg=-2 diff --git a/connection.js b/connection.js index aa61c6563..f120426f5 100644 --- a/connection.js +++ b/connection.js @@ -1154,7 +1154,8 @@ Connection.prototype.cmd_mail = function(line) { var self = this; this.init_transaction(function () { - self.transaction.mail_from = from + self.transaction.mail_from = from; + self.transaction.mail_from_raw = line; plugins.run_hooks('mail', self, [from, params]); }); }; @@ -1200,6 +1201,7 @@ Connection.prototype.cmd_rcpt = function(line) { } this.transaction.rcpt_to.push(recip); + this.transaction.rcpt_to_raw = line; plugins.run_hooks('rcpt', this, [recip, params]); }; diff --git a/docs/plugins/karma.md b/docs/plugins/karma.md new file mode 100644 index 000000000..eae7b5790 --- /dev/null +++ b/docs/plugins/karma.md @@ -0,0 +1,205 @@ + +karma - reward nice and penalize naughty mail senders +=========================== + +Karma tracks sender history, allowing varying QoS +to naughty, nice, and unknown senders. + +DESCRIPTION +----------------------- +Karma records the number of nice, naughty, and total connections from mail +senders. After sending a naughty message, if a sender has more naughty than +nice connections, they are penalized for *penalty_days*. Connections +from senders in the penalty box are rejected. + +Karma stores two connection notes that other plugins can use to be more +lenient or strict. + + connection.notes.karma - karma score on *this* connection + connection.notes.karma_history - karma history + +Karma history is computed as the number of nice - naughty connections. + +Karma is small, fast, and ruthlessly efficient. Karma can be used to craft +custom connection policies such as these two examples: + +1. Hi well known and well behaved sender. Help yourself to greater + concurrency (hosts_allow), multiple recipients (karma), and no + delays (early_sender). + +2. Hi there, naughty sender. You get a max concurrency of 1, max recipients + of 2, and SMTP delays. + + +CONFIG +==================== + +negative +-------------------- +How negative a senders karma can get before we penalize them. + +Default: 2 + +Examples: + + negative 1: 0 nice - 1 naughty = karma -1, penalize + negative 1: 1 nice - 1 naughty = karma 0, okay + negative 2: 1 nice - 2 naughty = karma -1, okay + negative 2: 1 nice - 3 naughty = karma -2, penalize + +With the default negative limit of one, there's a very small chance you could +penalize a "mostly good" sender. Raising it to 2 reduces that possibility to +improbable. + +penalty_days +-------------------- + +The number of days a naughty sender is refused connections. Use a decimal +value to penalize for portions of days. + + karma penalty_days 1 + +Default: 1 + +reject [ 0 | 1 ] +------------------- +0 will not reject any connections. +1 will reject naughty senders. + + +db_dir +-------------------- +Path to a directory in which the DB will be stored. This directory must be +writable by the qpsmtpd user. If unset, the first usable directory from the +following list will be used: + + /var/lib/qpsmtpd/karma + + BINDIR/var/db (where BINDIR is the location of the qpsmtpd binary) + + BINDIR/config + + +BENEFITS +-------------------- +Karma reduces the resources wasted by naughty mailers. + +The biggest gains to be had are by having heavy plugins (spamassassin, dspam, +virus filters) set the _karma_ connection note (see KARMA) when they encounter +naughty senders. Reasons to send servers to the penalty box could include +sending a virus, early talking, or sending messages with a very high spam +score. + +This plugin does not penalize connections with transaction notes I +or I set. These notes would have been set by the B, +B, and B plugins. Obviously, those plugins must +run before B for that to work. + +KARMA +------------------------ + +It is mostly up to other plugins to reward well behaved senders with positive +karma and smite poorly behaved senders with negative karma. +See B + +After the connection ends, B will record the result. Mail servers whose +naughty connections exceed nice ones are sent to the penalty box. Servers in +the penalty box will be tersely disconnected for I. Here is +an example connection from an IP in the penalty box: + + 73122 Connection from smtp.midsetmediacorp.com [64.185.226.65] + 73122 (connect) ident::geoip: US, United States + 73122 (connect) ident::p0f: Windows 7 or 8 + 73122 (connect) earlytalker: pass: 64.185.226.65 said nothing spontaneous + 73122 (connect) relay: skip: no match + 73122 (connect) karma: fail + 73122 550 You were naughty. You are cannot connect for 0.99 more days. + 73122 click, disconnecting + 73122 (post-connection) connection_time: 1.048 s. + +If we only set negative karma, we will almost certainly penalize servers we +want to receive mail from. For example, a Yahoo user sends an egregious spam +to a user on our server. Now nobody on our server can receive email from that +Yahoo server for I. This should happen approximately 0% of +the time if we are careful to also set positive karma. + +KARMA HISTORY +------------------------ +Karma maintains a history for each IP. When a senders history has decreased +below -5 and they have never sent a good message, they get a karma bonus. +The bonus tacks on an extra day of blocking for every naughty message they +send. + +Example: an unknown sender delivers a spam. They get a one day penalty_box. +After 5 days, 5 spams, 5 penalties, and 0 nice messages, they get a six day +penalty. The next offense gets a 7 day penalty, and so on. + +USING KARMA +----------------------- +To get rid of naughty connections as fast as possible, run karma before other +connection plugins. Plugins that trigger DNS lookups or impose time delays +should run after B. In this example, karma runs before all but the +ident plugins. + + 89011 Connection from Unknown [69.61.27.204] + 89011 (connect) ident::geoip: US, United States + 89011 (connect) ident::p0f: Linux 3.x + 89011 (connect) karma: fail, 1 naughty, 0 nice, 1 connects + 89011 550 You were naughty. You are penalized for 0.99 more days. + 89011 click, disconnecting + 89011 (post-connection) connection_time: 0.118 s. + 88798 cleaning up after 89011 + +Unlike RBLs, B only penalizes IPs that have sent us spam, and only when +those senders have sent us more spam than ham. + +USING KARMA IN OTHER PLUGINS +------------------------------ +This plugin sets the connection note I. Your plugin can +use the senders karma to be more gracious or rude to senders. The value of +I is the number of nice connections minus naughty +ones. The higher the number, the better you should treat the sender. + +To alter a connections karma based on its behavior, do this: + + $self->adjust_karma( -1 ); # lower karma (naughty) + $self->adjust_karma( 1 ); # raise karma (good) + + +EFFECTIVENESS +--------------------- + +In the first 24 hours, _karma_ rejected 8% of all connections. After one +week of running with I, karma has rejected 15% of all +connections. + +This plugins effectiveness results from the propensity of naughty senders +to be repeat offenders. Limiting them to a single offense per day(s) greatly +reduces the resources they can waste. + +Of the connections that had previously passed all other checks and were caught +only by spamassassin and/or dspam, B rejected 31 percent. Since +spamassassin and dspam consume more resources than others plugins, this plugin +seems to be a very big win. + +DATABASE +--------------------- + +Connection summaries are stored in a database. The DB value is a : delimited +list containing a penalty box start time (if the server is/was on timeout) +and the count of naughty, nice, and total connections. The database can be +listed and searched with the karma_tool script. + + +BUGS & LIMITATIONS +--------------------- + +This plugin is reactionary. Like the FBI, it doesn't do much until +after a crime has been committed. + +There is little to be gained by listing servers that are already on DNS +blacklists, send to invalid users, earlytalkers, etc. Those already have +very lightweight tests. + +* some type of ASN integration, for tracking karma of 'neighborhoods' + diff --git a/plugins/connect.karma.js b/plugins/connect.karma.js new file mode 100644 index 000000000..5c024aae4 --- /dev/null +++ b/plugins/connect.karma.js @@ -0,0 +1,229 @@ +// karma - reward nice and penalize naughty mail senders + +var ipaddr = require('ipaddr.js'); +var redis = require('redis'); +var db; + +var penalty_days = 1; +var spammy_tlds = []; + +exports.register = function () { + var plugin = this; + + this.register_hook('init_master', 'karma_onInit'); + this.register_hook('init_child', 'karma_onInit'); + this.register_hook('lookup_rdns', 'karma_onConnect'); + this.register_hook('mail', 'karma_onMailFrom'); + this.register_hook('rcpt', 'karma_onRcptTo'); + this.register_hook('data', 'karma_onData'); + this.register_hook('disconnect', 'karma_onDisconnect'); +}; + +exports.karma_onInit = function (next,server) { + var config = this.config.get('connect.karma.ini'); + + if ( config.main.penalty_days ) penalty_days = config.main.penalty_days; + + if ( ! config.spammy_tlds ) { + spammy_tlds = { + 'info':-3, 'pw' :-3, 'tw':-3, 'biz':-3, + 'cl' :-2, 'br' :-2, 'fr':-2, 'be':-2, 'jp':-2, 'no':-2, 'se':-2, 'sg':-2, + }; + } + else { + // TODO: test parsing spammy_tlds from .ini + for (var i=0; config.main.spammy_tlds.length(); i++) { + spammy_tlds[i] = config.main.spammy_tlds[i]; + }; + }; + + var redis_ip = '127.0.0.1'; + var redis_port = '6379'; + if ( config.redis ) { + redis_ip = config.redis.server_ip; + redis_port = config.redis.server_port; + }; + db = redis.createClient(redis_port, redis_ip); + return next(); +} + +exports.karma_onConnect = function (next, connection) { + var plugin = this; + + connection.notes.karma = 0; // defaults + connection.notes.karma_history = 0; + + var key = 'karma|'+connection.remote_ip; + + db.hgetall(key, function redisResults (err,obj) { + if (err) { + connection.logdebug(plugin,"err: "+err); + return next(); + }; + + if (obj === null) { // first connection by this IP + db.hmset(key, {'penalty_start_ts': 0, 'naughty': 0, 'nice': 0, 'concurrent':1, 'connections': 1}); + db.expire(key, 86400 * 60); // expire after 60 days + connection.logdebug(plugin,"no results"); + return next(); + }; + + db.hincrby(key, 'concurrent', 1); + db.hincrby(key, 'connections', 1); // total connections + db.expire(key, 86400 * 60); // extend expiration date + + var history = (obj.nice || 0) - (obj.naughty || 0); + connection.notes.karma_history = history; + + var summary = obj.naughty+" naughty, "+obj.nice+" nice, "+obj.connections+" connects, "+history+" history"; + + if ( config.concurrency ) { + var reject=0; + obj.concurrent++; // add this connection + if (history < 0 && obj.concurrent > config.concurrency.naughty) reject++; + if (history > 0 && obj.concurrent > config.concurrency.nice) reject++; + if (history ==0 && obj.concurrent > config.concurrency.neutral) reject++; + if (reject) { + connection.loginfo(plugin, "too many concurrent connections ("+summary+")"); + return next(DENY, "too many connections: "+obj.concurrent); + }; + }; + + if (obj.penalty_start_ts === '0') { + connection.loginfo(plugin, "no penalty ("+summary+")"); + return next(); + } + + var days_old = (Date.now() - Date.parse(obj.penalty_start_ts)) / 86.4; + if (days_old >= penalty_days) { + connection.loginfo(plugin, "penalty expired ("+summary+")"); + return next(); + } + + var left = +( penalty_days - days_old ).toFixed(2); + var mess = "Bad karma, you can try again in "+left+" more days."; + + return next(DENY, mess); + }); +}; + +exports.karma_onMailFrom = function (next, connection, params) { + var plugin = this; + var mail_from = params[0]; + var from_tld = mail_from.host.split('.').pop(); + // connection.logdebug(plugin, "from_tld: "+from_tld); + + var karma_penalty = spammy_tlds[from_tld] || 0; + if (karma_penalty) { + connection.logdebug(plugin, "spammy TLD award: "+karma_penalty); + connection.notes.karma -= karma_penalty; + }; + + var full_from = connection.transaction.mail_from_raw; + connection.logdebug(plugin, "mail_from raw: "+full_from); + +// test if sender has placed an illegal RFC (2)821 space in envelope from + if ( full_from.toUpperCase().substring(0,6) !== 'FROM:<' ) { + connection.loginfo(plugin, "illegal envelope address format: "+full_from ); + connection.notes.karma -= karma_penalty; + }; + + connection.loginfo(plugin, "karma score: "+ connection.notes.karma ); + return next(); +}; + +exports.karma_onRcptTo = function (next, connection, params) { + var plugin = this; + + var rcpt = params[0]; + var full_rcpt = connection.transaction.rcpt_to_raw; + + // check for an illegal RFC (2)821 space in envelope recipient + if ( full_rcpt.toUpperCase().substring(0,4) !== 'TO:<' ) { + connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); + connection.notes.karma -= karma_penalty; + }; + + var count = connection.rcpt_count.accept + connection.rcpt_count.tempfail + connection.rcpt_count.reject + 1; + if ( count <= 1 ) return next(); + + connection.loginfo(plugin, "recipient count: "+count ); + + var history = connection.notes.karma_history; + if ( history > 0 ) { + connection.loginfo(plugin, "good history"); + return next(); + }; + + var karma = connection.notes.karma; + if ( karma > 0 ) { + connection.loginfo(plugin, "good connection"); + return next(); + }; + + connection.loginfo(plugin, "karma score: "+ karma ); + + // limit recipients if host has negative or unknown karma + return next(DENY, "too many recipients for karma "+karma+" (h: "+history+")"); +} + +exports.karma_onData = function (next, connection) { +// cut off naughty senders at DATA to prevent receiving the message + var karma = connection.notes.karma; + if ( karma <= -4 ) { + return next(DENY, "very bad karma: "+karma); + }; + return next(); +}; + +exports.karma_onDisconnect = function (next, connection) { + var plugin = this; + + var key = 'karma|'+connection.remote_ip; + db.hincrby(key, 'concurrent', -1); + + var karma = connection.notes.karma; + var history = connection.notes.karma_history; + + if ( !karma ) { + connection.loginfo(plugin, "neutral, (msg: "+karma+", history: "+history+")"); + return next(); + }; + + var config = this.config.get('connect.karma.ini'); + var pos_lim = config.connection_limit.positive || 2; + + if (karma > pos_lim) { + db.hincrby(key, 'nice', 1); + connection.loginfo(plugin, "positive, (msg: "+karma+", history: "+history+")"); + return next(); + }; + + var negative_limit = config.connection_limit.negative || -3; + if (karma < negative_limit) { + db.hincrby(key, 'naughty', 1); + history--; + + if (history <= config.connection_limit.history_negative) { + if (history < -5) { + connection.loginfo(plugin, "penalty box bonus! (msg: "+karma+", history: "+history+")"); + log_mess = ", penalty box bonus!"; + db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); + } + else { + db.hset(key, 'penalty_start_ts', Date()); + } + connection.loginfo(plugin, "penalty box! (msg: "+karma+", history: "+history+")"); + next(); + } + } + connection.loginfo(plugin, "no action, msg: "+karma+", history: "+history+")"); + next(); +}; + +function addDays(date, days) { + var result = new Date(date); + result.setDate(date.getDate() + days); + return result; +} + From 9f57b44442ab739311bf0ff62ac0c4a9e9ba5b95 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 15 Jan 2014 01:01:55 -0500 Subject: [PATCH 072/160] moved concurrency tracking into separate redis key Steve pointed out that a connection that failed to complete properly (due to crashed plugin, or whatever) would cause the concurrent tracking to be miscounted and may cause problems. My solution: expire them after 4 minutes --- config/connect.karma.ini | 2 + connection.js | 2 - plugins/connect.karma.js | 161 ++++++++++++++++++++------------------- 3 files changed, 86 insertions(+), 79 deletions(-) diff --git a/config/connect.karma.ini b/config/connect.karma.ini index 2434f2584..9765a2346 100644 --- a/config/connect.karma.ini +++ b/config/connect.karma.ini @@ -7,6 +7,8 @@ penalty_days = 1 ; concurrency limits. Comment out this block of settings to disable +; Using this feature and the rate_limit plugin may produce unexpected results +; Use one or the other. [concurrency] naughty=1 neutral=2 diff --git a/connection.js b/connection.js index f120426f5..dd53bf6af 100644 --- a/connection.js +++ b/connection.js @@ -1155,7 +1155,6 @@ Connection.prototype.cmd_mail = function(line) { var self = this; this.init_transaction(function () { self.transaction.mail_from = from; - self.transaction.mail_from_raw = line; plugins.run_hooks('mail', self, [from, params]); }); }; @@ -1201,7 +1200,6 @@ Connection.prototype.cmd_rcpt = function(line) { } this.transaction.rcpt_to.push(recip); - this.transaction.rcpt_to_raw = line; plugins.run_hooks('rcpt', this, [recip, params]); }; diff --git a/plugins/connect.karma.js b/plugins/connect.karma.js index 5c024aae4..bbd0e9a7a 100644 --- a/plugins/connect.karma.js +++ b/plugins/connect.karma.js @@ -4,9 +4,6 @@ var ipaddr = require('ipaddr.js'); var redis = require('redis'); var db; -var penalty_days = 1; -var spammy_tlds = []; - exports.register = function () { var plugin = this; @@ -21,22 +18,6 @@ exports.register = function () { exports.karma_onInit = function (next,server) { var config = this.config.get('connect.karma.ini'); - - if ( config.main.penalty_days ) penalty_days = config.main.penalty_days; - - if ( ! config.spammy_tlds ) { - spammy_tlds = { - 'info':-3, 'pw' :-3, 'tw':-3, 'biz':-3, - 'cl' :-2, 'br' :-2, 'fr':-2, 'be':-2, 'jp':-2, 'no':-2, 'se':-2, 'sg':-2, - }; - } - else { - // TODO: test parsing spammy_tlds from .ini - for (var i=0; config.main.spammy_tlds.length(); i++) { - spammy_tlds[i] = config.main.spammy_tlds[i]; - }; - }; - var redis_ip = '127.0.0.1'; var redis_port = '6379'; if ( config.redis ) { @@ -45,87 +26,95 @@ exports.karma_onInit = function (next,server) { }; db = redis.createClient(redis_port, redis_ip); return next(); -} +}; exports.karma_onConnect = function (next, connection) { var plugin = this; + var config = this.config.get('connect.karma.ini'); - connection.notes.karma = 0; // defaults - connection.notes.karma_history = 0; + connection.notes.karma = new Number(); // defaults + connection.notes.karma_history = new Number(); var key = 'karma|'+connection.remote_ip; + var con_key = 'concurrent|'+connection.remote_ip; + + function initRemoteIP () { + db.multi() + .hmset(key, {'penalty_start_ts': 0, 'naughty': 0, 'nice': 0, 'connections': 1}) + .expire(key, 86400 * 60) // expire after 60 days + .exec(); + connection.logdebug(plugin,"first connect"); + }; - db.hgetall(key, function redisResults (err,obj) { - if (err) { - connection.logdebug(plugin,"err: "+err); - return next(); - }; + db.multi() + .get(con_key) + .hgetall(key) + .exec( function redisResults (err,replies) { + if (err) { + connection.logdebug(plugin,"err: "+err); + return next(); + }; - if (obj === null) { // first connection by this IP - db.hmset(key, {'penalty_start_ts': 0, 'naughty': 0, 'nice': 0, 'concurrent':1, 'connections': 1}); - db.expire(key, 86400 * 60); // expire after 60 days - connection.logdebug(plugin,"no results"); - return next(); - }; + if (replies[1] === null) { initRemoteIP(); return next(); }; - db.hincrby(key, 'concurrent', 1); - db.hincrby(key, 'connections', 1); // total connections - db.expire(key, 86400 * 60); // extend expiration date + db.hincrby(key, 'connections', 1); // total connections + db.expire(key, 86400 * 60); // extend expiration date - var history = (obj.nice || 0) - (obj.naughty || 0); - connection.notes.karma_history = history; + var kobj = replies[1]; + var history = (kobj.nice || 0) - (kobj.naughty || 0); + connection.notes.karma_history = history; - var summary = obj.naughty+" naughty, "+obj.nice+" nice, "+obj.connections+" connects, "+history+" history"; + var summary = kobj.naughty+" naughty, "+kobj.nice+" nice, "+kobj.connections+" connects, "+history+" history"; - if ( config.concurrency ) { - var reject=0; - obj.concurrent++; // add this connection - if (history < 0 && obj.concurrent > config.concurrency.naughty) reject++; - if (history > 0 && obj.concurrent > config.concurrency.nice) reject++; - if (history ==0 && obj.concurrent > config.concurrency.neutral) reject++; - if (reject) { - connection.loginfo(plugin, "too many concurrent connections ("+summary+")"); - return next(DENY, "too many connections: "+obj.concurrent); + var too_many = checkConcurrency(plugin, con_key, replies[0], history); + if ( too_many ) { + connection.loginfo(plugin, too_many + ", ("+summary+")"); + return next(DENYSOFT, too_many); }; - }; - if (obj.penalty_start_ts === '0') { - connection.loginfo(plugin, "no penalty ("+summary+")"); - return next(); - } + if (kobj.penalty_start_ts === '0') { + connection.loginfo(plugin, "no penalty ("+summary+")"); + return next(); + } - var days_old = (Date.now() - Date.parse(obj.penalty_start_ts)) / 86.4; - if (days_old >= penalty_days) { - connection.loginfo(plugin, "penalty expired ("+summary+")"); - return next(); - } + var days_old = (Date.now() - Date.parse(kobj.penalty_start_ts)) / 86.4; + var penalty_days = config.main.penalty_days; + if (days_old >= penalty_days) { + connection.loginfo(plugin, "penalty expired ("+summary+")"); + return next(); + } - var left = +( penalty_days - days_old ).toFixed(2); - var mess = "Bad karma, you can try again in "+left+" more days."; + var left = +( penalty_days - days_old ).toFixed(2); + var mess = "Bad karma, you can try again in "+left+" more days."; - return next(DENY, mess); - }); + return next(DENY, mess); + }); }; exports.karma_onMailFrom = function (next, connection, params) { var plugin = this; + var config = this.config.get('connect.karma.ini'); + var mail_from = params[0]; var from_tld = mail_from.host.split('.').pop(); - // connection.logdebug(plugin, "from_tld: "+from_tld); + connection.logdebug(plugin, "from_tld: "+from_tld); - var karma_penalty = spammy_tlds[from_tld] || 0; - if (karma_penalty) { - connection.logdebug(plugin, "spammy TLD award: "+karma_penalty); - connection.notes.karma -= karma_penalty; + if ( config.spammy_tlds ) { + var tld_penalty = (config.spammy_tlds[from_tld] || 0) * 1; // force numeric + + if (tld_penalty !== 0) { + connection.loginfo(plugin, "spammy TLD award: "+tld_penalty); + connection.notes.karma += tld_penalty; + }; }; - var full_from = connection.transaction.mail_from_raw; - connection.logdebug(plugin, "mail_from raw: "+full_from); + var full_from = connection.current_line; + connection.logdebug(plugin, "mail_from: "+full_from); // test if sender has placed an illegal RFC (2)821 space in envelope from - if ( full_from.toUpperCase().substring(0,6) !== 'FROM:<' ) { + if ( full_from.toUpperCase().substring(0,11) !== 'MAIL FROM:<' ) { connection.loginfo(plugin, "illegal envelope address format: "+full_from ); - connection.notes.karma -= karma_penalty; + connection.notes.karma.value--; }; connection.loginfo(plugin, "karma score: "+ connection.notes.karma ); @@ -136,12 +125,12 @@ exports.karma_onRcptTo = function (next, connection, params) { var plugin = this; var rcpt = params[0]; - var full_rcpt = connection.transaction.rcpt_to_raw; + var full_rcpt = connection.current_line; // check for an illegal RFC (2)821 space in envelope recipient - if ( full_rcpt.toUpperCase().substring(0,4) !== 'TO:<' ) { + if ( full_rcpt.toUpperCase().substring(0,9) !== 'RCPT TO:<' ) { connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); - connection.notes.karma -= karma_penalty; + connection.notes.karma--; }; var count = connection.rcpt_count.accept + connection.rcpt_count.tempfail + connection.rcpt_count.reject + 1; @@ -169,7 +158,7 @@ exports.karma_onRcptTo = function (next, connection, params) { exports.karma_onData = function (next, connection) { // cut off naughty senders at DATA to prevent receiving the message - var karma = connection.notes.karma; + var karma = connection.notes.karma * 1; if ( karma <= -4 ) { return next(DENY, "very bad karma: "+karma); }; @@ -178,9 +167,11 @@ exports.karma_onData = function (next, connection) { exports.karma_onDisconnect = function (next, connection) { var plugin = this; + var config = this.config.get('connect.karma.ini'); var key = 'karma|'+connection.remote_ip; - db.hincrby(key, 'concurrent', -1); + + if ( config.concurrency ) db.incrby('concurrent|'+connection.remote_ip, -1); var karma = connection.notes.karma; var history = connection.notes.karma_history; @@ -190,7 +181,6 @@ exports.karma_onDisconnect = function (next, connection) { return next(); }; - var config = this.config.get('connect.karma.ini'); var pos_lim = config.connection_limit.positive || 2; if (karma > pos_lim) { @@ -227,3 +217,20 @@ function addDays(date, days) { return result; } +function checkConcurrency(plugin, con_key, val, history) { + var config = plugin.config.get('connect.karma.ini'); + + if ( !config.concurrency ) return; + + var count = val || 0; // add this connection + count++; + db.incr(con_key); // increment Redis, (creates if needed) + db.expire(con_key, 4 * 60); // expire after 4 min + + var reject=0; + if (history < 0 && count > config.concurrency.naughty) reject++; + if (history > 0 && count > config.concurrency.nice) reject++; + if (history == 0 && count > config.concurrency.neutral) reject++; + if (reject) return "too many connections for you: "+count; + return; +}; From 80d9961e367c32394d210f0a24940f16aeb9ac97 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 15 Jan 2014 01:16:51 -0500 Subject: [PATCH 073/160] remove karma from TODO list --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index 6f025ea23..cbbade82d 100644 --- a/TODO +++ b/TODO @@ -7,7 +7,6 @@ - bogus_bounce (checks bounces have one recipient and no return-path) - dspam - greylisting - - karma? - virus/* Outbound improvements From 872f0b43427e1e0c795dad27813b1c1c3ce921d5 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Thu, 16 Jan 2014 23:23:54 -0500 Subject: [PATCH 074/160] added deny hook, made note an object to track plugin denials --- config/connect.karma.ini | 2 +- plugins/connect.karma.js | 113 ++++++++++++++++++++++++++++----------- 2 files changed, 84 insertions(+), 31 deletions(-) diff --git a/config/connect.karma.ini b/config/connect.karma.ini index 9765a2346..087483b95 100644 --- a/config/connect.karma.ini +++ b/config/connect.karma.ini @@ -14,7 +14,7 @@ naughty=1 neutral=2 nice=10 -[connection_limit] +[threshhold] ; Be conservative to avoid false positives! ; the threshhold below which a connection is considered naughty negative=-3 diff --git a/plugins/connect.karma.js b/plugins/connect.karma.js index bbd0e9a7a..9af4fc113 100644 --- a/plugins/connect.karma.js +++ b/plugins/connect.karma.js @@ -13,6 +13,7 @@ exports.register = function () { this.register_hook('mail', 'karma_onMailFrom'); this.register_hook('rcpt', 'karma_onRcptTo'); this.register_hook('data', 'karma_onData'); + this.register_hook('data_post', 'karma_onDataPost'); this.register_hook('disconnect', 'karma_onDisconnect'); }; @@ -32,8 +33,11 @@ exports.karma_onConnect = function (next, connection) { var plugin = this; var config = this.config.get('connect.karma.ini'); - connection.notes.karma = new Number(); // defaults - connection.notes.karma_history = new Number(); + connection.notes.karma = { + connection: 0, + history: 0, + penalties: [ ], + }; var key = 'karma|'+connection.remote_ip; var con_key = 'concurrent|'+connection.remote_ip; @@ -62,7 +66,7 @@ exports.karma_onConnect = function (next, connection) { var kobj = replies[1]; var history = (kobj.nice || 0) - (kobj.naughty || 0); - connection.notes.karma_history = history; + connection.notes.karma.history = history; var summary = kobj.naughty+" naughty, "+kobj.nice+" nice, "+kobj.connections+" connects, "+history+" history"; @@ -73,14 +77,14 @@ exports.karma_onConnect = function (next, connection) { }; if (kobj.penalty_start_ts === '0') { - connection.loginfo(plugin, "no penalty ("+summary+")"); + connection.loginfo(plugin, "no penalty "+karmaSummary(connection)); return next(); } var days_old = (Date.now() - Date.parse(kobj.penalty_start_ts)) / 86.4; var penalty_days = config.main.penalty_days; if (days_old >= penalty_days) { - connection.loginfo(plugin, "penalty expired ("+summary+")"); + connection.loginfo(plugin, "penalty expired "+karmaSummary(connection)); return next(); } @@ -91,6 +95,28 @@ exports.karma_onConnect = function (next, connection) { }); }; +exports.karma_onDeny = function (next, connection, params) { + /* params + ** [0] = plugin return value (constants.deny or constants.denysoft) + ** [1] = plugin return message + */ + + var pi_name = params[2]; + var pi_function = params[3]; + var pi_params = params[4]; + var pi_hook = params[5]; + + var plugin = this; + var transaction = connection.transaction; + + connection.notes.karma.connection--; + connection.notes.karma.penalties.push(pi_plugin); + + connection.loginfo(plugin, 'deny, '+karmaSummary(connection)); + + return next(); +}; + exports.karma_onMailFrom = function (next, connection, params) { var plugin = this; var config = this.config.get('connect.karma.ini'); @@ -104,20 +130,21 @@ exports.karma_onMailFrom = function (next, connection, params) { if (tld_penalty !== 0) { connection.loginfo(plugin, "spammy TLD award: "+tld_penalty); - connection.notes.karma += tld_penalty; + connection.notes.karma.connection += tld_penalty; }; }; var full_from = connection.current_line; connection.logdebug(plugin, "mail_from: "+full_from); -// test if sender has placed an illegal RFC (2)821 space in envelope from +// test if sender has placed an illegal (RFC 5321,2821,821) space in envelope from if ( full_from.toUpperCase().substring(0,11) !== 'MAIL FROM:<' ) { connection.loginfo(plugin, "illegal envelope address format: "+full_from ); - connection.notes.karma.value--; + connection.notes.karma.connection--; + connection.notes.karma.penalties.push('rfc5321.MailFrom'); }; - connection.loginfo(plugin, "karma score: "+ connection.notes.karma ); + connection.loginfo(plugin, karmaSummary(connection)); return next(); }; @@ -130,7 +157,8 @@ exports.karma_onRcptTo = function (next, connection, params) { // check for an illegal RFC (2)821 space in envelope recipient if ( full_rcpt.toUpperCase().substring(0,9) !== 'RCPT TO:<' ) { connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); - connection.notes.karma--; + connection.notes.karma.connection--; + connection.notes.karma.penalties.push('rfc5321.RcptTo'); }; var count = connection.rcpt_count.accept + connection.rcpt_count.tempfail + connection.rcpt_count.reject + 1; @@ -138,32 +166,43 @@ exports.karma_onRcptTo = function (next, connection, params) { connection.loginfo(plugin, "recipient count: "+count ); - var history = connection.notes.karma_history; + var history = connection.notes.karma.history; if ( history > 0 ) { connection.loginfo(plugin, "good history"); return next(); }; - var karma = connection.notes.karma; + var karma = connection.notes.karma.connection; if ( karma > 0 ) { connection.loginfo(plugin, "good connection"); return next(); }; - connection.loginfo(plugin, "karma score: "+ karma ); + connection.loginfo(plugin, karmaSummary(connection)); // limit recipients if host has negative or unknown karma - return next(DENY, "too many recipients for karma "+karma+" (h: "+history+")"); + return next(DENY, "too many recipients for poor karma: "+karmaSummary(connection)); } exports.karma_onData = function (next, connection) { // cut off naughty senders at DATA to prevent receiving the message + var config = this.config.get('connect.karma.ini'); + var negative_limit = config.threshhold.negative || -5; var karma = connection.notes.karma * 1; - if ( karma <= -4 ) { + + if ( karma.connection <= negative_limit ) { return next(DENY, "very bad karma: "+karma); - }; + } + return next(); -}; +} + +exports.karma_onDataPost = function (next, connection) { + connection.transaction.add_header('X-Haraka-Karma', + karmaSummary(connection) + ); + return next(); +} exports.karma_onDisconnect = function (next, connection) { var plugin = this; @@ -173,44 +212,58 @@ exports.karma_onDisconnect = function (next, connection) { if ( config.concurrency ) db.incrby('concurrent|'+connection.remote_ip, -1); - var karma = connection.notes.karma; - var history = connection.notes.karma_history; + var k = connection.notes.karma; + if ( !k ) { + connection.loginfo(plugin, "error: karma note missing!"); + return next(); + }; + var history = k.history; - if ( !karma ) { - connection.loginfo(plugin, "neutral, (msg: "+karma+", history: "+history+")"); + if ( !k.connection ) { + connection.loginfo(plugin, "neutral: "+karmaSummary(connection)); return next(); }; - var pos_lim = config.connection_limit.positive || 2; + var pos_lim = config.threshhold.positive || 2; - if (karma > pos_lim) { + if (k.connection > pos_lim) { db.hincrby(key, 'nice', 1); - connection.loginfo(plugin, "positive, (msg: "+karma+", history: "+history+")"); + connection.loginfo(plugin, "positive: "+karmaSummary(connection)); return next(); }; - var negative_limit = config.connection_limit.negative || -3; - if (karma < negative_limit) { + var negative_limit = config.threshhold.negative || -3; + if (k.connection < negative_limit) { db.hincrby(key, 'naughty', 1); + // connection.notes.karma.penalties.push('history'); history--; - if (history <= config.connection_limit.history_negative) { + if (history <= config.threshhold.history_negative) { if (history < -5) { - connection.loginfo(plugin, "penalty box bonus! (msg: "+karma+", history: "+history+")"); + connection.loginfo(plugin, "penalty box bonus! "+karmaSummary(connection)); log_mess = ", penalty box bonus!"; db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); } else { db.hset(key, 'penalty_start_ts', Date()); } - connection.loginfo(plugin, "penalty box! (msg: "+karma+", history: "+history+")"); + connection.loginfo(plugin, "penalty box! "+karmaSummary(connection)); next(); } } - connection.loginfo(plugin, "no action, msg: "+karma+", history: "+history+")"); + connection.loginfo(plugin, "no action, "+karmaSummary(connection)); next(); }; +function karmaSummary(c) { + var k = c.notes.karma; + return '('+ + 'conn:'+k.connection+ + ', hist: '+k.history+ + ', penalties: '+k.penalties+ + ')'; +} + function addDays(date, days) { var result = new Date(date); result.setDate(date.getDate() + days); From ef1c05fc5a50bf055ee8f5654fb4e8b0a2d173cf Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 17 Jan 2014 01:02:33 -0500 Subject: [PATCH 075/160] added award/penalty section in config --- config/{connect.karma.ini => karma.ini} | 31 ++++-- plugins/{connect.karma.js => karma.js} | 123 ++++++++++++++++-------- 2 files changed, 106 insertions(+), 48 deletions(-) rename config/{connect.karma.ini => karma.ini} (62%) rename plugins/{connect.karma.js => karma.js} (75%) diff --git a/config/connect.karma.ini b/config/karma.ini similarity index 62% rename from config/connect.karma.ini rename to config/karma.ini index 087483b95..b63af7f3a 100644 --- a/config/connect.karma.ini +++ b/config/karma.ini @@ -6,14 +6,21 @@ penalty_days = 1 -; concurrency limits. Comment out this block of settings to disable -; Using this feature and the rate_limit plugin may produce unexpected results -; Use one or the other. +; Redis is the database storage +[redis] +server_ip = 127.0.0.1 +server_port = 6379 + + +; concurrency limits. Using this *and* the rate_limit plugin may produce +; unexpected results. Use one or the other. +; Comment out this block of settings to disable [concurrency] naughty=1 neutral=2 nice=10 + [threshhold] ; Be conservative to avoid false positives! ; the threshhold below which a connection is considered naughty @@ -27,14 +34,9 @@ positive=2 history_negative=-3 -[redis] -server_ip = 127.0.0.1 -server_port = 6379 - - [spammy_tlds] ; award negative karma to spammy TLDs -; careful, awarding karma > msg_negative_limit may blacklist that TLD +; caution, awarding karma > msg_negative_limit may blacklist that TLD info=-4 pw=-4 tw=-3 @@ -47,3 +49,14 @@ jp=-2 no=-2 se=-2 sg=-2 + +; karma can award points based on other plugins results. +; the key is a note to inspect and the value is the karma award +[awards] +notes.auth_user=3 +notes.fcrdns.fcrdns.length=1 + +[penalties] +notes.fcrdns.no_rdns=-2 +notes.fcrdns.ip_in_rdns=-1 + diff --git a/plugins/connect.karma.js b/plugins/karma.js similarity index 75% rename from plugins/connect.karma.js rename to plugins/karma.js index 9af4fc113..6be616d1d 100644 --- a/plugins/connect.karma.js +++ b/plugins/karma.js @@ -9,6 +9,7 @@ exports.register = function () { this.register_hook('init_master', 'karma_onInit'); this.register_hook('init_child', 'karma_onInit'); + this.register_hook('deny', 'karma_onDeny'); this.register_hook('lookup_rdns', 'karma_onConnect'); this.register_hook('mail', 'karma_onMailFrom'); this.register_hook('rcpt', 'karma_onRcptTo'); @@ -18,7 +19,7 @@ exports.register = function () { }; exports.karma_onInit = function (next,server) { - var config = this.config.get('connect.karma.ini'); + var config = this.config.get('karma.ini'); var redis_ip = '127.0.0.1'; var redis_port = '6379'; if ( config.redis ) { @@ -31,16 +32,12 @@ exports.karma_onInit = function (next,server) { exports.karma_onConnect = function (next, connection) { var plugin = this; - var config = this.config.get('connect.karma.ini'); + var config = this.config.get('karma.ini'); - connection.notes.karma = { - connection: 0, - history: 0, - penalties: [ ], - }; + initConnectionNote(connection, config); - var key = 'karma|'+connection.remote_ip; - var con_key = 'concurrent|'+connection.remote_ip; + var r_ip = connection.remote_ip; + var key = 'karma|'+r_ip; function initRemoteIP () { db.multi() @@ -51,7 +48,7 @@ exports.karma_onConnect = function (next, connection) { }; db.multi() - .get(con_key) + .get('concurrent|'+r_ip) .hgetall(key) .exec( function redisResults (err,replies) { if (err) { @@ -70,7 +67,7 @@ exports.karma_onConnect = function (next, connection) { var summary = kobj.naughty+" naughty, "+kobj.nice+" nice, "+kobj.connections+" connects, "+history+" history"; - var too_many = checkConcurrency(plugin, con_key, replies[0], history); + var too_many = checkConcurrency(plugin, 'concurrent|'+r_ip, replies[0], history); if ( too_many ) { connection.loginfo(plugin, too_many + ", ("+summary+")"); return next(DENYSOFT, too_many); @@ -95,6 +92,17 @@ exports.karma_onConnect = function (next, connection) { }); }; +function initConnectionNote(connection, config) { + if (connection.notes.karma) return; + connection.notes.karma = { + connection: 0, + history: 0, + awards: [], + penalties: [ ], + todo: getTodo(config, connection), + }; +}; + exports.karma_onDeny = function (next, connection, params) { /* params ** [0] = plugin return value (constants.deny or constants.denysoft) @@ -109,8 +117,11 @@ exports.karma_onDeny = function (next, connection, params) { var plugin = this; var transaction = connection.transaction; + var config = this.config.get('karma.ini'); + initConnectionNote(connection, config); + connection.notes.karma.connection--; - connection.notes.karma.penalties.push(pi_plugin); + connection.notes.karma.penalties.push(pi_name); connection.loginfo(plugin, 'deny, '+karmaSummary(connection)); @@ -119,7 +130,7 @@ exports.karma_onDeny = function (next, connection, params) { exports.karma_onMailFrom = function (next, connection, params) { var plugin = this; - var config = this.config.get('connect.karma.ini'); + var config = this.config.get('karma.ini'); var mail_from = params[0]; var from_tld = mail_from.host.split('.').pop(); @@ -186,7 +197,7 @@ exports.karma_onRcptTo = function (next, connection, params) { exports.karma_onData = function (next, connection) { // cut off naughty senders at DATA to prevent receiving the message - var config = this.config.get('connect.karma.ini'); + var config = this.config.get('karma.ini'); var negative_limit = config.threshhold.negative || -5; var karma = connection.notes.karma * 1; @@ -206,7 +217,7 @@ exports.karma_onDataPost = function (next, connection) { exports.karma_onDisconnect = function (next, connection) { var plugin = this; - var config = this.config.get('connect.karma.ini'); + var config = this.config.get('karma.ini'); var key = 'karma|'+connection.remote_ip; @@ -224,33 +235,35 @@ exports.karma_onDisconnect = function (next, connection) { return next(); }; - var pos_lim = config.threshhold.positive || 2; + if (config.threshhold) { + var pos_lim = config.threshhold.positive || 2; - if (k.connection > pos_lim) { - db.hincrby(key, 'nice', 1); - connection.loginfo(plugin, "positive: "+karmaSummary(connection)); - return next(); - }; + if (k.connection > pos_lim) { + db.hincrby(key, 'nice', 1); + connection.loginfo(plugin, "positive: "+karmaSummary(connection)); + return next(); + }; - var negative_limit = config.threshhold.negative || -3; - if (k.connection < negative_limit) { - db.hincrby(key, 'naughty', 1); - // connection.notes.karma.penalties.push('history'); - history--; - - if (history <= config.threshhold.history_negative) { - if (history < -5) { - connection.loginfo(plugin, "penalty box bonus! "+karmaSummary(connection)); - log_mess = ", penalty box bonus!"; - db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); - } - else { - db.hset(key, 'penalty_start_ts', Date()); + var negative_limit = config.threshhold.negative || -3; + if (k.connection < negative_limit) { + db.hincrby(key, 'naughty', 1); + // connection.notes.karma.penalties.push('history'); + history--; + + if (history <= config.threshhold.history_negative) { + if (history < -5) { + connection.loginfo(plugin, "penalty box bonus! "+karmaSummary(connection)); + log_mess = ", penalty box bonus!"; + db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); + } + else { + db.hset(key, 'penalty_start_ts', Date()); + } + connection.loginfo(plugin, "penalty box! "+karmaSummary(connection)); + next(); } - connection.loginfo(plugin, "penalty box! "+karmaSummary(connection)); - next(); } - } + }; connection.loginfo(plugin, "no action, "+karmaSummary(connection)); next(); }; @@ -271,7 +284,7 @@ function addDays(date, days) { } function checkConcurrency(plugin, con_key, val, history) { - var config = plugin.config.get('connect.karma.ini'); + var config = plugin.config.get('karma.ini'); if ( !config.concurrency ) return; @@ -287,3 +300,35 @@ function checkConcurrency(plugin, con_key, val, history) { if (reject) return "too many connections for you: "+count; return; }; + +function getTodo(config, connection) { + var plugin = connection; + var awards = config.awards; + + connection.logdebug(plugin, "awards: "+awards); + var result = {}; + + if ( awards ) { + result['awards'] = awards; + +// for( var i=0; i < awards.keys; i++) { +// result['awards'][i] = awards[i]; +// }; + }; + + return result; + + var penalties = config.penalties; +}; + +function checkAwards ( ) { +/* +[awards] +connection.notes.auth_user=3 +connection.notes.fcrdns.fcrdns.length=1 + +[penalties] +connection.notes.fcrdns.no_rdns=-2 +connection.notes.fcrdns.ip_in_rdns=-1 +*/ +} From a934b835d9761935b9c66815f846c7927d514e3b Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Fri, 17 Jan 2014 13:03:19 -0500 Subject: [PATCH 076/160] Proof of ownership --- freenode.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 freenode.txt diff --git a/freenode.txt b/freenode.txt new file mode 100644 index 000000000..6d67dcc5f --- /dev/null +++ b/freenode.txt @@ -0,0 +1,3 @@ +8322c5dc2f5c6b7cd3ff602d64c025be23da0709 + +Proof of project ownership From d2c736295d1f7223e43347912124c780c1e14f20 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Fri, 17 Jan 2014 13:07:53 -0500 Subject: [PATCH 077/160] Delete freenode.txt --- freenode.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 freenode.txt diff --git a/freenode.txt b/freenode.txt deleted file mode 100644 index 6d67dcc5f..000000000 --- a/freenode.txt +++ /dev/null @@ -1,3 +0,0 @@ -8322c5dc2f5c6b7cd3ff602d64c025be23da0709 - -Proof of project ownership From 270ada380ba5f01c8157781379a0d15261b97fa5 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 17 Jan 2014 18:24:14 -0500 Subject: [PATCH 078/160] karma: added awards from other plugins based on inspecting their saved notes --- config/karma.ini | 2 +- plugins/karma.js | 86 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index b63af7f3a..67dd6b8c4 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -52,11 +52,11 @@ sg=-2 ; karma can award points based on other plugins results. ; the key is a note to inspect and the value is the karma award +; NOTE: karma awards can be positive or negative! [awards] notes.auth_user=3 notes.fcrdns.fcrdns.length=1 -[penalties] notes.fcrdns.no_rdns=-2 notes.fcrdns.ip_in_rdns=-1 diff --git a/plugins/karma.js b/plugins/karma.js index 6be616d1d..b2acedf7e 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -90,6 +90,8 @@ exports.karma_onConnect = function (next, connection) { return next(DENY, mess); }); + + checkAwards (config, connection, plugin); }; function initConnectionNote(connection, config) { @@ -99,7 +101,7 @@ function initConnectionNote(connection, config) { history: 0, awards: [], penalties: [ ], - todo: getTodo(config, connection), + todo: populateTodo(config, connection), }; }; @@ -125,6 +127,7 @@ exports.karma_onDeny = function (next, connection, params) { connection.loginfo(plugin, 'deny, '+karmaSummary(connection)); + checkAwards (config, connection, plugin); return next(); }; @@ -155,13 +158,13 @@ exports.karma_onMailFrom = function (next, connection, params) { connection.notes.karma.penalties.push('rfc5321.MailFrom'); }; + checkAwards (config, connection, plugin); connection.loginfo(plugin, karmaSummary(connection)); return next(); }; exports.karma_onRcptTo = function (next, connection, params) { var plugin = this; - var rcpt = params[0]; var full_rcpt = connection.current_line; @@ -189,6 +192,9 @@ exports.karma_onRcptTo = function (next, connection, params) { return next(); }; + var config = this.config.get('karma.ini'); + checkAwards (config, connection, plugin); + connection.loginfo(plugin, karmaSummary(connection)); // limit recipients if host has negative or unknown karma @@ -205,6 +211,7 @@ exports.karma_onData = function (next, connection) { return next(DENY, "very bad karma: "+karma); } + checkAwards (config, connection, this); return next(); } @@ -212,6 +219,8 @@ exports.karma_onDataPost = function (next, connection) { connection.transaction.add_header('X-Haraka-Karma', karmaSummary(connection) ); + var config = this.config.get('karma.ini'); + checkAwards (config, connection, this); return next(); } @@ -264,6 +273,7 @@ exports.karma_onDisconnect = function (next, connection) { } } }; + checkAwards (config, connection, plugin); connection.loginfo(plugin, "no action, "+karmaSummary(connection)); next(); }; @@ -274,6 +284,7 @@ function karmaSummary(c) { 'conn:'+k.connection+ ', hist: '+k.history+ ', penalties: '+k.penalties+ + ', awards: '+k.awards+ ')'; } @@ -301,34 +312,71 @@ function checkConcurrency(plugin, con_key, val, history) { return; }; -function getTodo(config, connection) { +function populateTodo(config, connection) { var plugin = connection; var awards = config.awards; - connection.logdebug(plugin, "awards: "+awards); +// toDo is a list of connection notes to 'watch' for. +// When discovered, we award their karma points and remove +// them from the ToDo list. + var result = {}; if ( awards ) { - result['awards'] = awards; - -// for( var i=0; i < awards.keys; i++) { -// result['awards'][i] = awards[i]; -// }; + Object.keys(awards).forEach(function(key) { + connection.logdebug(plugin, "key: "+key+", award: "+awards[key]); + result[key] = awards[key]; + }); }; return result; +}; - var penalties = config.penalties; +function assembleNoteObj(prefix,key) { + var note = prefix; + var parts = key.split('.'); + while(parts.length > 0) { + note = note[parts.shift()]; + if (note == null) break; + } + return note; }; -function checkAwards ( ) { -/* -[awards] -connection.notes.auth_user=3 -connection.notes.fcrdns.fcrdns.length=1 +function checkAwards (config, connection, plugin) { + if (!connection.notes.karma) return; + if (!connection.notes.karma.todo) return; -[penalties] -connection.notes.fcrdns.no_rdns=-2 -connection.notes.fcrdns.ip_in_rdns=-1 -*/ + if (!plugin) { plugin = connection }; + var awards = config.awards; + + Object.keys(connection.notes.karma.todo).forEach(function(key) { + + // assemble the object path using the note name + var note = assembleNoteObj(connection, key); + if (note == null || note === false) { + // connection.logdebug(plugin, "no connection note: "+key); + if (!connection.transaction) return; + var txn_note = assembleNoteObj(connection.transaction, key); + if (txn_note == null || txn_note === false) { + // connection.logdebug(plugin, "no transaction note: "+key); + return; + } + }; + + var karma_to_apply = connection.notes.karma.todo[key]; + + if ( karma_to_apply && Number(karma_to_apply) !== 'NaN' ) { + connection.notes.karma.connection += karma_to_apply; + + if ( karma_to_apply > 0 ) { + connection.notes.karma.awards.push(key); + } + else if ( karma_to_apply < 0 ) { + connection.notes.karma.penalties.push(key); + }; + + connection.loginfo(plugin, "applied karma: "+karma_to_apply); + delete connection.notes.karma.todo[key]; + }; + }); } From ddca3e68bec2f376495cec297031e2ed84811b15 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 00:09:01 -0500 Subject: [PATCH 079/160] edited big functions into paragraph sized ones added recipient limits to config added expire_days to config (was hard coded) --- config/karma.ini | 38 +++-- docs/plugins/karma.md | 204 +++++++------------------ plugins/karma.js | 335 +++++++++++++++++++++--------------------- 3 files changed, 248 insertions(+), 329 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index 67dd6b8c4..b5b5c7ec8 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -1,35 +1,47 @@ - -; nice: get a karma score greater than the positive connection limit -; naughty: achieve a karma score lower than the negative connection limit ; -; how many days to penalize naughty senders +; good: get a karma score greater than the positive connection limit +; bad: achieve a karma score lower than the negative connection limit +; +; how many days to penalize bad senders penalty_days = 1 +; each IPs karma history is expired after this many days +; Note: this value is refreshed upon each connection. Frequent +; senders karma may never expire. +expire_days = 60 -; Redis is the database storage +; Redis is our super-lightweight key/value store [redis] server_ip = 127.0.0.1 server_port = 6379 -; concurrency limits. Using this *and* the rate_limit plugin may produce -; unexpected results. Use one or the other. +; concurrency limits. Limit how many concurrent connections an IP can make. +; Caution: Using this *and* the rate_limit plugin may produce unexpected +; results. Use one or the other. ; Comment out this block of settings to disable [concurrency] -naughty=1 +bad=1 neutral=2 -nice=10 +good=10 + + +; maximum number of recipients allowed +[recipients] +bad=1 +neutral=3 +good=10 [threshhold] ; Be conservative to avoid false positives! -; the threshhold below which a connection is considered naughty +; the threshhold below which a connection is considered bad negative=-3 -; score at which a connection is considered nice +; score at which a connection is considered good positive=2 -; karma history = nice - naughty connections. To achieve a negative score, -; senders must send more naughty than nice messages. Is it worth getting +; karma history = good - bad connections. To achieve a negative score, +; senders must send more bad than good messages. Is it worth getting ; 5 spam and 2 ham? Adjust this knob accordingly. history_negative=-3 diff --git a/docs/plugins/karma.md b/docs/plugins/karma.md index eae7b5790..07920ab26 100644 --- a/docs/plugins/karma.md +++ b/docs/plugins/karma.md @@ -1,199 +1,102 @@ -karma - reward nice and penalize naughty mail senders +karma - reward good and penalize bad mail senders =========================== +Karma tracks sender history and varyies QoS for based on the senders reputation. -Karma tracks sender history, allowing varying QoS -to naughty, nice, and unknown senders. -DESCRIPTION ------------------------ -Karma records the number of nice, naughty, and total connections from mail -senders. After sending a naughty message, if a sender has more naughty than -nice connections, they are penalized for *penalty_days*. Connections -from senders in the penalty box are rejected. - -Karma stores two connection notes that other plugins can use to be more -lenient or strict. - - connection.notes.karma - karma score on *this* connection - connection.notes.karma_history - karma history - -Karma history is computed as the number of nice - naughty connections. +SYNOPSIS +--------------------------- +Karma can be used to craft custom connection policies such as these examples: -Karma is small, fast, and ruthlessly efficient. Karma can be used to craft -custom connection policies such as these two examples: +1. Hi well known and well behaved sender. Help yourself to greater concurrency, + more recipients, and no delays. -1. Hi well known and well behaved sender. Help yourself to greater - concurrency (hosts_allow), multiple recipients (karma), and no - delays (early_sender). - -2. Hi there, naughty sender. You get a max concurrency of 1, max recipients +2. Hi there, bad sender. You get a max concurrency of 1, max recipients of 2, and SMTP delays. -CONFIG -==================== - -negative --------------------- -How negative a senders karma can get before we penalize them. - -Default: 2 - -Examples: - - negative 1: 0 nice - 1 naughty = karma -1, penalize - negative 1: 1 nice - 1 naughty = karma 0, okay - negative 2: 1 nice - 2 naughty = karma -1, okay - negative 2: 1 nice - 3 naughty = karma -2, penalize - -With the default negative limit of one, there's a very small chance you could -penalize a "mostly good" sender. Raising it to 2 reduces that possibility to -improbable. - -penalty_days --------------------- - -The number of days a naughty sender is refused connections. Use a decimal -value to penalize for portions of days. - - karma penalty_days 1 +DESCRIPTION +----------------------- +Karma records the number of good, bad, and total connections. When a sender +has more bad than good connections, they are penalized for *penalty_days*. +Connections from senders in the penalty box are rejected until the penalty +expires. -Default: 1 +Karma stores a connection note (*connection.notes.karma*) that other +plugins can use. It contains the following; -reject [ 0 | 1 ] -------------------- -0 will not reject any connections. -1 will reject naughty senders. + connection: 0, <- score for this connection + history: 0, <- score for all connections + awards: [], <- tests that added positive karma + penalties: [ ], <- tests that added negative karma -db_dir --------------------- -Path to a directory in which the DB will be stored. This directory must be -writable by the qpsmtpd user. If unset, the first usable directory from the -following list will be used: +HISTORY +----------------------- +Karma history is computed as the number of good - bad connections. - /var/lib/qpsmtpd/karma - BINDIR/var/db (where BINDIR is the location of the qpsmtpd binary) +CONFIG +==================== - BINDIR/config +See config/karma.ini. It has lots of options and inline documentation. BENEFITS -------------------- -Karma reduces the resources wasted by naughty mailers. +Karma reduces the resources wasted by bad mailers. -The biggest gains to be had are by having heavy plugins (spamassassin, dspam, -virus filters) set the _karma_ connection note (see KARMA) when they encounter -naughty senders. Reasons to send servers to the penalty box could include -sending a virus, early talking, or sending messages with a very high spam -score. +The biggest gains to be had are by assigning lots of negative karma by the +heavy plugins (spamassassin, dspam, virus filters) when they encounter spam. +Karma will notice and reward them appropriately in the future. -This plugin does not penalize connections with transaction notes I -or I set. These notes would have been set by the B, -B, and B plugins. Obviously, those plugins must -run before B for that to work. KARMA ------------------------ +When the connection ends, B records the result. Mail servers whose +bad connections exceed good ones are sent to the penalty box. Servers in +the penalty box are tersely disconnected for *penalty_days*. Here is +an example connection from an IP in the penalty box: -It is mostly up to other plugins to reward well behaved senders with positive -karma and smite poorly behaved senders with negative karma. -See B +If only negative karma is set, desirable mailers will be penalized. For +example, a Yahoo user sends an egregious spam to a user on our server. +Now nobody on our server can receive email from that Yahoo server for +*penalty_days*. This will happen approximately 0% of the time if we also +set positive karma. -After the connection ends, B will record the result. Mail servers whose -naughty connections exceed nice ones are sent to the penalty box. Servers in -the penalty box will be tersely disconnected for I. Here is -an example connection from an IP in the penalty box: - 73122 Connection from smtp.midsetmediacorp.com [64.185.226.65] - 73122 (connect) ident::geoip: US, United States - 73122 (connect) ident::p0f: Windows 7 or 8 - 73122 (connect) earlytalker: pass: 64.185.226.65 said nothing spontaneous - 73122 (connect) relay: skip: no match - 73122 (connect) karma: fail - 73122 550 You were naughty. You are cannot connect for 0.99 more days. - 73122 click, disconnecting - 73122 (post-connection) connection_time: 1.048 s. - -If we only set negative karma, we will almost certainly penalize servers we -want to receive mail from. For example, a Yahoo user sends an egregious spam -to a user on our server. Now nobody on our server can receive email from that -Yahoo server for I. This should happen approximately 0% of -the time if we are careful to also set positive karma. - -KARMA HISTORY +KARMA BONUS ------------------------ Karma maintains a history for each IP. When a senders history has decreased below -5 and they have never sent a good message, they get a karma bonus. -The bonus tacks on an extra day of blocking for every naughty message they +The bonus tacks on an extra day of blocking for every bad message they send. -Example: an unknown sender delivers a spam. They get a one day penalty_box. -After 5 days, 5 spams, 5 penalties, and 0 nice messages, they get a six day +Example: an unknown sender delivers a spam. They get a one day penalty. +After 5 days, 5 spams, 5 penalties, and 0 good messages, they get a six day penalty. The next offense gets a 7 day penalty, and so on. + USING KARMA ----------------------- -To get rid of naughty connections as fast as possible, run karma before other -connection plugins. Plugins that trigger DNS lookups or impose time delays -should run after B. In this example, karma runs before all but the -ident plugins. - - 89011 Connection from Unknown [69.61.27.204] - 89011 (connect) ident::geoip: US, United States - 89011 (connect) ident::p0f: Linux 3.x - 89011 (connect) karma: fail, 1 naughty, 0 nice, 1 connects - 89011 550 You were naughty. You are penalized for 0.99 more days. - 89011 click, disconnecting - 89011 (post-connection) connection_time: 0.118 s. - 88798 cleaning up after 89011 - -Unlike RBLs, B only penalizes IPs that have sent us spam, and only when +Unlike RBLs, *karma* only penalizes IPs that have sent us spam, and only when those senders have sent us more spam than ham. -USING KARMA IN OTHER PLUGINS ------------------------------- -This plugin sets the connection note I. Your plugin can -use the senders karma to be more gracious or rude to senders. The value of -I is the number of nice connections minus naughty -ones. The higher the number, the better you should treat the sender. - -To alter a connections karma based on its behavior, do this: - - $self->adjust_karma( -1 ); # lower karma (naughty) - $self->adjust_karma( 1 ); # raise karma (good) - EFFECTIVENESS --------------------- - -In the first 24 hours, _karma_ rejected 8% of all connections. After one -week of running with I, karma has rejected 15% of all -connections. - -This plugins effectiveness results from the propensity of naughty senders -to be repeat offenders. Limiting them to a single offense per day(s) greatly -reduces the resources they can waste. +Effectiveness results from the propensity of bad senders to be repeat +offenders. Limiting them to a single offense per day(s) greatly reduces +the resources they can waste. Of the connections that had previously passed all other checks and were caught -only by spamassassin and/or dspam, B rejected 31 percent. Since -spamassassin and dspam consume more resources than others plugins, this plugin -seems to be a very big win. - -DATABASE ---------------------- - -Connection summaries are stored in a database. The DB value is a : delimited -list containing a penalty box start time (if the server is/was on timeout) -and the count of naughty, nice, and total connections. The database can be -listed and searched with the karma_tool script. +only by spamassassin and/or dspam, karma rejected 31 percent. Since +spamassassin and dspam consume far more resources than karma, this plugin +can be a very big win. BUGS & LIMITATIONS --------------------- - This plugin is reactionary. Like the FBI, it doesn't do much until after a crime has been committed. @@ -201,5 +104,8 @@ There is little to be gained by listing servers that are already on DNS blacklists, send to invalid users, earlytalkers, etc. Those already have very lightweight tests. -* some type of ASN integration, for tracking karma of 'neighborhoods' + +TODO +----------------------- +* ASN integration, for tracking the karma of 'neighborhoods' diff --git a/plugins/karma.js b/plugins/karma.js index b2acedf7e..3a6d0cb62 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -1,4 +1,4 @@ -// karma - reward nice and penalize naughty mail senders +// karma - reward good and penalize bad mail senders var ipaddr = require('ipaddr.js'); var redis = require('redis'); @@ -23,8 +23,8 @@ exports.karma_onInit = function (next,server) { var redis_ip = '127.0.0.1'; var redis_port = '6379'; if ( config.redis ) { - redis_ip = config.redis.server_ip; - redis_port = config.redis.server_port; + redis_ip = config.redis.server_ip || '127.0.0.1'; + redis_port = config.redis.server_port || '6379'; }; db = redis.createClient(redis_port, redis_ip); return next(); @@ -37,19 +37,20 @@ exports.karma_onConnect = function (next, connection) { initConnectionNote(connection, config); var r_ip = connection.remote_ip; - var key = 'karma|'+r_ip; + var dbkey = 'karma|'+r_ip; + var expire = (config.main.expire_days || 60) * 86400; // convert to days function initRemoteIP () { db.multi() - .hmset(key, {'penalty_start_ts': 0, 'naughty': 0, 'nice': 0, 'connections': 1}) - .expire(key, 86400 * 60) // expire after 60 days + .hmset(dbkey, {'penalty_start_ts': 0, 'bad': 0, 'good': 0, 'connections': 1}) + .expire(dbkey, expire) .exec(); connection.logdebug(plugin,"first connect"); }; db.multi() .get('concurrent|'+r_ip) - .hgetall(key) + .hgetall(dbkey) .exec( function redisResults (err,replies) { if (err) { connection.logdebug(plugin,"err: "+err); @@ -58,14 +59,15 @@ exports.karma_onConnect = function (next, connection) { if (replies[1] === null) { initRemoteIP(); return next(); }; - db.hincrby(key, 'connections', 1); // total connections - db.expire(key, 86400 * 60); // extend expiration date + db.hincrby(dbkey, 'connections', 1); // increment total connections + db.expire(dbkey, expire); // extend expiration date - var kobj = replies[1]; - var history = (kobj.nice || 0) - (kobj.naughty || 0); + var dbr = replies[1]; // 2nd element of DB reply is our karma object + var history = (dbr.good || 0) - (dbr.bad || 0); connection.notes.karma.history = history; - var summary = kobj.naughty+" naughty, "+kobj.nice+" nice, "+kobj.connections+" connects, "+history+" history"; + var summary = dbr.bad+" bad, "+dbr.good+" good, " + +dbr.connections+" connects, "+history+" history"; var too_many = checkConcurrency(plugin, 'concurrent|'+r_ip, replies[0], history); if ( too_many ) { @@ -73,15 +75,15 @@ exports.karma_onConnect = function (next, connection) { return next(DENYSOFT, too_many); }; - if (kobj.penalty_start_ts === '0') { - connection.loginfo(plugin, "no penalty "+karmaSummary(connection)); + if (dbr.penalty_start_ts === '0') { + connection.loginfo(plugin, "no penalty, "+karmaSummary(connection)); return next(); } - var days_old = (Date.now() - Date.parse(kobj.penalty_start_ts)) / 86.4; + var days_old = (Date.now() - Date.parse(dbr.penalty_start_ts)) / 86.4; var penalty_days = config.main.penalty_days; if (days_old >= penalty_days) { - connection.loginfo(plugin, "penalty expired "+karmaSummary(connection)); + connection.loginfo(plugin, "penalty expired, "+karmaSummary(connection)); return next(); } @@ -91,18 +93,7 @@ exports.karma_onConnect = function (next, connection) { return next(DENY, mess); }); - checkAwards (config, connection, plugin); -}; - -function initConnectionNote(connection, config) { - if (connection.notes.karma) return; - connection.notes.karma = { - connection: 0, - history: 0, - awards: [], - penalties: [ ], - todo: populateTodo(config, connection), - }; + checkAwards(config, connection, plugin); }; exports.karma_onDeny = function (next, connection, params) { @@ -122,12 +113,13 @@ exports.karma_onDeny = function (next, connection, params) { var config = this.config.get('karma.ini'); initConnectionNote(connection, config); +// TO CONSIDER: decrement karma two points for a 5XX deny? connection.notes.karma.connection--; connection.notes.karma.penalties.push(pi_name); connection.loginfo(plugin, 'deny, '+karmaSummary(connection)); - checkAwards (config, connection, plugin); + checkAwards(config, connection, plugin); return next(); }; @@ -135,30 +127,10 @@ exports.karma_onMailFrom = function (next, connection, params) { var plugin = this; var config = this.config.get('karma.ini'); - var mail_from = params[0]; - var from_tld = mail_from.host.split('.').pop(); - connection.logdebug(plugin, "from_tld: "+from_tld); - - if ( config.spammy_tlds ) { - var tld_penalty = (config.spammy_tlds[from_tld] || 0) * 1; // force numeric - - if (tld_penalty !== 0) { - connection.loginfo(plugin, "spammy TLD award: "+tld_penalty); - connection.notes.karma.connection += tld_penalty; - }; - }; - - var full_from = connection.current_line; - connection.logdebug(plugin, "mail_from: "+full_from); - -// test if sender has placed an illegal (RFC 5321,2821,821) space in envelope from - if ( full_from.toUpperCase().substring(0,11) !== 'MAIL FROM:<' ) { - connection.loginfo(plugin, "illegal envelope address format: "+full_from ); - connection.notes.karma.connection--; - connection.notes.karma.penalties.push('rfc5321.MailFrom'); - }; + checkSpammyTLD(params[0], config, connection, plugin); + checkSyntaxMailFrom(connection, plugin); - checkAwards (config, connection, plugin); + checkAwards(config, connection, plugin); connection.loginfo(plugin, karmaSummary(connection)); return next(); }; @@ -166,52 +138,28 @@ exports.karma_onMailFrom = function (next, connection, params) { exports.karma_onRcptTo = function (next, connection, params) { var plugin = this; var rcpt = params[0]; - var full_rcpt = connection.current_line; - - // check for an illegal RFC (2)821 space in envelope recipient - if ( full_rcpt.toUpperCase().substring(0,9) !== 'RCPT TO:<' ) { - connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); - connection.notes.karma.connection--; - connection.notes.karma.penalties.push('rfc5321.RcptTo'); - }; - - var count = connection.rcpt_count.accept + connection.rcpt_count.tempfail + connection.rcpt_count.reject + 1; - if ( count <= 1 ) return next(); - - connection.loginfo(plugin, "recipient count: "+count ); - - var history = connection.notes.karma.history; - if ( history > 0 ) { - connection.loginfo(plugin, "good history"); - return next(); - }; - - var karma = connection.notes.karma.connection; - if ( karma > 0 ) { - connection.loginfo(plugin, "good connection"); - return next(); - }; - var config = this.config.get('karma.ini'); - checkAwards (config, connection, plugin); - connection.loginfo(plugin, karmaSummary(connection)); + checkSyntaxRcptTo(connection, plugin); + var too_many = checkMaxRecipients(connection, plugin, config); + if (too_many) return next(DENY, too_many); - // limit recipients if host has negative or unknown karma - return next(DENY, "too many recipients for poor karma: "+karmaSummary(connection)); -} + checkAwards(config, connection, plugin); + connection.loginfo(plugin, karmaSummary(connection)); + return next(); +}; exports.karma_onData = function (next, connection) { -// cut off naughty senders at DATA to prevent receiving the message +// cut off bad senders at DATA to prevent transferring the message var config = this.config.get('karma.ini'); var negative_limit = config.threshhold.negative || -5; var karma = connection.notes.karma * 1; - if ( karma.connection <= negative_limit ) { - return next(DENY, "very bad karma: "+karma); + if (karma.connection <= negative_limit) { + return next(DENY, "very bad karma score: "+karma); } - checkAwards (config, connection, this); + checkAwards(config, connection, this); return next(); } @@ -220,7 +168,7 @@ exports.karma_onDataPost = function (next, connection) { karmaSummary(connection) ); var config = this.config.get('karma.ini'); - checkAwards (config, connection, this); + checkAwards(config, connection, this); return next(); } @@ -228,64 +176,59 @@ exports.karma_onDisconnect = function (next, connection) { var plugin = this; var config = this.config.get('karma.ini'); - var key = 'karma|'+connection.remote_ip; - if ( config.concurrency ) db.incrby('concurrent|'+connection.remote_ip, -1); var k = connection.notes.karma; - if ( !k ) { - connection.loginfo(plugin, "error: karma note missing!"); - return next(); - }; - var history = k.history; + if (!k) { connection.logerror(plugin, "karma note missing!"); return next(); }; - if ( !k.connection ) { + if (!k.connection) { connection.loginfo(plugin, "neutral: "+karmaSummary(connection)); return next(); }; + var key = 'karma|'+connection.remote_ip; + var history = k.history; + if (config.threshhold) { var pos_lim = config.threshhold.positive || 2; if (k.connection > pos_lim) { - db.hincrby(key, 'nice', 1); + db.hincrby(key, 'good', 1); connection.loginfo(plugin, "positive: "+karmaSummary(connection)); return next(); }; - var negative_limit = config.threshhold.negative || -3; - if (k.connection < negative_limit) { - db.hincrby(key, 'naughty', 1); - // connection.notes.karma.penalties.push('history'); + var bad_limit = config.threshhold.negative || -3; + if (k.connection < bad_limit) { + db.hincrby(key, 'bad', 1); history--; if (history <= config.threshhold.history_negative) { if (history < -5) { - connection.loginfo(plugin, "penalty box bonus! "+karmaSummary(connection)); - log_mess = ", penalty box bonus!"; db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); + connection.loginfo(plugin, "penalty box bonus!: "+karmaSummary(connection)); } else { db.hset(key, 'penalty_start_ts', Date()); + connection.loginfo(plugin, "penalty box: "+karmaSummary(connection)); } - connection.loginfo(plugin, "penalty box! "+karmaSummary(connection)); - next(); + return next(); } } }; - checkAwards (config, connection, plugin); + checkAwards(config, connection, plugin); connection.loginfo(plugin, "no action, "+karmaSummary(connection)); - next(); + return next(); }; function karmaSummary(c) { var k = c.notes.karma; - return '('+ - 'conn:'+k.connection+ - ', hist: '+k.history+ - ', penalties: '+k.penalties+ - ', awards: '+k.awards+ - ')'; + return '(' + +'conn:'+k.connection + +', hist: '+k.history + +(k.penalties.length ? ', penalties: '+k.penalties : '') + +(k.awards.length ? ', awards: ' +k.awards : '') + +')'; } function addDays(date, days) { @@ -294,6 +237,40 @@ function addDays(date, days) { return result; } +function checkAwards (config, connection, plugin) { + if (!connection.notes.karma) return; + if (!connection.notes.karma.todo) return; + + if (!plugin) { plugin = connection }; + var awards = config.awards; + + Object.keys(connection.notes.karma.todo).forEach(function(key) { + + // assemble the object path using the note name + var note = assembleNoteObj(connection, key); + if (note == null || note === false) { + // connection.logdebug(plugin, "no connection note: "+key); + if (!connection.transaction) return; + var txn_note = assembleNoteObj(connection.transaction, key); + if (txn_note == null || txn_note === false) { + // connection.logdebug(plugin, "no transaction note: "+key); + return; + } + }; + + var karma_to_apply = connection.notes.karma.todo[key]; + if (!karma_to_apply) return; + if ( Number(karma_to_apply) === 'NaN' ) return; // garbage in config + + connection.notes.karma.connection += karma_to_apply; + connection.loginfo(plugin, "applied "+key+" karma: "+karma_to_apply); + delete connection.notes.karma.todo[key]; + + if ( karma_to_apply > 0 ) { connection.notes.karma.awards.push(key); }; + if ( karma_to_apply < 0 ) { connection.notes.karma.penalties.push(key); }; + }); +} + function checkConcurrency(plugin, con_key, val, history) { var config = plugin.config.get('karma.ini'); @@ -305,31 +282,94 @@ function checkConcurrency(plugin, con_key, val, history) { db.expire(con_key, 4 * 60); // expire after 4 min var reject=0; - if (history < 0 && count > config.concurrency.naughty) reject++; - if (history > 0 && count > config.concurrency.nice) reject++; + if (history < 0 && count > config.concurrency.bad) reject++; + if (history > 0 && count > config.concurrency.good) reject++; if (history == 0 && count > config.concurrency.neutral) reject++; if (reject) return "too many connections for you: "+count; return; }; -function populateTodo(config, connection) { - var plugin = connection; - var awards = config.awards; +function checkMaxRecipients(connection, plugin, config) { + if (!config.recipients) return; // disabled in config file + var c = connection.rcpt_count; + var count = c.accept + c.tempfail + c.reject + 1; + if ( count <= 1 ) return; // everybody is allowed one -// toDo is a list of connection notes to 'watch' for. -// When discovered, we award their karma points and remove -// them from the ToDo list. + connection.logdebug(plugin, "recipient count: "+count ); - var result = {}; + var desc = history > 3 ? 'good' : history >= 0 ? 'neutral' : 'bad'; - if ( awards ) { - Object.keys(awards).forEach(function(key) { - connection.logdebug(plugin, "key: "+key+", award: "+awards[key]); - result[key] = awards[key]; - }); + var cr = config.recipients; + + // the deeds of their past shall not go unnoticed! + var history = connection.notes.karma.history; + if ( history > 3 && count <= cr.good) return; + if ( history > -1 && count <= cr.neutral) return; + + // this is *more* strict than history, b/c they have fewer opportunity + // to score positive karma this early in the connection. senders with + // good history will rarely see these limits. + var karma = connection.notes.karma.connection; + if ( karma > 3 && count <= cr.good) return; + if ( karma >= 0 && count <= cr.neutral) return; + + return 'too many recipients ('+count+') for '+desc+' karma'; +} + +function checkSpammyTLD(mail_from, config, connection, plugin) { + if (!config.spammy_tlds) return; + + var from_tld = mail_from.host.split('.').pop(); + // connection.logdebug(plugin, "from_tld: "+from_tld); + + var tld_penalty = (config.spammy_tlds[from_tld] || 0) * 1; // force numeric + if (tld_penalty === 0) return; + + connection.loginfo(plugin, "spammy TLD: "+tld_penalty); + connection.notes.karma.connection += tld_penalty; + connection.notes.karma.penalties.push('spammy.TLD'); +}; + +function checkSyntaxMailFrom(connection, plugin) { + var full_from = connection.current_line; + // connection.logdebug(plugin, "mail_from: "+full_from); + +// look for an illegal (RFC 5321,2821,821) space in envelope from + if (full_from.toUpperCase().substring(0,11) === 'MAIL FROM:<' ) return; + + connection.loginfo(plugin, "illegal envelope address format: "+full_from ); + connection.notes.karma.connection--; + connection.notes.karma.penalties.push('rfc5321.MailFrom'); +}; + +function checkSyntaxRcptTo(connection, plugin) { + // check for an illegal RFC (2)821 space in envelope recipient + var full_rcpt = connection.current_line; + if ( full_rcpt.toUpperCase().substring(0,9) === 'RCPT TO:<' ) return; + + connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); + connection.notes.karma.connection--; + connection.notes.karma.penalties.push('rfc5321.RcptTo'); +}; + +function initConnectionNote(connection, config) { + if (connection.notes.karma) return; // init once per connection + connection.notes.karma = { + connection: 0, + history: 0, + awards: [], + penalties: [ ], + todo: {}, }; - return result; + // todo is a list of connection/transaction notes to 'watch' for. + // When discovered, award their karma points to the connection + // and remove them from todo. + var awards = config.awards; + if (!awards) return; + Object.keys(awards).forEach(function(key) { + connection.notes.karma.todo[key] = awards[key]; + }); }; function assembleNoteObj(prefix,key) { @@ -341,42 +381,3 @@ function assembleNoteObj(prefix,key) { } return note; }; - -function checkAwards (config, connection, plugin) { - if (!connection.notes.karma) return; - if (!connection.notes.karma.todo) return; - - if (!plugin) { plugin = connection }; - var awards = config.awards; - - Object.keys(connection.notes.karma.todo).forEach(function(key) { - - // assemble the object path using the note name - var note = assembleNoteObj(connection, key); - if (note == null || note === false) { - // connection.logdebug(plugin, "no connection note: "+key); - if (!connection.transaction) return; - var txn_note = assembleNoteObj(connection.transaction, key); - if (txn_note == null || txn_note === false) { - // connection.logdebug(plugin, "no transaction note: "+key); - return; - } - }; - - var karma_to_apply = connection.notes.karma.todo[key]; - - if ( karma_to_apply && Number(karma_to_apply) !== 'NaN' ) { - connection.notes.karma.connection += karma_to_apply; - - if ( karma_to_apply > 0 ) { - connection.notes.karma.awards.push(key); - } - else if ( karma_to_apply < 0 ) { - connection.notes.karma.penalties.push(key); - }; - - connection.loginfo(plugin, "applied karma: "+karma_to_apply); - delete connection.notes.karma.todo[key]; - }; - }); -} From 71eceb710255e201a6798de1be4936b90224f472 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 02:07:41 -0500 Subject: [PATCH 080/160] karma config: added relay award --- config/karma.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/karma.ini b/config/karma.ini index b5b5c7ec8..de20fc067 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -66,7 +66,8 @@ sg=-2 ; the key is a note to inspect and the value is the karma award ; NOTE: karma awards can be positive or negative! [awards] -notes.auth_user=3 +relaying=1 +notes.auth_user=2 notes.fcrdns.fcrdns.length=1 notes.fcrdns.no_rdns=-2 From c8a7763bc800318576b4d5c1493f0ab4d92d2ac5 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 00:42:18 -0500 Subject: [PATCH 081/160] karma: awards for notes with defined values ie, match connection.notes.spamassassin.flag===Yes --- config/karma.ini | 6 +++++- plugins/karma.js | 21 ++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index de20fc067..d4006f57e 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -65,11 +65,15 @@ sg=-2 ; karma can award points based on other plugins results. ; the key is a note to inspect and the value is the karma award ; NOTE: karma awards can be positive or negative! +; Any true value in the specified note will match. +; A required value in the note can be specified with an @ postfix +this special syntax '@' [awards] relaying=1 notes.auth_user=2 notes.fcrdns.fcrdns.length=1 +notes.rdns_access@white=1 notes.fcrdns.no_rdns=-2 notes.fcrdns.ip_in_rdns=-1 - +notes.spamassassin.flag@Yes=-2 diff --git a/plugins/karma.js b/plugins/karma.js index 3a6d0cb62..81f4cff45 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -240,29 +240,36 @@ function addDays(date, days) { function checkAwards (config, connection, plugin) { if (!connection.notes.karma) return; if (!connection.notes.karma.todo) return; - - if (!plugin) { plugin = connection }; + if (!plugin) plugin = connection; var awards = config.awards; Object.keys(connection.notes.karma.todo).forEach(function(key) { + var e = key.split('@').slice(0,2); + var suffix = e[0]; + var wants = e[1]; // assemble the object path using the note name - var note = assembleNoteObj(connection, key); + var note = assembleNoteObj(connection, suffix); if (note == null || note === false) { // connection.logdebug(plugin, "no connection note: "+key); if (!connection.transaction) return; - var txn_note = assembleNoteObj(connection.transaction, key); - if (txn_note == null || txn_note === false) { + note = assembleNoteObj(connection.transaction, suffix); + if (note == null || note === false) { // connection.logdebug(plugin, "no transaction note: "+key); return; } }; + if (wants && note && (wants !== note)) { + connection.logdebug(plugin, "key "+suffix+" wants: "+wants+" but got: "+note); + return; + }; + var karma_to_apply = connection.notes.karma.todo[key]; if (!karma_to_apply) return; if ( Number(karma_to_apply) === 'NaN' ) return; // garbage in config - connection.notes.karma.connection += karma_to_apply; + connection.notes.karma.connection += karma_to_apply * 1; connection.loginfo(plugin, "applied "+key+" karma: "+karma_to_apply); delete connection.notes.karma.todo[key]; @@ -372,7 +379,7 @@ function initConnectionNote(connection, config) { }); }; -function assembleNoteObj(prefix,key) { +function assembleNoteObj(prefix, key) { var note = prefix; var parts = key.split('.'); while(parts.length > 0) { From 67a481503b95f54b71db839ad50ef95299aa1e69 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 01:39:17 -0500 Subject: [PATCH 082/160] karma: added neg karma for SPF deny/fail --- config/karma.ini | 8 ++++++-- plugins/rcpt_to.max_count.js | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index d4006f57e..aa533248f 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -29,8 +29,8 @@ good=10 ; maximum number of recipients allowed [recipients] bad=1 -neutral=3 -good=10 +neutral=5 +good=20 [threshhold] @@ -77,3 +77,7 @@ notes.rdns_access@white=1 notes.fcrdns.no_rdns=-2 notes.fcrdns.ip_in_rdns=-1 notes.spamassassin.flag@Yes=-2 +notes.spf_helo@3=-2 +notes.spf_helo@4=-1 +notes.spf_helo@6=-1 +notes.spf_helo@7=-1 diff --git a/plugins/rcpt_to.max_count.js b/plugins/rcpt_to.max_count.js index 578478a2c..fe415e984 100644 --- a/plugins/rcpt_to.max_count.js +++ b/plugins/rcpt_to.max_count.js @@ -2,6 +2,9 @@ // this helps guard against some spammers who send RCPT TO a gazillion times // as a way of probing for a working address +// Consider using the karma plugin. It supports limiting the number +// of recipients based on past behavior (good, bad, unknown) + exports.hook_rcpt = function (next, connection) { if (connection.transaction.notes.rcpt_to_count) { connection.transaction.notes.rcpt_to_count++; From 6751c21c967988cccfae2cf0911eb72f72fa76cb Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 02:42:02 -0500 Subject: [PATCH 083/160] karma: check mail_from for null sender before attempting to extract the host --- plugins/karma.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/karma.js b/plugins/karma.js index 81f4cff45..c30a4ebfb 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -325,8 +325,9 @@ function checkMaxRecipients(connection, plugin, config) { function checkSpammyTLD(mail_from, config, connection, plugin) { if (!config.spammy_tlds) return; + if (mail_from.isNull()) return; // null sender (bounce) - var from_tld = mail_from.host.split('.').pop(); + var from_tld = mail_from.host.split('.').pop(); // connection.logdebug(plugin, "from_tld: "+from_tld); var tld_penalty = (config.spammy_tlds[from_tld] || 0) * 1; // force numeric From 372a60600586e936c1f44d40212b2e6ec25e6ed1 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 03:14:28 -0500 Subject: [PATCH 084/160] SPF: bug fix, adding missing parens so that the auth_results header shows up properly --- plugins/spf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/spf.js b/plugins/spf.js index 2f94039d1..c8b29d13b 100644 --- a/plugins/spf.js +++ b/plugins/spf.js @@ -55,7 +55,7 @@ exports.hook_mail = function (next, connection, params) { var auth_result; if (connection.notes.spf_helo) { - auth_result = spf.result(connection.notes.spf_helo).toLowerCase; + auth_result = spf.result(connection.notes.spf_helo).toLowerCase(); // Add a trace header txn.add_leading_header('Received-SPF', spf.result(connection.notes.spf_helo) + From a332e0e5809d94919b09bf554f6e053c1e6bb2ce Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 03:24:34 -0500 Subject: [PATCH 085/160] moved outbound rate limiting into outbound section --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 6f025ea23..0371f1e03 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,3 @@ -- Rate Limiting for outbound mail (there's a branch for this but it's incomplete) - Milter support - Ability to modify the body of email (e.g add a banner) - Create a config file for each of the core shipping configs, so people have something as a baseline @@ -11,6 +10,7 @@ - virus/* Outbound improvements + - Rate Limiting (there's a branch for this but it's incomplete) - Provide better command line tools for manipulating/inspecting the queue - Add the ability to force a run on a specific queue file or destination domain - Make retry times configurable (handle RFC requirements for 5 days and DSN queued warnings) From 322e66676192efd921f3b08ca628e311d2a417ed Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 03:43:16 -0500 Subject: [PATCH 086/160] karma: trim 'notes' prefix off test names removes repetitive and meaningless data from headers --- plugins/karma.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/karma.js b/plugins/karma.js index c30a4ebfb..bb06a099c 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -273,8 +273,9 @@ function checkAwards (config, connection, plugin) { connection.loginfo(plugin, "applied "+key+" karma: "+karma_to_apply); delete connection.notes.karma.todo[key]; - if ( karma_to_apply > 0 ) { connection.notes.karma.awards.push(key); }; - if ( karma_to_apply < 0 ) { connection.notes.karma.penalties.push(key); }; + var trimmed = key.substring(0,5) === 'notes' ? key.substring(6) : key; + if ( karma_to_apply > 0 ) { connection.notes.karma.awards.push(trimmed); }; + if ( karma_to_apply < 0 ) { connection.notes.karma.penalties.push(trimmed); }; }); } From 874d08767dee907c1d297e666acf1f0d2a88e161 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 12:46:41 -0500 Subject: [PATCH 087/160] rm mail_from.is_resolvable.timeout this is integrated into the main .ini, no longer used --- config/mail_from.is_resolvable.timeout | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/mail_from.is_resolvable.timeout diff --git a/config/mail_from.is_resolvable.timeout b/config/mail_from.is_resolvable.timeout deleted file mode 100644 index 573541ac9..000000000 --- a/config/mail_from.is_resolvable.timeout +++ /dev/null @@ -1 +0,0 @@ -0 From 09602ede77592b357aeff237d8e4723788a3697a Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 13:58:52 -0500 Subject: [PATCH 088/160] karma.ini: commented out most spammy tlds these will vary, depending on the site --- config/karma.ini | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index aa533248f..b586f4ea0 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -50,17 +50,17 @@ history_negative=-3 ; award negative karma to spammy TLDs ; caution, awarding karma > msg_negative_limit may blacklist that TLD info=-4 -pw=-4 -tw=-3 biz=-3 -cl=-2 -br=-2 -fr=-2 -be=-2 -jp=-2 -no=-2 -se=-2 -sg=-2 +pw=-4 +;tw=-3 +;cl=-2 +;br=-2 +;fr=-2 +;be=-2 +;jp=-2 +;no=-2 +;se=-2 +;sg=-2 ; karma can award points based on other plugins results. ; the key is a note to inspect and the value is the karma award From 8d1e7f0bb2ca9642e949df274e885660bc365bd6 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 14:06:49 -0500 Subject: [PATCH 089/160] karma: added Redis reconnection logic (does a ping at connect/disconnect, and reconnects if needed) --- plugins/karma.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/plugins/karma.js b/plugins/karma.js index bb06a099c..73bb15b77 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -18,22 +18,35 @@ exports.register = function () { this.register_hook('disconnect', 'karma_onDisconnect'); }; -exports.karma_onInit = function (next,server) { - var config = this.config.get('karma.ini'); +exports.karma_onInit = function (next, server) { + initRedisConnection(this); + return next(); +}; + +function initRedisConnection(self) { + if (db && db.ping()) return; // connection is good + + var config = self.config.get('karma.ini'); var redis_ip = '127.0.0.1'; var redis_port = '6379'; if ( config.redis ) { redis_ip = config.redis.server_ip || '127.0.0.1'; redis_port = config.redis.server_port || '6379'; }; + db = redis.createClient(redis_port, redis_ip); - return next(); + db.on('error', function (error) { + self.logerror('Redis error: ' + error.message); + db.end(); + db = null; + }); }; exports.karma_onConnect = function (next, connection) { var plugin = this; var config = this.config.get('karma.ini'); + initRedisConnection(this); initConnectionNote(connection, config); var r_ip = connection.remote_ip; @@ -176,6 +189,7 @@ exports.karma_onDisconnect = function (next, connection) { var plugin = this; var config = this.config.get('karma.ini'); + initRedisConnection(this); if ( config.concurrency ) db.incrby('concurrent|'+connection.remote_ip, -1); var k = connection.notes.karma; From 810e342e09334a61d34048c70994e5736882ed47 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 14:14:36 -0500 Subject: [PATCH 090/160] karma: fixed misspelled threshold --- config/karma.ini | 4 ++-- plugins/karma.js | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index b586f4ea0..5177101d5 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -33,9 +33,9 @@ neutral=5 good=20 -[threshhold] +[thresholds] ; Be conservative to avoid false positives! -; the threshhold below which a connection is considered bad +; the threshold below which a connection is considered bad negative=-3 ; score at which a connection is considered good positive=2 diff --git a/plugins/karma.js b/plugins/karma.js index 73bb15b77..908a1acb9 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -29,7 +29,7 @@ function initRedisConnection(self) { var config = self.config.get('karma.ini'); var redis_ip = '127.0.0.1'; var redis_port = '6379'; - if ( config.redis ) { + if (config.redis) { redis_ip = config.redis.server_ip || '127.0.0.1'; redis_port = config.redis.server_port || '6379'; }; @@ -83,7 +83,7 @@ exports.karma_onConnect = function (next, connection) { +dbr.connections+" connects, "+history+" history"; var too_many = checkConcurrency(plugin, 'concurrent|'+r_ip, replies[0], history); - if ( too_many ) { + if (too_many) { connection.loginfo(plugin, too_many + ", ("+summary+")"); return next(DENYSOFT, too_many); }; @@ -165,7 +165,7 @@ exports.karma_onRcptTo = function (next, connection, params) { exports.karma_onData = function (next, connection) { // cut off bad senders at DATA to prevent transferring the message var config = this.config.get('karma.ini'); - var negative_limit = config.threshhold.negative || -5; + var negative_limit = config.thresholds.negative || -5; var karma = connection.notes.karma * 1; if (karma.connection <= negative_limit) { @@ -190,7 +190,7 @@ exports.karma_onDisconnect = function (next, connection) { var config = this.config.get('karma.ini'); initRedisConnection(this); - if ( config.concurrency ) db.incrby('concurrent|'+connection.remote_ip, -1); + if (config.concurrency) db.incrby('concurrent|'+connection.remote_ip, -1); var k = connection.notes.karma; if (!k) { connection.logerror(plugin, "karma note missing!"); return next(); }; @@ -203,8 +203,8 @@ exports.karma_onDisconnect = function (next, connection) { var key = 'karma|'+connection.remote_ip; var history = k.history; - if (config.threshhold) { - var pos_lim = config.threshhold.positive || 2; + if (config.threshold) { + var pos_lim = config.thresholds.positive || 2; if (k.connection > pos_lim) { db.hincrby(key, 'good', 1); @@ -212,14 +212,14 @@ exports.karma_onDisconnect = function (next, connection) { return next(); }; - var bad_limit = config.threshhold.negative || -3; + var bad_limit = config.thresholds.negative || -3; if (k.connection < bad_limit) { db.hincrby(key, 'bad', 1); history--; - if (history <= config.threshhold.history_negative) { + if (history <= config.thresholds.history_negative) { if (history < -5) { - db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1 ) ); + db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1)); connection.loginfo(plugin, "penalty box bonus!: "+karmaSummary(connection)); } else { @@ -281,22 +281,22 @@ function checkAwards (config, connection, plugin) { var karma_to_apply = connection.notes.karma.todo[key]; if (!karma_to_apply) return; - if ( Number(karma_to_apply) === 'NaN' ) return; // garbage in config + if (Number(karma_to_apply) === 'NaN') return; // garbage in config connection.notes.karma.connection += karma_to_apply * 1; connection.loginfo(plugin, "applied "+key+" karma: "+karma_to_apply); delete connection.notes.karma.todo[key]; var trimmed = key.substring(0,5) === 'notes' ? key.substring(6) : key; - if ( karma_to_apply > 0 ) { connection.notes.karma.awards.push(trimmed); }; - if ( karma_to_apply < 0 ) { connection.notes.karma.penalties.push(trimmed); }; + if (karma_to_apply > 0) { connection.notes.karma.awards.push(trimmed); }; + if (karma_to_apply < 0) { connection.notes.karma.penalties.push(trimmed); }; }); } function checkConcurrency(plugin, con_key, val, history) { var config = plugin.config.get('karma.ini'); - if ( !config.concurrency ) return; + if (!config.concurrency) return; var count = val || 0; // add this connection count++; @@ -315,7 +315,7 @@ function checkMaxRecipients(connection, plugin, config) { if (!config.recipients) return; // disabled in config file var c = connection.rcpt_count; var count = c.accept + c.tempfail + c.reject + 1; - if ( count <= 1 ) return; // everybody is allowed one + if (count <= 1) return; // everybody is allowed one connection.logdebug(plugin, "recipient count: "+count ); @@ -325,15 +325,15 @@ function checkMaxRecipients(connection, plugin, config) { // the deeds of their past shall not go unnoticed! var history = connection.notes.karma.history; - if ( history > 3 && count <= cr.good) return; - if ( history > -1 && count <= cr.neutral) return; + if (history > 3 && count <= cr.good) return; + if (history > -1 && count <= cr.neutral) return; // this is *more* strict than history, b/c they have fewer opportunity // to score positive karma this early in the connection. senders with // good history will rarely see these limits. var karma = connection.notes.karma.connection; - if ( karma > 3 && count <= cr.good) return; - if ( karma >= 0 && count <= cr.neutral) return; + if (karma > 3 && count <= cr.good) return; + if (karma >= 0 && count <= cr.neutral) return; return 'too many recipients ('+count+') for '+desc+' karma'; } @@ -358,7 +358,7 @@ function checkSyntaxMailFrom(connection, plugin) { // connection.logdebug(plugin, "mail_from: "+full_from); // look for an illegal (RFC 5321,2821,821) space in envelope from - if (full_from.toUpperCase().substring(0,11) === 'MAIL FROM:<' ) return; + if (full_from.toUpperCase().substring(0,11) === 'MAIL FROM:<') return; connection.loginfo(plugin, "illegal envelope address format: "+full_from ); connection.notes.karma.connection--; @@ -368,7 +368,7 @@ function checkSyntaxMailFrom(connection, plugin) { function checkSyntaxRcptTo(connection, plugin) { // check for an illegal RFC (2)821 space in envelope recipient var full_rcpt = connection.current_line; - if ( full_rcpt.toUpperCase().substring(0,9) === 'RCPT TO:<' ) return; + if (full_rcpt.toUpperCase().substring(0,9) === 'RCPT TO:<') return; connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); connection.notes.karma.connection--; From 74ca347136f4a74053f7f3212fa27ba221273a8d Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 14:31:19 -0500 Subject: [PATCH 091/160] karma: whitespace changes --- plugins/karma.js | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/plugins/karma.js b/plugins/karma.js index 908a1acb9..0fd834987 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -50,7 +50,7 @@ exports.karma_onConnect = function (next, connection) { initConnectionNote(connection, config); var r_ip = connection.remote_ip; - var dbkey = 'karma|'+r_ip; + var dbkey = 'karma|' + r_ip; var expire = (config.main.expire_days || 60) * 86400; // convert to days function initRemoteIP () { @@ -62,11 +62,11 @@ exports.karma_onConnect = function (next, connection) { }; db.multi() - .get('concurrent|'+r_ip) + .get('concurrent|' + r_ip) .hgetall(dbkey) - .exec( function redisResults (err,replies) { + .exec(function redisResults (err,replies) { if (err) { - connection.logdebug(plugin,"err: "+err); + connection.logdebug(plugin,"err: " + err); return next(); }; @@ -79,29 +79,29 @@ exports.karma_onConnect = function (next, connection) { var history = (dbr.good || 0) - (dbr.bad || 0); connection.notes.karma.history = history; - var summary = dbr.bad+" bad, "+dbr.good+" good, " - +dbr.connections+" connects, "+history+" history"; + var summary = dbr.bad + " bad, " + dbr.good + " good, " + + dbr.connections + " connects, " + history + " history"; - var too_many = checkConcurrency(plugin, 'concurrent|'+r_ip, replies[0], history); + var too_many = checkConcurrency(plugin, 'concurrent|' + r_ip, replies[0], history); if (too_many) { - connection.loginfo(plugin, too_many + ", ("+summary+")"); + connection.loginfo(plugin, too_many + ", (" + summary + ")"); return next(DENYSOFT, too_many); }; if (dbr.penalty_start_ts === '0') { - connection.loginfo(plugin, "no penalty, "+karmaSummary(connection)); + connection.loginfo(plugin, "no penalty, " + karmaSummary(connection)); return next(); } var days_old = (Date.now() - Date.parse(dbr.penalty_start_ts)) / 86.4; var penalty_days = config.main.penalty_days; if (days_old >= penalty_days) { - connection.loginfo(plugin, "penalty expired, "+karmaSummary(connection)); + connection.loginfo(plugin, "penalty expired, " + karmaSummary(connection)); return next(); } - var left = +( penalty_days - days_old ).toFixed(2); - var mess = "Bad karma, you can try again in "+left+" more days."; + var left = +(penalty_days - days_old).toFixed(2); + var mess = "Bad karma, you can try again in " + left + " more days."; return next(DENY, mess); }); @@ -126,11 +126,11 @@ exports.karma_onDeny = function (next, connection, params) { var config = this.config.get('karma.ini'); initConnectionNote(connection, config); -// TO CONSIDER: decrement karma two points for a 5XX deny? + // CONSIDER: decrement karma two points for a 5XX deny? connection.notes.karma.connection--; connection.notes.karma.penalties.push(pi_name); - connection.loginfo(plugin, 'deny, '+karmaSummary(connection)); + connection.loginfo(plugin, 'deny, ' + karmaSummary(connection)); checkAwards(config, connection, plugin); return next(); @@ -169,7 +169,7 @@ exports.karma_onData = function (next, connection) { var karma = connection.notes.karma * 1; if (karma.connection <= negative_limit) { - return next(DENY, "very bad karma score: "+karma); + return next(DENY, "very bad karma score: " + karma); } checkAwards(config, connection, this); @@ -190,17 +190,17 @@ exports.karma_onDisconnect = function (next, connection) { var config = this.config.get('karma.ini'); initRedisConnection(this); - if (config.concurrency) db.incrby('concurrent|'+connection.remote_ip, -1); + if (config.concurrency) db.incrby('concurrent|' + connection.remote_ip, -1); var k = connection.notes.karma; if (!k) { connection.logerror(plugin, "karma note missing!"); return next(); }; if (!k.connection) { - connection.loginfo(plugin, "neutral: "+karmaSummary(connection)); + connection.loginfo(plugin, "neutral: " + karmaSummary(connection)); return next(); }; - var key = 'karma|'+connection.remote_ip; + var key = 'karma|' + connection.remote_ip; var history = k.history; if (config.threshold) { @@ -208,7 +208,7 @@ exports.karma_onDisconnect = function (next, connection) { if (k.connection > pos_lim) { db.hincrby(key, 'good', 1); - connection.loginfo(plugin, "positive: "+karmaSummary(connection)); + connection.loginfo(plugin, "positive: " + karmaSummary(connection)); return next(); }; @@ -220,29 +220,29 @@ exports.karma_onDisconnect = function (next, connection) { if (history <= config.thresholds.history_negative) { if (history < -5) { db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1)); - connection.loginfo(plugin, "penalty box bonus!: "+karmaSummary(connection)); + connection.loginfo(plugin, "penalty box bonus!: " + karmaSummary(connection)); } else { db.hset(key, 'penalty_start_ts', Date()); - connection.loginfo(plugin, "penalty box: "+karmaSummary(connection)); + connection.loginfo(plugin, "penalty box: " + karmaSummary(connection)); } return next(); } } }; checkAwards(config, connection, plugin); - connection.loginfo(plugin, "no action, "+karmaSummary(connection)); + connection.loginfo(plugin, "no action, " + karmaSummary(connection)); return next(); }; function karmaSummary(c) { var k = c.notes.karma; return '(' - +'conn:'+k.connection - +', hist: '+k.history - +(k.penalties.length ? ', penalties: '+k.penalties : '') - +(k.awards.length ? ', awards: ' +k.awards : '') - +')'; + + 'conn:' + k.connection + + ', hist: ' + k.history + + (k.penalties.length ? ', penalties: '+ k.penalties : '') + + (k.awards.length ? ', awards: ' + k.awards : '') + + ')'; } function addDays(date, days) { @@ -265,17 +265,17 @@ function checkAwards (config, connection, plugin) { // assemble the object path using the note name var note = assembleNoteObj(connection, suffix); if (note == null || note === false) { - // connection.logdebug(plugin, "no connection note: "+key); + // connection.logdebug(plugin, "no connection note: " + key); if (!connection.transaction) return; note = assembleNoteObj(connection.transaction, suffix); if (note == null || note === false) { - // connection.logdebug(plugin, "no transaction note: "+key); + // connection.logdebug(plugin, "no transaction note: " + key); return; } }; if (wants && note && (wants !== note)) { - connection.logdebug(plugin, "key "+suffix+" wants: "+wants+" but got: "+note); + connection.logdebug(plugin, "key " + suffix + " wants: " + wants + " but got: " + note); return; }; @@ -284,7 +284,7 @@ function checkAwards (config, connection, plugin) { if (Number(karma_to_apply) === 'NaN') return; // garbage in config connection.notes.karma.connection += karma_to_apply * 1; - connection.loginfo(plugin, "applied "+key+" karma: "+karma_to_apply); + connection.loginfo(plugin, "applied " + key + " karma: " + karma_to_apply); delete connection.notes.karma.todo[key]; var trimmed = key.substring(0,5) === 'notes' ? key.substring(6) : key; @@ -307,7 +307,7 @@ function checkConcurrency(plugin, con_key, val, history) { if (history < 0 && count > config.concurrency.bad) reject++; if (history > 0 && count > config.concurrency.good) reject++; if (history == 0 && count > config.concurrency.neutral) reject++; - if (reject) return "too many connections for you: "+count; + if (reject) return "too many connections for you: " + count; return; }; @@ -317,7 +317,7 @@ function checkMaxRecipients(connection, plugin, config) { var count = c.accept + c.tempfail + c.reject + 1; if (count <= 1) return; // everybody is allowed one - connection.logdebug(plugin, "recipient count: "+count ); + connection.logdebug(plugin, "recipient count: " + count ); var desc = history > 3 ? 'good' : history >= 0 ? 'neutral' : 'bad'; @@ -335,7 +335,7 @@ function checkMaxRecipients(connection, plugin, config) { if (karma > 3 && count <= cr.good) return; if (karma >= 0 && count <= cr.neutral) return; - return 'too many recipients ('+count+') for '+desc+' karma'; + return 'too many recipients (' + count + ') for ' + desc + ' karma'; } function checkSpammyTLD(mail_from, config, connection, plugin) { @@ -343,24 +343,24 @@ function checkSpammyTLD(mail_from, config, connection, plugin) { if (mail_from.isNull()) return; // null sender (bounce) var from_tld = mail_from.host.split('.').pop(); - // connection.logdebug(plugin, "from_tld: "+from_tld); + // connection.logdebug(plugin, "from_tld: " + from_tld); var tld_penalty = (config.spammy_tlds[from_tld] || 0) * 1; // force numeric if (tld_penalty === 0) return; - connection.loginfo(plugin, "spammy TLD: "+tld_penalty); + connection.loginfo(plugin, "spammy TLD: " + tld_penalty); connection.notes.karma.connection += tld_penalty; connection.notes.karma.penalties.push('spammy.TLD'); }; function checkSyntaxMailFrom(connection, plugin) { var full_from = connection.current_line; - // connection.logdebug(plugin, "mail_from: "+full_from); + // connection.logdebug(plugin, "mail_from: " + full_from); // look for an illegal (RFC 5321,2821,821) space in envelope from if (full_from.toUpperCase().substring(0,11) === 'MAIL FROM:<') return; - connection.loginfo(plugin, "illegal envelope address format: "+full_from ); + connection.loginfo(plugin, "illegal envelope address format: " + full_from ); connection.notes.karma.connection--; connection.notes.karma.penalties.push('rfc5321.MailFrom'); }; @@ -370,7 +370,7 @@ function checkSyntaxRcptTo(connection, plugin) { var full_rcpt = connection.current_line; if (full_rcpt.toUpperCase().substring(0,9) === 'RCPT TO:<') return; - connection.loginfo(plugin, "illegal envelope address format: "+full_rcpt ); + connection.loginfo(plugin, "illegal envelope address format: " + full_rcpt ); connection.notes.karma.connection--; connection.notes.karma.penalties.push('rfc5321.RcptTo'); }; From 6adf0e3c1ee7b18a909ac5fe018b4fdf9c703978 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 15:17:46 -0500 Subject: [PATCH 092/160] karma: note values can be pattern matched --- config/karma.ini | 16 ++++++++++++---- plugins/karma.js | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index 5177101d5..d5063162e 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -63,11 +63,12 @@ pw=-4 ;sg=-2 ; karma can award points based on other plugins results. -; the key is a note to inspect and the value is the karma award +; the key is a note to inspect and the value is a karma award ; NOTE: karma awards can be positive or negative! -; Any true value in the specified note will match. -; A required value in the note can be specified with an @ postfix -this special syntax '@' +; +; Any true value in the specified note will match. If that's not sufficient, +; a pattern can be specified with an @ postfix. The note value will be +; matched case insensitively. [awards] relaying=1 notes.auth_user=2 @@ -77,7 +78,14 @@ notes.rdns_access@white=1 notes.fcrdns.no_rdns=-2 notes.fcrdns.ip_in_rdns=-1 notes.spamassassin.flag@Yes=-2 +notes.bounce@invalid=-3 + +; SPF results: 3=Fail, 4=SoftFail, 6=Temperror, 7=Permerror notes.spf_helo@3=-2 notes.spf_helo@4=-1 notes.spf_helo@6=-1 notes.spf_helo@7=-1 +notes.spf_mail_result@3=-2 +notes.spf_mail_result@4=-1 +notes.spf_mail_result@6=-1 +notes.spf_mail_result@7=-1 diff --git a/plugins/karma.js b/plugins/karma.js index 0fd834987..4d176eff9 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -274,8 +274,8 @@ function checkAwards (config, connection, plugin) { } }; - if (wants && note && (wants !== note)) { - connection.logdebug(plugin, "key " + suffix + " wants: " + wants + " but got: " + note); + if (wants && note && (!note.toString().match(new RegExp(wants, 'i')))) { + // connection.logdebug(plugin, "key " + suffix + " wants: " + wants + " but saw: " + note); return; }; From 43b4f43cfe90d26adf54ea29136016a8c7139b23 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 15:29:43 -0500 Subject: [PATCH 093/160] karma: return a 4XX error on max recipients was 5XX --- plugins/karma.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/karma.js b/plugins/karma.js index 4d176eff9..a791d2f25 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -155,7 +155,7 @@ exports.karma_onRcptTo = function (next, connection, params) { checkSyntaxRcptTo(connection, plugin); var too_many = checkMaxRecipients(connection, plugin, config); - if (too_many) return next(DENY, too_many); + if (too_many) return next(DENYSOFT, too_many); checkAwards(config, connection, plugin); connection.loginfo(plugin, karmaSummary(connection)); From af0f2966cc5c95ab2d53c33bc489cfe39278617f Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 15:39:21 -0500 Subject: [PATCH 094/160] karma: added timer to concurrent limit disconnect --- config/karma.ini | 3 +++ plugins/karma.js | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/karma.ini b/config/karma.ini index d5063162e..9c61e73a3 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -10,6 +10,7 @@ penalty_days = 1 ; senders karma may never expire. expire_days = 60 + ; Redis is our super-lightweight key/value store [redis] server_ip = 127.0.0.1 @@ -25,6 +26,8 @@ bad=1 neutral=2 good=10 +; delay excess connections this many seconds before disconnecting +disconnect_delay=10 ; maximum number of recipients allowed [recipients] diff --git a/plugins/karma.js b/plugins/karma.js index a791d2f25..fc84d0b04 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -85,7 +85,10 @@ exports.karma_onConnect = function (next, connection) { var too_many = checkConcurrency(plugin, 'concurrent|' + r_ip, replies[0], history); if (too_many) { connection.loginfo(plugin, too_many + ", (" + summary + ")"); - return next(DENYSOFT, too_many); + var delay = config.concurrency.disconnect_delay || 10; + setTimeout(function () { + return next(DENYSOFTDISCONNECT, too_many); + }, delay * 1000); }; if (dbr.penalty_start_ts === '0') { From e1aa3677d3cbb2f9c3283a45704bc8651426c99e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 16:44:23 -0500 Subject: [PATCH 095/160] karma: shed some camelCase --- plugins/karma.js | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/plugins/karma.js b/plugins/karma.js index fc84d0b04..e984eb7f7 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -19,11 +19,11 @@ exports.register = function () { }; exports.karma_onInit = function (next, server) { - initRedisConnection(this); + init_redis_connection(this); return next(); }; -function initRedisConnection(self) { +function init_redis_connection(self) { if (db && db.ping()) return; // connection is good var config = self.config.get('karma.ini'); @@ -46,8 +46,8 @@ exports.karma_onConnect = function (next, connection) { var plugin = this; var config = this.config.get('karma.ini'); - initRedisConnection(this); - initConnectionNote(connection, config); + init_redis_connection(this); + init_connection_note(connection, config); var r_ip = connection.remote_ip; var dbkey = 'karma|' + r_ip; @@ -82,7 +82,7 @@ exports.karma_onConnect = function (next, connection) { var summary = dbr.bad + " bad, " + dbr.good + " good, " + dbr.connections + " connects, " + history + " history"; - var too_many = checkConcurrency(plugin, 'concurrent|' + r_ip, replies[0], history); + var too_many = check_concurrency(plugin, 'concurrent|' + r_ip, replies[0], history); if (too_many) { connection.loginfo(plugin, too_many + ", (" + summary + ")"); var delay = config.concurrency.disconnect_delay || 10; @@ -92,14 +92,14 @@ exports.karma_onConnect = function (next, connection) { }; if (dbr.penalty_start_ts === '0') { - connection.loginfo(plugin, "no penalty, " + karmaSummary(connection)); + connection.loginfo(plugin, "no penalty, " + karma_summary(connection)); return next(); } var days_old = (Date.now() - Date.parse(dbr.penalty_start_ts)) / 86.4; var penalty_days = config.main.penalty_days; if (days_old >= penalty_days) { - connection.loginfo(plugin, "penalty expired, " + karmaSummary(connection)); + connection.loginfo(plugin, "penalty expired, " + karma_summary(connection)); return next(); } @@ -109,7 +109,7 @@ exports.karma_onConnect = function (next, connection) { return next(DENY, mess); }); - checkAwards(config, connection, plugin); + check_awards(config, connection, plugin); }; exports.karma_onDeny = function (next, connection, params) { @@ -127,15 +127,15 @@ exports.karma_onDeny = function (next, connection, params) { var transaction = connection.transaction; var config = this.config.get('karma.ini'); - initConnectionNote(connection, config); + init_connection_note(connection, config); // CONSIDER: decrement karma two points for a 5XX deny? connection.notes.karma.connection--; connection.notes.karma.penalties.push(pi_name); - connection.loginfo(plugin, 'deny, ' + karmaSummary(connection)); + connection.loginfo(plugin, 'deny, ' + karma_summary(connection)); - checkAwards(config, connection, plugin); + check_awards(config, connection, plugin); return next(); }; @@ -143,11 +143,11 @@ exports.karma_onMailFrom = function (next, connection, params) { var plugin = this; var config = this.config.get('karma.ini'); - checkSpammyTLD(params[0], config, connection, plugin); - checkSyntaxMailFrom(connection, plugin); + check_spammy_tld(params[0], config, connection, plugin); + check_syntax_mailfrom(connection, plugin); - checkAwards(config, connection, plugin); - connection.loginfo(plugin, karmaSummary(connection)); + check_awards(config, connection, plugin); + connection.loginfo(plugin, karma_summary(connection)); return next(); }; @@ -156,12 +156,12 @@ exports.karma_onRcptTo = function (next, connection, params) { var rcpt = params[0]; var config = this.config.get('karma.ini'); - checkSyntaxRcptTo(connection, plugin); - var too_many = checkMaxRecipients(connection, plugin, config); + check_syntax_RcptTo(connection, plugin); + var too_many = max_recipients(connection, plugin, config); if (too_many) return next(DENYSOFT, too_many); - checkAwards(config, connection, plugin); - connection.loginfo(plugin, karmaSummary(connection)); + check_awards(config, connection, plugin); + connection.loginfo(plugin, karma_summary(connection)); return next(); }; @@ -175,16 +175,16 @@ exports.karma_onData = function (next, connection) { return next(DENY, "very bad karma score: " + karma); } - checkAwards(config, connection, this); + check_awards(config, connection, this); return next(); } exports.karma_onDataPost = function (next, connection) { connection.transaction.add_header('X-Haraka-Karma', - karmaSummary(connection) + karma_summary(connection) ); var config = this.config.get('karma.ini'); - checkAwards(config, connection, this); + check_awards(config, connection, this); return next(); } @@ -192,14 +192,14 @@ exports.karma_onDisconnect = function (next, connection) { var plugin = this; var config = this.config.get('karma.ini'); - initRedisConnection(this); + init_redis_connection(this); if (config.concurrency) db.incrby('concurrent|' + connection.remote_ip, -1); var k = connection.notes.karma; if (!k) { connection.logerror(plugin, "karma note missing!"); return next(); }; if (!k.connection) { - connection.loginfo(plugin, "neutral: " + karmaSummary(connection)); + connection.loginfo(plugin, "neutral: " + karma_summary(connection)); return next(); }; @@ -211,7 +211,7 @@ exports.karma_onDisconnect = function (next, connection) { if (k.connection > pos_lim) { db.hincrby(key, 'good', 1); - connection.loginfo(plugin, "positive: " + karmaSummary(connection)); + connection.loginfo(plugin, "positive: " + karma_summary(connection)); return next(); }; @@ -222,23 +222,23 @@ exports.karma_onDisconnect = function (next, connection) { if (history <= config.thresholds.history_negative) { if (history < -5) { - db.hset(key, 'penalty_start_ts', addDays(Date(), history * -1)); - connection.loginfo(plugin, "penalty box bonus!: " + karmaSummary(connection)); + db.hset(key, 'penalty_start_ts', add_days(Date(), history * -1)); + connection.loginfo(plugin, "penalty box bonus!: " + karma_summary(connection)); } else { db.hset(key, 'penalty_start_ts', Date()); - connection.loginfo(plugin, "penalty box: " + karmaSummary(connection)); + connection.loginfo(plugin, "penalty box: " + karma_summary(connection)); } return next(); } } }; - checkAwards(config, connection, plugin); - connection.loginfo(plugin, "no action, " + karmaSummary(connection)); + check_awards(config, connection, plugin); + connection.loginfo(plugin, "no action, " + karma_summary(connection)); return next(); }; -function karmaSummary(c) { +function karma_summary(c) { var k = c.notes.karma; return '(' + 'conn:' + k.connection @@ -248,13 +248,13 @@ function karmaSummary(c) { + ')'; } -function addDays(date, days) { +function add_days(date, days) { var result = new Date(date); result.setDate(date.getDate() + days); return result; } -function checkAwards (config, connection, plugin) { +function check_awards (config, connection, plugin) { if (!connection.notes.karma) return; if (!connection.notes.karma.todo) return; if (!plugin) plugin = connection; @@ -266,11 +266,11 @@ function checkAwards (config, connection, plugin) { var wants = e[1]; // assemble the object path using the note name - var note = assembleNoteObj(connection, suffix); + var note = assemble_note_obj(connection, suffix); if (note == null || note === false) { // connection.logdebug(plugin, "no connection note: " + key); if (!connection.transaction) return; - note = assembleNoteObj(connection.transaction, suffix); + note = assemble_note_obj(connection.transaction, suffix); if (note == null || note === false) { // connection.logdebug(plugin, "no transaction note: " + key); return; @@ -296,7 +296,7 @@ function checkAwards (config, connection, plugin) { }); } -function checkConcurrency(plugin, con_key, val, history) { +function check_concurrency(plugin, con_key, val, history) { var config = plugin.config.get('karma.ini'); if (!config.concurrency) return; @@ -314,7 +314,7 @@ function checkConcurrency(plugin, con_key, val, history) { return; }; -function checkMaxRecipients(connection, plugin, config) { +function max_recipients(connection, plugin, config) { if (!config.recipients) return; // disabled in config file var c = connection.rcpt_count; var count = c.accept + c.tempfail + c.reject + 1; @@ -341,7 +341,7 @@ function checkMaxRecipients(connection, plugin, config) { return 'too many recipients (' + count + ') for ' + desc + ' karma'; } -function checkSpammyTLD(mail_from, config, connection, plugin) { +function check_spammy_tld(mail_from, config, connection, plugin) { if (!config.spammy_tlds) return; if (mail_from.isNull()) return; // null sender (bounce) @@ -356,7 +356,7 @@ function checkSpammyTLD(mail_from, config, connection, plugin) { connection.notes.karma.penalties.push('spammy.TLD'); }; -function checkSyntaxMailFrom(connection, plugin) { +function check_syntax_mailfrom(connection, plugin) { var full_from = connection.current_line; // connection.logdebug(plugin, "mail_from: " + full_from); @@ -368,7 +368,7 @@ function checkSyntaxMailFrom(connection, plugin) { connection.notes.karma.penalties.push('rfc5321.MailFrom'); }; -function checkSyntaxRcptTo(connection, plugin) { +function check_syntax_RcptTo(connection, plugin) { // check for an illegal RFC (2)821 space in envelope recipient var full_rcpt = connection.current_line; if (full_rcpt.toUpperCase().substring(0,9) === 'RCPT TO:<') return; @@ -378,7 +378,7 @@ function checkSyntaxRcptTo(connection, plugin) { connection.notes.karma.penalties.push('rfc5321.RcptTo'); }; -function initConnectionNote(connection, config) { +function init_connection_note(connection, config) { if (connection.notes.karma) return; // init once per connection connection.notes.karma = { connection: 0, @@ -398,7 +398,7 @@ function initConnectionNote(connection, config) { }); }; -function assembleNoteObj(prefix, key) { +function assemble_note_obj(prefix, key) { var note = prefix; var parts = key.split('.'); while(parts.length > 0) { From ccf61359d5cf198bef6f9d5e95d17493e83bfe19 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 17:39:26 -0500 Subject: [PATCH 096/160] note how to specify a transation note --- config/karma.ini | 6 +++++- plugins/karma.js | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config/karma.ini b/config/karma.ini index 9c61e73a3..0bb6ab69b 100644 --- a/config/karma.ini +++ b/config/karma.ini @@ -67,7 +67,11 @@ pw=-4 ; karma can award points based on other plugins results. ; the key is a note to inspect and the value is a karma award -; NOTE: karma awards can be positive or negative! +; +; Connection and transaction notes are checked by default. Use a transaction +; prefix to only check the transaction note. +; +; karma awards can be positive or negative. ; ; Any true value in the specified note will match. If that's not sufficient, ; a pattern can be specified with an @ postfix. The note value will be diff --git a/plugins/karma.js b/plugins/karma.js index e984eb7f7..e91d5a485 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -166,7 +166,7 @@ exports.karma_onRcptTo = function (next, connection, params) { }; exports.karma_onData = function (next, connection) { -// cut off bad senders at DATA to prevent transferring the message + // cut off bad senders at DATA to prevent transferring the message var config = this.config.get('karma.ini'); var negative_limit = config.thresholds.negative || -5; var karma = connection.notes.karma * 1; @@ -268,17 +268,17 @@ function check_awards (config, connection, plugin) { // assemble the object path using the note name var note = assemble_note_obj(connection, suffix); if (note == null || note === false) { - // connection.logdebug(plugin, "no connection note: " + key); + connection.logdebug(plugin, "no connection note: " + key); if (!connection.transaction) return; note = assemble_note_obj(connection.transaction, suffix); if (note == null || note === false) { - // connection.logdebug(plugin, "no transaction note: " + key); + connection.logdebug(plugin, "no transaction note: " + key); return; } }; if (wants && note && (!note.toString().match(new RegExp(wants, 'i')))) { - // connection.logdebug(plugin, "key " + suffix + " wants: " + wants + " but saw: " + note); + connection.logdebug(plugin, "key " + suffix + " wants: " + wants + " but saw: " + note); return; }; @@ -360,7 +360,7 @@ function check_syntax_mailfrom(connection, plugin) { var full_from = connection.current_line; // connection.logdebug(plugin, "mail_from: " + full_from); -// look for an illegal (RFC 5321,2821,821) space in envelope from + // look for an illegal (RFC 5321,2821,821) space in envelope from if (full_from.toUpperCase().substring(0,11) === 'MAIL FROM:<') return; connection.loginfo(plugin, "illegal envelope address format: " + full_from ); From 1bdc895daa2c690b0bb50feadfcc538cbc116934 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 02:39:29 -0500 Subject: [PATCH 097/160] bounce: initial authoring --- TODO | 2 +- config/bounce.ini | 5 +++ docs/plugins/bounces.md | 23 +++++++++++++ plugins/bounce.js | 59 ++++++++++++++++++++++++++++++++++ plugins/mail_from.nobounces.js | 7 ++++ 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 config/bounce.ini create mode 100644 docs/plugins/bounces.md create mode 100644 plugins/bounce.js diff --git a/TODO b/TODO index 3e1786a78..44a04b0e1 100644 --- a/TODO +++ b/TODO @@ -3,7 +3,6 @@ - Create a config file for each of the core shipping configs, so people have something as a baseline - IMAP server (long shot for now) - Plugins to copy from Qpsmtpd: - - bogus_bounce (checks bounces have one recipient and no return-path) - dspam - greylisting - virus/* @@ -23,6 +22,7 @@ Remove the following deprecated plugins - data.noreceived - data.rfc5322_header_checks - daemonize + - mail_from.nobounces Rename the following plugins - toobusy -> connect.toobusy diff --git a/config/bounce.ini b/config/bounce.ini new file mode 100644 index 000000000..1ff10d319 --- /dev/null +++ b/config/bounce.ini @@ -0,0 +1,5 @@ +; reject all bounce messages (generally not a good idea) +reject_all=0 + +; reject bounces that are not RFC compliant (likely faked) +reject_invalid=1 diff --git a/docs/plugins/bounces.md b/docs/plugins/bounces.md new file mode 100644 index 000000000..cb5946242 --- /dev/null +++ b/docs/plugins/bounces.md @@ -0,0 +1,23 @@ +bounce +=================== +This plugin provides options for bounce processing. + + +Configuration +------------------- + +- reject_all + +Blocks all bounce messages using the simple rule of checking +for `MAIL FROM:<>`. + +This is useful to enable if you have a mail server that gets spoofed too +much but very few legitimate users. It is potentially bad to block all +bounce messages, but unfortunately for some hosts, sometimes necessary. + + +- reject_invalid +-------------------- +This option tries to assure the message really is a bounce. It makes +sure the message has a single recipient and that the return path is +empty. diff --git a/plugins/bounce.js b/plugins/bounce.js new file mode 100644 index 000000000..a079bb191 --- /dev/null +++ b/plugins/bounce.js @@ -0,0 +1,59 @@ +// bounce tests + +exports.register = function () { + var plugin = this; + + this.register_hook('mail', 'bounce_mail'); + // this.register_hook('rcpt', 'bounce_rcpt'); + // this.register_hook('data', 'bounce_data'); + this.register_hook('data_post', 'bounce_data_post'); +}; + +exports.bounce_mail = function (next, connection, params) { + var mail_from = params[0]; + if (!mail_from.isNull()) return next(); // not a bounce + var cfg = this.config.get('bounce.ini'); + if (cfg.reject_all) + return next(DENY, "No bounces accepted here"); + return next(); +} + +exports.bounce_data_post = function(next, connection) { + var plugin = connection; + + if (!has_null_sender(connection)) return next(); // not a bounce. + + var cfg = this.config.get('bounce.ini'); + var rej = cfg.reject_invalid; + + // Valid bounces have a single recipient + var err = has_single_recipient(connection, plugin); + if (err && rej) return next(DENY, err); + + // validate that Return-Path is empty, RFC 3834 + err = has_empty_return_path(connection, plugin); + if (err && rej) return next(DENY, err); + + return next(); +}; + +function has_empty_return_path(connection, plugin) { + var rp = connection.transaction.header.get('Return-Path'); + if (!rp) return; + if (rp === '<>') return; + connection.loginfo(plugin, "bounce messages must not have a Return-Path"); + return "a bounce return path must be empty (RFC 3834)"; +}; + +function has_single_recipient(connection, plugin) { + if (connection.transaction.rcpt_to.length === 1) return; + + connection.loginfo(plugin, "bogus bounce to: " + + connection.transaction.rcpt_to.join(',')); + + return "this bounce message does not have 1 recipient"; +}; + +function has_null_sender(connection) { + return connection.transaction.mail_from.isNull() ? true : false; +}; diff --git a/plugins/mail_from.nobounces.js b/plugins/mail_from.nobounces.js index 1babfb2f4..5519863ee 100644 --- a/plugins/mail_from.nobounces.js +++ b/plugins/mail_from.nobounces.js @@ -1,6 +1,13 @@ // I don't allow MAIL FROM:<> on my server, because it's all crap and I send // so little mail anyway that I rarely get real bounces +// this plugin is deprecated. Use the 'bounce' plugin instead, and set +// config/bounce.ini reject_all=1 + +exports.register = function () { + this.logwarn("NOTICE: plugin deprecated, use 'bounce' instead!"); +} + exports.hook_mail = function (next, connection, params) { var mail_from = params[0]; if (mail_from.isNull()) { From 241d4890f76f62dfd4986f7488fea5d03c59fd66 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 03:00:26 -0500 Subject: [PATCH 098/160] dkim: bugfix, handle null sender properly --- plugins/dkim_sign.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/dkim_sign.js b/plugins/dkim_sign.js index c2e07896e..001bfe235 100644 --- a/plugins/dkim_sign.js +++ b/plugins/dkim_sign.js @@ -187,6 +187,8 @@ function getKeyDir(plugin, conn, cb) { // then we must parse and use the domain in the Sender header. // var domain = self.header.get('from').host; + if (conn.transaction.mail_from.isNull()) return cb(); // null sender + // In all cases I have seen, but likely not all cases, this suffices var domain = conn.transaction.mail_from.host; From d8311189a587bbdfaf8770efe5ad2c915c92e599 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 03:09:49 -0500 Subject: [PATCH 099/160] bounce: store invalid note in transaction note --- plugins/bounce.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/bounce.js b/plugins/bounce.js index a079bb191..9dec972bb 100644 --- a/plugins/bounce.js +++ b/plugins/bounce.js @@ -41,6 +41,7 @@ function has_empty_return_path(connection, plugin) { var rp = connection.transaction.header.get('Return-Path'); if (!rp) return; if (rp === '<>') return; + connection.transaction.notes.bounce='invalid'; connection.loginfo(plugin, "bounce messages must not have a Return-Path"); return "a bounce return path must be empty (RFC 3834)"; }; @@ -51,6 +52,7 @@ function has_single_recipient(connection, plugin) { connection.loginfo(plugin, "bogus bounce to: " + connection.transaction.rcpt_to.join(',')); + connection.transaction.notes.bounce='invalid'; return "this bounce message does not have 1 recipient"; }; From dc642d202cb389e5f181da0f43ac71e980ad7432 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 15:48:56 -0500 Subject: [PATCH 100/160] bounce: corrected config value use --- plugins/bounce.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/bounce.js b/plugins/bounce.js index 9dec972bb..beb3c3d66 100644 --- a/plugins/bounce.js +++ b/plugins/bounce.js @@ -1,30 +1,26 @@ // bounce tests exports.register = function () { - var plugin = this; - this.register_hook('mail', 'bounce_mail'); - // this.register_hook('rcpt', 'bounce_rcpt'); - // this.register_hook('data', 'bounce_data'); - this.register_hook('data_post', 'bounce_data_post'); + this.register_hook('data', 'bounce_data'); }; exports.bounce_mail = function (next, connection, params) { var mail_from = params[0]; if (!mail_from.isNull()) return next(); // not a bounce var cfg = this.config.get('bounce.ini'); - if (cfg.reject_all) + if (cfg.main.reject_all) return next(DENY, "No bounces accepted here"); return next(); } -exports.bounce_data_post = function(next, connection) { +exports.bounce_data = function(next, connection) { var plugin = connection; if (!has_null_sender(connection)) return next(); // not a bounce. var cfg = this.config.get('bounce.ini'); - var rej = cfg.reject_invalid; + var rej = cfg.main.reject_invalid; // Valid bounces have a single recipient var err = has_single_recipient(connection, plugin); From 9d91f126992596b1e7f5871b251510a5eced7e67 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 18:14:05 -0500 Subject: [PATCH 101/160] bounce: removed return-path header check --- plugins/bounce.js | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/plugins/bounce.js b/plugins/bounce.js index beb3c3d66..471f23645 100644 --- a/plugins/bounce.js +++ b/plugins/bounce.js @@ -9,8 +9,7 @@ exports.bounce_mail = function (next, connection, params) { var mail_from = params[0]; if (!mail_from.isNull()) return next(); // not a bounce var cfg = this.config.get('bounce.ini'); - if (cfg.main.reject_all) - return next(DENY, "No bounces accepted here"); + if (cfg.main.reject_all) return next(DENY, "No bounces accepted here"); return next(); } @@ -22,29 +21,16 @@ exports.bounce_data = function(next, connection) { var cfg = this.config.get('bounce.ini'); var rej = cfg.main.reject_invalid; - // Valid bounces have a single recipient var err = has_single_recipient(connection, plugin); if (err && rej) return next(DENY, err); - // validate that Return-Path is empty, RFC 3834 - err = has_empty_return_path(connection, plugin); - if (err && rej) return next(DENY, err); - return next(); }; -function has_empty_return_path(connection, plugin) { - var rp = connection.transaction.header.get('Return-Path'); - if (!rp) return; - if (rp === '<>') return; - connection.transaction.notes.bounce='invalid'; - connection.loginfo(plugin, "bounce messages must not have a Return-Path"); - return "a bounce return path must be empty (RFC 3834)"; -}; - function has_single_recipient(connection, plugin) { if (connection.transaction.rcpt_to.length === 1) return; + // Valid bounces have a single recipient connection.loginfo(plugin, "bogus bounce to: " + connection.transaction.rcpt_to.join(',')); From 5e23043722fb77de0c9a6a1681cf7573ec966a1c Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 18:43:29 -0500 Subject: [PATCH 102/160] bounce: now has_null_sender is called twice --- plugins/bounce.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/bounce.js b/plugins/bounce.js index 471f23645..a480cfd41 100644 --- a/plugins/bounce.js +++ b/plugins/bounce.js @@ -7,7 +7,7 @@ exports.register = function () { exports.bounce_mail = function (next, connection, params) { var mail_from = params[0]; - if (!mail_from.isNull()) return next(); // not a bounce + if (!has_null_sender(mail_from)) return next(); // not a bounce var cfg = this.config.get('bounce.ini'); if (cfg.main.reject_all) return next(DENY, "No bounces accepted here"); return next(); @@ -16,7 +16,7 @@ exports.bounce_mail = function (next, connection, params) { exports.bounce_data = function(next, connection) { var plugin = connection; - if (!has_null_sender(connection)) return next(); // not a bounce. + if (!has_null_sender(connection.transaction.mail_from)) return next(); var cfg = this.config.get('bounce.ini'); var rej = cfg.main.reject_invalid; @@ -38,6 +38,11 @@ function has_single_recipient(connection, plugin) { return "this bounce message does not have 1 recipient"; }; -function has_null_sender(connection) { - return connection.transaction.mail_from.isNull() ? true : false; +function has_null_sender(mail_from) { + // bounces have a null sender. + return mail_from.isNull() ? true : false; + + // this could also be tested with. + // mail_from.user ? false : true + // Why would isNull() exist if it wasn't the right way to test this? }; From 07a1bacf4d241302891f6443838411b9484d994c Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 12:48:29 -0500 Subject: [PATCH 103/160] import Steve's ip_country as connect.geoip --- plugins/connect.geoip.js | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 plugins/connect.geoip.js diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js new file mode 100644 index 000000000..fd3738be8 --- /dev/null +++ b/plugins/connect.geoip.js @@ -0,0 +1,60 @@ +var geoip = require('geoip-lite'); +var net = require('net'); + +exports.hook_connect = function (next, connection) { + connection.notes.geoip = geoip.lookup(connection.remote_ip); + if (connection.notes.geoip) { + connection.loginfo(this, 'country: ' + connection.notes.geoip.country); + } + return next(); +} + +exports.hook_data_post = function (next, connection) { + var txn = connection.transaction; + txn.remove_header('X-Haraka-GeoIP'); + txn.remove_header('X-Haraka-GeoIP-Received'); + if (connection.notes.geoip) { + txn.add_header('X-Haraka-GeoIP', connection.notes.geoip.country); + } + + var results = []; + var received = txn.header.get_all('received'); + if (received.length) { + // Try and parse each received header + for (var i=0; i < received.length; i++) { + var match = /\[(\d+\.\d+\.\d+\.\d+)\]/.exec(received[i]); + if (match && net.isIPv4(match[1])) { + var gi = geoip.lookup(match[1]); + connection.loginfo(this, 'received=' + match[1] + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); + results.push(match[1] + ':' + ((gi) ? gi.country : 'UNKNOWN')); + } + } + } + else { + // No received headers. + // Check for User-Agent + var ua = txn.header.get('user-agent'); + var xm = txn.header.get('x-mailer'); + var xmu = txn.header.get('x-mua'); + if (ua || xm || xmu) { + connection.loginfo(this, 'direct-to-mx?'); + } + } + // Try and parse any originating IP headers + var orig = txn.header.get('x-originating-ip') || + txn.header.get('x-ip') || + txn.header.get('x-remote-ip'); + if (orig) { + var match = /(\d+\.\d+\.\d+\.\d+)/.exec(orig); + if (match && net.isIPv4(match[1])) { + var gi = geoip.lookup(match[1]); + connection.loginfo(this, 'originating=' + match[1] + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); + results.push(match[1] + ':' + ((gi) ? gi.country : 'UNKNOWN')); + } + } + // Add any results to a trace header + if (results.length) { + txn.add_header('X-Haraka-GeoIP-Received', results.join(' ')); + } + return next(); +} From 470a6cda9cbe3671fe282d81357db1a05c779820 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sun, 19 Jan 2014 14:55:45 -0500 Subject: [PATCH 104/160] refactored: added display of region & city --- config/connect.geoip.ini | 17 +++++ docs/plugins/connect.geoip.md | 62 ++++++++++++++++++ plugins/connect.geoip.js | 117 +++++++++++++++++++++++----------- 3 files changed, 160 insertions(+), 36 deletions(-) create mode 100644 config/connect.geoip.ini create mode 100644 docs/plugins/connect.geoip.md diff --git a/config/connect.geoip.ini b/config/connect.geoip.ini new file mode 100644 index 000000000..b3ff8daa9 --- /dev/null +++ b/config/connect.geoip.ini @@ -0,0 +1,17 @@ +; enable distance calculations. If you don't use the distance, leave it +; disabled to save few CPU cycles. +calc_distance=0 + +; public_ip: the public IP address of *this* mail server +; if your mail server is not bound to a public IP, you'll have to provide +; this for distance calculations to work. +; public_ip= + + +; show_city: show city data in logs and headers +; note: city data is less accurate than country +show_city=1 + +; show_region: show regional data (US states, CA provinces, etc..) +show_region=1 + diff --git a/docs/plugins/connect.geoip.md b/docs/plugins/connect.geoip.md new file mode 100644 index 000000000..a829d3d10 --- /dev/null +++ b/docs/plugins/connect.geoip.md @@ -0,0 +1,62 @@ + +# geoip + +provide geographic information about mail senders. + +# SYNOPSIS + +Use MaxMind's GeoIP databases and the geoip-lite node module to report +geographic information about incoming connections. + +# DESCRIPTION + +This plugin stores results in connection.notes.geoip. The following +keys are typically available: + + range: [ 3479299040, 3479299071 ], + country: 'US', + region: 'CA', + city: 'San Francisco', + ll: [37.7484, -122.4156], + distance: 1539 // in kilometers + +Adds entries like this to your logs: + + [connect.geoip] US + [connect.geoip] US, WA + [connect.geoip] US, WA, Seattle + [connect.geoip] US, WA, Seattle, 1319km + +Calculating the distance requires the public IP of this mail server. This may +be the IP that Haraka is bound to, but if not you'll need to supply it. + +# CONFIG + +- distance + +Perform the geodesic distance calculations. Calculates the distance "as the +crow flies" from the remote mail server. + +- public_ip: + +The IP address to calculate the distance from. This will typically be +the public IP of your mail server. + + +# LIMITATIONS + +The distance calculations are more concerned with being fast than +accurate. The MaxMind location data is collected from whois and is of +limited accuracy. MaxMind offers more accurate data for a fee. + +For distance calculations, the earth is considered a perfect sphere. In +reality, it is not. Accuracy should be within 1%. + +This plugin does not update the GeoIP databases. You may want to. + + +# SEE ALSO + +MaxMind: http://www.maxmind.com/ + +Databases: http://geolite.maxmind.com/download/geoip/database diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index fd3738be8..369ea7e40 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -3,58 +3,103 @@ var net = require('net'); exports.hook_connect = function (next, connection) { connection.notes.geoip = geoip.lookup(connection.remote_ip); - if (connection.notes.geoip) { - connection.loginfo(this, 'country: ' + connection.notes.geoip.country); - } + + if (!connection.notes.geoip) return next(); + + var cfg = this.config.get('connect.geoip.ini'); + connection.loginfo(this, get_results(connection, cfg)); + return next(); } +function get_results(connection, cfg) { + var r = connection.notes.geoip; + if (!r) return ''; + + // geoip.lookup results look like this: + // range: [ 3479299040, 3479299071 ], + // country: 'US', + // region: 'CA', + // city: 'San Francisco', + // ll: [37.7484, -122.4156] + + var show = [ r.country ]; + if ( r.region && cfg.main.show_region) show.push(r.region); + if ( r.city && cfg.main.show_city ) show.push(r.city); + + return show.join(', '); +}; + exports.hook_data_post = function (next, connection) { var txn = connection.transaction; txn.remove_header('X-Haraka-GeoIP'); txn.remove_header('X-Haraka-GeoIP-Received'); if (connection.notes.geoip) { - txn.add_header('X-Haraka-GeoIP', connection.notes.geoip.country); + var cfg = this.config.get('connect.geoip.ini'); + txn.add_header('X-Haraka-GeoIP', get_results(connection, cfg)); } - var results = []; - var received = txn.header.get_all('received'); + var received = []; + + var rh = received_headers(connection, this); + if (rh) received.push(rh); + if (!rh) user_agent(connection, this); // No received headers. + + var oh = originating_headers(connection, this); + if (oh) received.push(oh); + + // Add any received results to a trace header if (received.length) { - // Try and parse each received header - for (var i=0; i < received.length; i++) { - var match = /\[(\d+\.\d+\.\d+\.\d+)\]/.exec(received[i]); - if (match && net.isIPv4(match[1])) { - var gi = geoip.lookup(match[1]); - connection.loginfo(this, 'received=' + match[1] + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); - results.push(match[1] + ':' + ((gi) ? gi.country : 'UNKNOWN')); - } - } + txn.add_header('X-Haraka-GeoIP-Received', received.join(' ')); } - else { - // No received headers. - // Check for User-Agent - var ua = txn.header.get('user-agent'); - var xm = txn.header.get('x-mailer'); - var xmu = txn.header.get('x-mua'); - if (ua || xm || xmu) { - connection.loginfo(this, 'direct-to-mx?'); - } + return next(); +}; + +function user_agent(connection, plugin) { + // Check for User-Agent + var ua = connection.transaction.header.get('user-agent'); + var xm = connection.transaction.header.get('x-mailer'); + var xmu = connection.transaction.header.get('x-mua'); + if (ua || xm || xmu) { + connection.loginfo(plugin, 'direct-to-mx?'); } - // Try and parse any originating IP headers - var orig = txn.header.get('x-originating-ip') || - txn.header.get('x-ip') || - txn.header.get('x-remote-ip'); - if (orig) { - var match = /(\d+\.\d+\.\d+\.\d+)/.exec(orig); +}; + +function received_headers(connection, plugin) { + var txn = connection.transaction; + var received = txn.header.get_all('received'); + if (!received.length) return; + + var results = []; + + // Try and parse each received header + for (var i=0; i < received.length; i++) { + var match = /\[(\d+\.\d+\.\d+\.\d+)\]/.exec(received[i]); if (match && net.isIPv4(match[1])) { var gi = geoip.lookup(match[1]); - connection.loginfo(this, 'originating=' + match[1] + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); + connection.loginfo(plugin, 'received=' + match[1] + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); results.push(match[1] + ':' + ((gi) ? gi.country : 'UNKNOWN')); } } - // Add any results to a trace header - if (results.length) { - txn.add_header('X-Haraka-GeoIP-Received', results.join(' ')); - } - return next(); + return results; +}; + +function originating_headers(connection, plugin) { + var txn = connection.transaction; + + // Try and parse any originating IP headers + var orig = txn.header.get('x-originating-ip') || + txn.header.get('x-ip') || + txn.header.get('x-remote-ip'); + + if (!orig) return; + + var match = /(\d+\.\d+\.\d+\.\d+)/.exec(orig); + if (!match) return; + var found_ip = match[1]; + if (!net.isIPv4(found_ip)) return; + + var gi = geoip.lookup(found_ip); + connection.loginfo(plugin, 'originating=' + found_ip + ' country=' + ((gi) ? gi.country : 'UNKNOWN')); + return found_ip + ':' + ((gi) ? gi.country : 'UNKNOWN'); } From 954f86cfce14bc03fbc969d3dff046f9ede1c4fa Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 04:01:05 -0500 Subject: [PATCH 105/160] geoip: added distance calculations --- plugins/connect.geoip.js | 116 ++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 19 deletions(-) diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index 369ea7e40..a5e097851 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -1,35 +1,22 @@ var geoip = require('geoip-lite'); var net = require('net'); +var local_ip, local_geoip; + exports.hook_connect = function (next, connection) { + var plugin = this; connection.notes.geoip = geoip.lookup(connection.remote_ip); if (!connection.notes.geoip) return next(); var cfg = this.config.get('connect.geoip.ini'); - connection.loginfo(this, get_results(connection, cfg)); + calculate_distance(plugin, connection, cfg); + + connection.loginfo(plugin, get_results(connection, cfg)); return next(); } -function get_results(connection, cfg) { - var r = connection.notes.geoip; - if (!r) return ''; - - // geoip.lookup results look like this: - // range: [ 3479299040, 3479299071 ], - // country: 'US', - // region: 'CA', - // city: 'San Francisco', - // ll: [37.7484, -122.4156] - - var show = [ r.country ]; - if ( r.region && cfg.main.show_region) show.push(r.region); - if ( r.city && cfg.main.show_city ) show.push(r.city); - - return show.join(', '); -}; - exports.hook_data_post = function (next, connection) { var txn = connection.transaction; txn.remove_header('X-Haraka-GeoIP'); @@ -55,6 +42,97 @@ exports.hook_data_post = function (next, connection) { return next(); }; +function calculate_distance(plugin, connection, cfg) { + if (!cfg.main.calc_distance) return; + + if (!local_ip) { local_ip = cfg.main.public_ip; }; + if (!local_ip) { local_ip = connection.local_ip; }; + if (!local_ip) return; + + if (!local_geoip) { + local_geoip = geoip.lookup(local_ip) + connection.loginfo(plugin, "local_geoip set: "+local_geoip); + }; + if (!local_geoip) return; + +// two formulas for calculating Great Circle Distance. Perhaps worth +// benchmarking them? + +// var gcd = geodatasource(local_geoip.ll[0], local_geoip.ll[1], +// connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); + + var gcd = haversine(local_geoip.ll[0], local_geoip.ll[1], + connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); + + connection.notes.geoip.distance = gcd; +}; + +function haversine(lat1, lon1, lat2, lon2) { + var R = 6371; // km + function toRad(v) { return v * Math.PI / 180; }; + var dLat = toRad(lat2-lat1); + var dLon = toRad(lon2-lon1); + var lat1 = toRad(lat1); + var lat2 = toRad(lat2); + + var a = Math.sin(dLat/2) * Math.sin(dLat/2) + + Math.sin(dLon/2) * Math.sin(dLon/2) * Math.cos(lat1) * Math.cos(lat2); + var c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a)); + var d = R * c; + return d.toFixed(0); +} + +function geodatasource(lat1, lon1, lat2, lon2) { +//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +//::: From: http://www.geodatasource.com/developers/javascript +//::: ::: +//::: This routine calculates the distance between two points (given the ::: +//::: latitude/longitude of those points). +//::: ::: +//::: Definitions: ::: +//::: South latitudes are negative, east longitudes are positive ::: +//::: ::: +//::: Passed to function: ::: +//::: lat1, lon1 = Latitude and Longitude of point 1 (in decimal degrees) ::: +//::: lat2, lon2 = Latitude and Longitude of point 2 (in decimal degrees) ::: +//::: ::: +//::: GeoDataSource.com (C) All Rights Reserved 2014 ::: +//::: ::: +//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + + var radlat1 = Math.PI * lat1/180; + var radlat2 = Math.PI * lat2/180; + var radlon1 = Math.PI * lon1/180; + var radlon2 = Math.PI * lon2/180; + var theta = lon1-lon2; + var radtheta = Math.PI * theta/180; + var dist = Math.sin(radlat1) * Math.sin(radlat2) + Math.cos(radlat1) * Math.cos(radlat2) * Math.cos(radtheta); + dist = Math.acos(dist); + dist = dist * 180/Math.PI; + dist = dist * 60 * 1.1515; + dist = dist * 1.609344; + return dist.toFixed(0); +} + +function get_results(connection, cfg) { + var r = connection.notes.geoip; + if (!r) return ''; + + // geoip.lookup results look like this: + // range: [ 3479299040, 3479299071 ], + // country: 'US', + // region: 'CA', + // city: 'San Francisco', + // ll: [37.7484, -122.4156] + + var show = [ r.country ]; + if ( r.region && cfg.main.show_region) show.push(r.region); + if ( r.city && cfg.main.show_city ) show.push(r.city); + if ( r.distance && cfg.main.calc_distance ) show.push(r.distance+'km'); + + return show.join(', '); +}; + function user_agent(connection, plugin) { // Check for User-Agent var ua = connection.transaction.header.get('user-agent'); From 7d2e97bead608294de3ffceeff6eda4b1022711e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 19:41:16 -0500 Subject: [PATCH 106/160] geoip: remove geodatasource GCD function and clean up whitespace --- plugins/connect.geoip.js | 46 +++++----------------------------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index a5e097851..10558e0ec 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -55,12 +55,6 @@ function calculate_distance(plugin, connection, cfg) { }; if (!local_geoip) return; -// two formulas for calculating Great Circle Distance. Perhaps worth -// benchmarking them? - -// var gcd = geodatasource(local_geoip.ll[0], local_geoip.ll[1], -// connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); - var gcd = haversine(local_geoip.ll[0], local_geoip.ll[1], connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); @@ -68,6 +62,8 @@ function calculate_distance(plugin, connection, cfg) { }; function haversine(lat1, lon1, lat2, lon2) { + // calculate the great circle distance using the haversine formula + // found here: http://www.movable-type.co.uk/scripts/latlong.html var R = 6371; // km function toRad(v) { return v * Math.PI / 180; }; var dLat = toRad(lat2-lat1); @@ -82,38 +78,6 @@ function haversine(lat1, lon1, lat2, lon2) { return d.toFixed(0); } -function geodatasource(lat1, lon1, lat2, lon2) { -//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -//::: From: http://www.geodatasource.com/developers/javascript -//::: ::: -//::: This routine calculates the distance between two points (given the ::: -//::: latitude/longitude of those points). -//::: ::: -//::: Definitions: ::: -//::: South latitudes are negative, east longitudes are positive ::: -//::: ::: -//::: Passed to function: ::: -//::: lat1, lon1 = Latitude and Longitude of point 1 (in decimal degrees) ::: -//::: lat2, lon2 = Latitude and Longitude of point 2 (in decimal degrees) ::: -//::: ::: -//::: GeoDataSource.com (C) All Rights Reserved 2014 ::: -//::: ::: -//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: - - var radlat1 = Math.PI * lat1/180; - var radlat2 = Math.PI * lat2/180; - var radlon1 = Math.PI * lon1/180; - var radlon2 = Math.PI * lon2/180; - var theta = lon1-lon2; - var radtheta = Math.PI * theta/180; - var dist = Math.sin(radlat1) * Math.sin(radlat2) + Math.cos(radlat1) * Math.cos(radlat2) * Math.cos(radtheta); - dist = Math.acos(dist); - dist = dist * 180/Math.PI; - dist = dist * 60 * 1.1515; - dist = dist * 1.609344; - return dist.toFixed(0); -} - function get_results(connection, cfg) { var r = connection.notes.geoip; if (!r) return ''; @@ -126,9 +90,9 @@ function get_results(connection, cfg) { // ll: [37.7484, -122.4156] var show = [ r.country ]; - if ( r.region && cfg.main.show_region) show.push(r.region); - if ( r.city && cfg.main.show_city ) show.push(r.city); - if ( r.distance && cfg.main.calc_distance ) show.push(r.distance+'km'); + if (r.region && cfg.main.show_region ) show.push(r.region); + if (r.city && cfg.main.show_city ) show.push(r.city); + if (r.distance && cfg.main.calc_distance) show.push(r.distance+'km'); return show.join(', '); }; From 270028f84c3466e45e6090b249bac474b018137f Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 19:45:11 -0500 Subject: [PATCH 107/160] geoip: added show_city and show_region options to docs --- docs/plugins/connect.geoip.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/plugins/connect.geoip.md b/docs/plugins/connect.geoip.md index a829d3d10..99a389081 100644 --- a/docs/plugins/connect.geoip.md +++ b/docs/plugins/connect.geoip.md @@ -42,6 +42,13 @@ crow flies" from the remote mail server. The IP address to calculate the distance from. This will typically be the public IP of your mail server. +- show_city + +show city data in logs and headers. City data is less accurate than country. + +- show_region in logs and headers. Regional data are US states, Canadian + provinces and such. + # LIMITATIONS From 321c97155873536a106a5973be0123ed53925bd4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 20:37:32 -0500 Subject: [PATCH 108/160] geoip: removed a debugging comment --- plugins/connect.geoip.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index 10558e0ec..fa979bd11 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -49,10 +49,7 @@ function calculate_distance(plugin, connection, cfg) { if (!local_ip) { local_ip = connection.local_ip; }; if (!local_ip) return; - if (!local_geoip) { - local_geoip = geoip.lookup(local_ip) - connection.loginfo(plugin, "local_geoip set: "+local_geoip); - }; + if (!local_geoip) { local_geoip = geoip.lookup(local_ip) }; if (!local_geoip) return; var gcd = haversine(local_geoip.ll[0], local_geoip.ll[1], From 793d5b3a2e698dd4f36079da30ec2806185c5f32 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 00:33:40 -0500 Subject: [PATCH 109/160] restored heavily commented and disabled data_from explaining why Return-Path checking isn't enabled --- plugins/bounce.js | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/plugins/bounce.js b/plugins/bounce.js index a480cfd41..e840df839 100644 --- a/plugins/bounce.js +++ b/plugins/bounce.js @@ -3,6 +3,7 @@ exports.register = function () { this.register_hook('mail', 'bounce_mail'); this.register_hook('data', 'bounce_data'); +// this.register_hook('data_post', 'bounce_data_post'); }; exports.bounce_mail = function (next, connection, params) { @@ -15,7 +16,6 @@ exports.bounce_mail = function (next, connection, params) { exports.bounce_data = function(next, connection) { var plugin = connection; - if (!has_null_sender(connection.transaction.mail_from)) return next(); var cfg = this.config.get('bounce.ini'); @@ -27,12 +27,39 @@ exports.bounce_data = function(next, connection) { return next(); }; +exports.bounce_data_post = function(next, connection) { + return next(); + + // Bounce messages generally do not have a Return-Path set. This checks + // for that. But whether it should is worth questioning... + + // On Jan 20, 2014, Matt Simerson examined the most recent 50,000 mail + // connections for the presence of Return-Path in bounce messages. I + // found 14 hits, 12 of which were from Google, in response to + // undeliverable DMARC reports (IE, automated messages that Google + // shouldn't have replied to). Another appears to be a valid bounce from + // a poorly configured mailer, and the 14th was a confirmed spam kill. + // Unless new data demonstrate otherwise, this should remain disabled. + + // Return-Path, aka Reverse-PATH, Envelope FROM, RFC5321.MailFrom + // validate that the Return-Path header is empty, RFC 3834 + + if (!has_null_sender(connection.transaction.mail_from)) return next(); + var plugin = connection; + var rp = connection.transaction.header.get('Return-Path'); + if (rp && rp !== '<>') { + connection.loginfo(plugin, "bounce with non-empty Return-Path"); + return next(DENY, "bounce with non-empty Return-Path (RFC 3834)"); + }; + return next(); +}; + function has_single_recipient(connection, plugin) { if (connection.transaction.rcpt_to.length === 1) return; // Valid bounces have a single recipient - connection.loginfo(plugin, "bogus bounce to: " + - connection.transaction.rcpt_to.join(',')); + connection.loginfo(plugin, "bounce with too many recipients to: " + + connection.transaction.rcpt_to.join(',')); connection.transaction.notes.bounce='invalid'; return "this bounce message does not have 1 recipient"; From d765edfe7598e012d29888a7f2545063e05e4163 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 21 Jan 2014 15:04:32 -0500 Subject: [PATCH 110/160] Fix all underscores (to be escaped). --- README.md | 2 +- docs/Config.md | 2 +- docs/Connection.md | 16 +++++------ docs/CoreConfig.md | 32 +++++++++++----------- docs/Header.md | 4 +-- docs/Net_Utils.md | 2 +- docs/Outbound.md | 12 ++++---- docs/Plugins.md | 32 +++++++++++----------- docs/Transaction.md | 26 +++++++++--------- docs/plugins/aliases.md | 2 +- docs/plugins/auth/auth_ldap.md | 2 +- docs/plugins/auth/auth_proxy.md | 2 +- docs/plugins/auth/auth_vpopmaild.js | 2 +- docs/plugins/auth/flat_file.md | 2 +- docs/plugins/avg.md | 4 +-- docs/plugins/block_me.md | 2 +- docs/plugins/clamd.md | 10 +++---- docs/plugins/connect.rdns_access.md | 22 +++++++-------- docs/plugins/daemonize.md | 6 ++-- docs/plugins/data.rfc5322_header_checks.md | 2 +- docs/plugins/data.uribl.md | 8 +++--- docs/plugins/dkim_sign.md | 10 +++---- docs/plugins/dnsbl.md | 8 +++--- docs/plugins/dnswl.md | 10 +++---- docs/plugins/early_talker.md | 6 ++-- docs/plugins/graph.md | 8 +++--- docs/plugins/helo.checks.md | 14 +++++----- docs/plugins/log.syslog.md | 2 +- docs/plugins/lookup_rdns.strict.md | 26 +++++++++--------- docs/plugins/mail_from.access.md | 22 +++++++-------- docs/plugins/mail_from.blocklist.md | 8 +++--- docs/plugins/mail_from.is_resolvable.md | 6 ++-- docs/plugins/mail_from.nobounces.md | 2 +- docs/plugins/max_unrecognized_commands.md | 6 ++-- docs/plugins/messagesniffer.md | 12 ++++---- docs/plugins/process_title.md | 2 +- docs/plugins/queue/quarantine.md | 2 +- docs/plugins/queue/smtp_forward.md | 10 +++---- docs/plugins/queue/smtp_proxy.md | 10 +++---- docs/plugins/rate_limit.md | 18 ++++++------ docs/plugins/rcpt_to.access.md | 26 +++++++++--------- docs/plugins/rcpt_to.blocklist.md | 8 +++--- docs/plugins/rcpt_to.in_host_list.md | 6 ++-- docs/plugins/rcpt_to.max_count.md | 4 +-- docs/plugins/rcpt_to.qmail_deliverable.md | 5 ++-- docs/plugins/rdns.regexp.md | 8 +++--- docs/plugins/relay_acl.md | 6 ++-- docs/plugins/relay_all.md | 2 +- docs/plugins/relay_force_routing.md | 8 +++--- docs/plugins/reseed_rng.md | 4 +-- docs/plugins/spf.md | 16 +++++------ docs/plugins/toobusy.md | 2 +- docs/tutorials/Migrating_from_v1_to_v2.md | 4 +-- docs/tutorials/SettingUpOutbound.md | 4 +-- 54 files changed, 238 insertions(+), 239 deletions(-) diff --git a/README.md b/README.md index ddf3b7cbb..09d0d2087 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ code in Haraka, or maybe someone has already written this plugin. Plugins are already provided for running mail through SpamAssassin, checking for known bad HELO patterns, checking DNS Blocklists, and watching for -violators of the SMTP protocol via the "early_talker" plugin. +violators of the SMTP protocol via the "early\_talker" plugin. Furthermore Haraka comes with a simple plugin called "graph" which shows you real-time charts of which plugins rejected the most mail, allowing you to diff --git a/docs/Config.md b/docs/Config.md index d0d73aeb0..cb32b9023 100644 --- a/docs/Config.md +++ b/docs/Config.md @@ -112,7 +112,7 @@ var configfile = require('./configfile'); var cfg = configfile.read_config('/path/to/file', type); ``` -read_config() handles the caching for you and will return cached values +`read_config()` handles the caching for you and will return cached values if there have been no updates since the file was read. You can also optionally pass in a callback that is run if the file is diff --git a/docs/Connection.md b/docs/Connection.md index 66e125931..18e5a55c1 100644 --- a/docs/Connection.md +++ b/docs/Connection.md @@ -10,19 +10,19 @@ API A unique UUID for this connection. -* connection.remote_ip +* connection.remote\_ip The remote IP address -* connection.remote_host +* connection.remote\_host The rDNS of the remote IP -* connection.local_ip +* connection.local\_ip The bound IP address of the server as reported by the OS -* connection.local_port +* connection.local\_port The bound port number of the server which is handling the connection. If you have specified multiple listen= ports this variable is useful @@ -33,7 +33,7 @@ port Either 'EHLO' or 'HELO' whichever the remote end used -* connection.hello_host +* connection.hello\_host The hostname given to HELO or EHLO @@ -53,15 +53,15 @@ A boolean flag to say whether this connection is allowed to relay mails (i.e. deliver mails outbound). This is normally set by SMTP AUTH, or sometimes via an IP address check. -* connection.current_line +* connection.current\_line For low level use. Contains the current line sent from the remote end, verbatim as it was sent. Can be useful in certain botnet detection techniques. -* connection.last_response +* connection.last\_response Contains the last SMTP response sent to the client. -* connection.remote_close +* connection.remote\_close For low level use. This value is set when the remote host drops the connection. diff --git a/docs/CoreConfig.md b/docs/CoreConfig.md index 1c6a76260..b54ceb665 100644 --- a/docs/CoreConfig.md +++ b/docs/CoreConfig.md @@ -28,14 +28,14 @@ different levels available. children as there are CPUs (default: 0, which disables cluster mode) * user - optionally a user to drop privileges to. Can be a string or UID. * group - optionally a group to drop privileges to. Can be a string or GID. - * ignore_bad_plugins - If a plugin fails to compile by default Haraka will stop at load time. + * ignore\_bad\_plugins - If a plugin fails to compile by default Haraka will stop at load time. If, however, you wish to continue on without that plugin's facilities, then set this config option * daemonize - enable this to cause Haraka to fork into the background on start-up (default: 0) - * daemon_log_file - (default: /var/log/haraka.log) where to redirect stdout/stderr when daemonized - * daemon_pid_file - (default: /var/run/haraka.pid) where to write a PID file to - * spool_dir - (default: none) directory to create temporary spool files in - * spool_after - (default: -1) if message exceeds this size in bytes, then spool the message to disk + * daemon\_log\_file - (default: /var/log/haraka.log) where to redirect stdout/stderr when daemonized + * daemon\_pid\_file - (default: /var/run/haraka.pid) where to write a PID file to + * spool\_dir - (default: none) directory to create temporary spool files in + * spool\_after - (default: -1) if message exceeds this size in bytes, then spool the message to disk specify -1 to disable spooling completely or 0 to force all messages to be spooled to disk. [1]: http://learnboost.github.com/cluster/ or node version >= 0.8 @@ -45,7 +45,7 @@ different levels available. A name to use for this server. Used in received lines and elsewhere. Setup by default to be your hostname. -* deny_includes_uuid +* deny\_includes\_uuid Each connection and mail in Haraka includes a UUID which is also in most log messages. If you put a `1` in this file then every denied mail (either via @@ -57,7 +57,7 @@ different levels available. file, it will be truncated to that length. We recommend a 6 as a good balance of finding in the logs and not making lines too long. -* banner_include_uuid +* banner\_include\_uuid This will add the full UUID to the first line of the SMTP greeting banner. @@ -94,19 +94,19 @@ different levels available. if required (this may cause some connecting machines to fail - though usually only spam-bots). -* max_received_count +* max\_received\_count The maximum number of "Received" headers allowed in an email. This is a simple protection against mail loops. Defaults to 100. -* max_line_length +* max\_line\_length The maximum length of lines in SMTP session commands (e.g. RCPT, HELO etc). Defaults to 512 (bytes) which is mandated by RFC 5321 §4.5.3.1.4. Clients exceeding this limit will be immediately disconnected with a "521 Command line too long" error. -* max_data_line_length +* max\_data\_line\_length The maximum length of lines in the DATA section of emails. Defaults to 992 (bytes) which is the limit set by Sendmail. When this limit is exceeded the @@ -116,7 +116,7 @@ different levels available. as Sendmail. Also when the data line length limit is exceeded `transaction.notes.data_line_length_exceeded` is set to `true`. -* outbound.concurrency_max +* outbound.concurrency\_max Maximum concurrency to use when delivering mails outbound. Defaults to 100. @@ -125,23 +125,23 @@ different levels available. Put a `1` in this file to temporarily disable outbound delivery. Useful to do while you're figuring out network issues, or just testing things. -* outbound.bounce_message +* outbound.bounce\_message The bounce message should delivery of the mail fail. See the source of. The default is normally fine. Bounce messages contain a number of template replacement values which are best discovered by looking at the source code. -* haproxy_hosts +* haproxy\_hosts A list of HAProxy hosts that Haraka should enable the PROXY protocol from. See HAProxy.md -* strict_rfc1869 +* strict\_rfc1869 When enabled, this setting requires senders to conform to RFC 1869 and RFC 821 when sending the MAIL FROM and RCPT TO commands. In particular, the inclusion of spurious spaces or missing angle brackets will be rejected. - to enable: echo '1' > /path/to/haraka/config/strict_rfc1869 - to disable: echo '0' > /path/to/haraka/config/strict_rfc1869 + to enable: `echo 1 > /path/to/haraka/config/strict_rfc1869` + to disable: `echo 0 > /path/to/haraka/config/strict_rfc1869` diff --git a/docs/Header.md b/docs/Header.md index 753a19d39..0089206b8 100644 --- a/docs/Header.md +++ b/docs/Header.md @@ -14,12 +14,12 @@ Returns the header with the name `key`. If there are multiple headers with the given name (as is usually the case with "Received" for example) they will be concatenated together with "\n". -* header.get_all(key) +* header.get\_all(key) Returns the headers with the name `key` as an array. Multi-valued headers will have multiple entries in the array. -* header.get_decoded(key) +* header.get\_decoded(key) Works like `get(key)`, only it gives you headers decoded from any MIME encoding they may have used. diff --git a/docs/Net_Utils.md b/docs/Net_Utils.md index 69cfed162..b0a3c894f 100644 --- a/docs/Net_Utils.md +++ b/docs/Net_Utils.md @@ -1,4 +1,4 @@ -Net_Utils +Net\_Utils ========= This module provides network utility functions. diff --git a/docs/Outbound.md b/docs/Outbound.md index c4424afce..15aac84c6 100644 --- a/docs/Outbound.md +++ b/docs/Outbound.md @@ -20,13 +20,13 @@ process with the SIGHUP signal (via the `kill` command line tool). Outbound Configuration Files ---------------------------- -### outbound.concurrency_max +### outbound.concurrency\_max Default: 100. Specifies the maximum concurrent connections to make. Note that if using cluster (multiple CPUs) then this will be multiplied by the number of CPUs that you have. -### outbound.enable_tls +### outbound.enable\_tls Default: 0. Put a "1" in this file to enable TLS for outbound mail when the remote end is capable of receiving TLS connections. @@ -35,7 +35,7 @@ This uses the same `tls_key.pem` and `tls_cert.pem` files that the `tls` plugin uses. See the plugin documentation for information on generating those files. -### outbound.bounce_message +### outbound.bounce\_message See "Bounce Messages" below for details. @@ -48,7 +48,7 @@ how Haraka watches for config file changes. Outbound Mail Hooks ------------------- -### The queue_outbound hook +### The queue\_outbound hook The first hook that is called prior to queueing an outbound mail is the `queue_outbound` hook. Only if all these hooks return `CONT` (or if there are @@ -57,7 +57,7 @@ indicate that the mail has been queued in some custom manner for outbound delivery. Any of the `DENY` return codes will cause the message to be appropriately rejected. -### The get_mx hook +### The get\_mx hook Upon starting delivery the `get_mx` hook is called, with the parameter set to the domain in question (for example a mail to `user@example.com` will call the @@ -86,7 +86,7 @@ parameter is the error message received from the remote end. If you do not wish to have a bounce message sent to the originating sender of the email then you can return `OK` from this hook to stop it from sending a bounce message. -The variable hmail.bounce_extra can be accessed from this hook. This is an +The variable `hmail.bounce_extra` can be accessed from this hook. This is an Object which contains each recipient as the key and the value is the code and response received from the upstream server for that recipient. diff --git a/docs/Plugins.md b/docs/Plugins.md index c48dd4ca9..9c0aefceb 100644 --- a/docs/Plugins.md +++ b/docs/Plugins.md @@ -123,10 +123,10 @@ on the deny hook will override the result to CONT. Once a plugin calls next(OK) no further plugins on the same hook will run after it. -* HOOK_NEXT +* HOOK\_NEXT This is a special return value that is currently only available on the -unrecognized_command hook. It instructs Haraka to run a different plugin +`unrecognized_command` hook. It instructs Haraka to run a different plugin hook instead of responding normally. The `msg` argument is required and must be set to the name of the hook that is to be run. @@ -136,12 +136,12 @@ Available Hooks These are just the name of the hook, with any parameter sent to it: -* init_master - called when the main (master) process is started -* init_child - called whenever a child process is started when using multiple "nodes" -* lookup_rdns - called to look up the rDNS - return the rDNS via `next(OK, rdns)` +* init\_master - called when the main (master) process is started +* init\_child - called whenever a child process is started when using multiple "nodes" +* lookup\_rdns - called to look up the rDNS - return the rDNS via `next(OK, rdns)` * connect - called after we got rDNS * capabilities - called to get the ESMTP capabilities (such as STARTTLS) -* unrecognized_command - called when the remote end sends a command we don't recognise +* unrecognized\_command - called when the remote end sends a command we don't recognise * disconnect - called upon disconnect * helo (hostname) * ehlo (hostname) @@ -151,20 +151,20 @@ These are just the name of the hook, with any parameter sent to it: * rset * mail ([from, esmtp\_params]) * rcpt ([to, esmtp\_params]) -* rcpt_ok (to) +* rcpt\_ok (to) * data - called at the DATA command -* data_post - called at the end-of-data marker -* max_data_exceeded - called if the message is bigger than connection.max_bytes +* data\_post - called at the end-of-data marker +* max\_data\_exceeded - called if the message is bigger than connection.max\_bytes * queue - called to queue the mail -* queue_outbound - called to queue the mail when connection.relaying is set -* queue_ok - called when a mail has been queued successfully -* reset_transaction - called before the transaction is reset (via RSET, or MAIL) +* queue\_outbound - called to queue the mail when connection.relaying is set +* queue\_ok - called when a mail has been queued successfully +* reset\_transaction - called before the transaction is reset (via RSET, or MAIL) * deny - called if a plugin returns one of DENY, DENYSOFT or DENYDISCONNECT -* get_mx (hmail, domain) - called when sending outbound mail to lookup the MX record +* get\_mx (hmail, domain) - called when sending outbound mail to lookup the MX record * bounce (hmail, err) - called when sending outbound mail if the mail would bounce * delivered (hmail, [host, ip, response, delay]) - callen when outbound mail is delivered to the destination -* send_email (hmail) - called when outbound is about to be sent +* send\_email (hmail) - called when outbound is about to be sent The `rcpt` hook is slightly special. If we have a plugin (prior to rcpt) that sets the `connection.relaying = true` flag, then we do not need any rcpt @@ -176,7 +176,7 @@ obvious choice for this activity is the `rcpt_to.in_host_list` plugin, which lists the domains for which you wish to receive email. If a rcpt plugin DOES call `next(OK)` then the `rcpt_ok` hook is run. This -is primarily used by the queue/smtp_proxy plugin which needs to run after +is primarily used by the `queue/smtp_proxy` plugin which needs to run after all rcpt hooks. Sharing State @@ -204,7 +204,7 @@ plugins. This is done using `notes` - there are three types available: * connection.transaction.notes Available on any hook that passes 'connection' as a function parameter - between hook_mail and hook_data_post. + between hook\_mail and hook\_data\_post. This is shared amongst all plugins for this transaction (e.g. MAIL FROM through until a message is received or the connection is reset). Typical uses for notes at this level would be to store information diff --git a/docs/Transaction.md b/docs/Transaction.md index 8aede5ff3..aaa8eea14 100644 --- a/docs/Transaction.md +++ b/docs/Transaction.md @@ -19,7 +19,7 @@ The value of the MAIL FROM command as an `Address` object. An Array of `Address` objects of recipients from the RCPT TO command. -* transaction.message_stream +* transaction.message\_stream A node.js Readable Stream object for the message. @@ -48,7 +48,7 @@ e.g. The number of bytes in the email after DATA. -* transaction.add_data(line) +* transaction.add\_data(line) Adds a line of data to the email. Note this is RAW email - it isn't useful for adding banners to the email. @@ -57,16 +57,16 @@ for adding banners to the email. A safe place to store transaction specific values. -* transaction.add_leading_header(key, value) +* transaction.add\_leading\_header(key, value) Adds a header to the top of the header list. This should only be used in -very specific cases. Most people will want to use add_header() instead. +very specific cases. Most people will want to use `add_header()` instead. -* transaction.add_header(key, value) +* transaction.add\_header(key, value) Adds a header to the email. -* transaction.remove_header(key) +* transaction.remove\_header(key) Deletes a header from the email. @@ -74,16 +74,16 @@ Deletes a header from the email. The header of the email. See `Header Object`. -* transaction.parse_body = true|false [default: false] +* transaction.parse\_body = true|false [default: false] Set to `true` to enable parsing of the mail body. Make sure you set this in -hook_data or before. +hook\_data or before. * transaction.body The body of the email if you set `parse_body` above. See `Body Object`. -* transaction.attachment_hooks(start) +* transaction.attachment\_hooks(start) Sets a callback for when we see an attachment if `parse_body` has been set. @@ -132,15 +132,15 @@ the `tmp` library from npm and tells us the size of the file: }); } -* transaction.discard_data = true|false [default: false] +* transaction.discard\_data = true|false [default: false] Set this flag to true to discard all data as it arrives and not store in -memory or on disk (in the message_stream property). You can still access -the attachments and body if you set parse_body to true. This is useful +memory or on disk (in the message\_stream property). You can still access +the attachments and body if you set parse\_body to true. This is useful for systems which do not need the full email, just the attachments or mail text. -* transaction.set_banner(text, html) +* transaction.set\_banner(text, html) Sets a banner to be added to the end of the email. If the html part is not given (optional) then the text part will have each line ending replaced with diff --git a/docs/plugins/aliases.md b/docs/plugins/aliases.md index 3c8cbae8e..c8e68eca2 100644 --- a/docs/plugins/aliases.md +++ b/docs/plugins/aliases.md @@ -7,7 +7,7 @@ a JSON formatted configuration file, and must have at very least an action. Any syntax error found in the JSON format config file will stop the server from running. -WARNING: DO NOT USE THIS PLUGIN WITH queue/smtp_proxy. +WARNING: DO NOT USE THIS PLUGIN WITH queue/smtp\_proxy. Configuration ------------- diff --git a/docs/plugins/auth/auth_ldap.md b/docs/plugins/auth/auth_ldap.md index c9eb23190..900068b2a 100644 --- a/docs/plugins/auth/auth_ldap.md +++ b/docs/plugins/auth/auth_ldap.md @@ -1,4 +1,4 @@ -auth/auth_ldap +auth/auth\_ldap ============== The `auth/auth_ldap` plugin uses an LDAP bind to authenticate a user. Currently diff --git a/docs/plugins/auth/auth_proxy.md b/docs/plugins/auth/auth_proxy.md index 2021cb509..6cf56921d 100644 --- a/docs/plugins/auth/auth_proxy.md +++ b/docs/plugins/auth/auth_proxy.md @@ -1,4 +1,4 @@ -auth/auth_proxy +auth/auth\_proxy =============== This plugin allows you to authenticate users by domain to remote SMTP servers diff --git a/docs/plugins/auth/auth_vpopmaild.js b/docs/plugins/auth/auth_vpopmaild.js index ccfeb373e..1fa73ae18 100644 --- a/docs/plugins/auth/auth_vpopmaild.js +++ b/docs/plugins/auth/auth_vpopmaild.js @@ -1,4 +1,4 @@ -auth/auth_vpopmaild +auth/auth\_vpopmaild ============== The `auth/vpopmaild` plugin allows you to authenticate against a vpopmaild diff --git a/docs/plugins/auth/flat_file.md b/docs/plugins/auth/flat_file.md index b1b140f21..ac9185ff9 100644 --- a/docs/plugins/auth/flat_file.md +++ b/docs/plugins/auth/flat_file.md @@ -1,4 +1,4 @@ -auth/flat_file +auth/flat\_file ============== The `auth/flat_file` plugin allows you to create a file containing username diff --git a/docs/plugins/avg.md b/docs/plugins/avg.md index 23840c6fa..cbc0af7e4 100644 --- a/docs/plugins/avg.md +++ b/docs/plugins/avg.md @@ -26,14 +26,14 @@ This plugin uses avg.ini for configuration. The available options are listed be AVG TCPD requires that the message be written to disk and scanned. This setting configures where any temporary files are written to. Once the scan is complete the temporary files are automatically removed. -- connect_timeout +- connect\_timeout Default: 10 Maximum number of seconds to wait for the socket to become connected before failing. Any connections taking longer than this will cause a temporary failure to be sent to the connected client. -- session_timeout +- session\_timeout Maximum number of seconds to wait for a reply to a command before failing. Any timeout will cause a temporary failure to be sent to the connected client. diff --git a/docs/plugins/block_me.md b/docs/plugins/block_me.md index c84439963..f1f23a685 100644 --- a/docs/plugins/block_me.md +++ b/docs/plugins/block_me.md @@ -1,4 +1,4 @@ -block_me +block\_me ======== This plugin allows you to configure an address which mail sent to will be diff --git a/docs/plugins/clamd.md b/docs/plugins/clamd.md index d309e6436..c500493fb 100644 --- a/docs/plugins/clamd.md +++ b/docs/plugins/clamd.md @@ -13,7 +13,7 @@ Configuration ### clamd.ini -* clamd_socket (default: localhost:3310) +* clamd\_socket (default: localhost:3310) ip.ip.ip.ip:port, [ipv6::literal]:port, host:port or /path/to/socket of the clamd daemon to send the message to for scanning. @@ -27,13 +27,13 @@ Configuration a temporary failure. -* randomize_host_order (default: false) +* randomize\_host\_order (default: false) If this is set then the list of hosts with be randomized before a connection is attempted. -* only_with_attachment (default: 0) +* only\_with\_attachment (default: 0) Set this option to only scan messages that contain non-textual attachments. This is a performance optimization, however it will @@ -41,7 +41,7 @@ Configuration or HTML messages. -* connect_timout (default: 10) +* connect\_timout (default: 10) Timeout connection to host after this many seconds. A timeout will cause the next host in the list to be tried. Once all hosts have @@ -55,7 +55,7 @@ Configuration with a tempoary failure. -* max_size (default: 26214400) +* max\_size (default: 26214400) The maximum size of message that should be sent to clamd in bytes. This option should not be larger than the StreamMaxLength value in diff --git a/docs/plugins/connect.rdns_access.md b/docs/plugins/connect.rdns_access.md index 1fcda5245..5397ffef0 100644 --- a/docs/plugins/connect.rdns_access.md +++ b/docs/plugins/connect.rdns_access.md @@ -1,26 +1,26 @@ -connect.rdns_access +connect.rdns\_access =================== This plugin will evaluate the remote IP address and the remote rDNS hostname against a set of white and black lists. The lists are applied in the following way: -connect.rdns_access.whitelist (pass) -connect.rdns_access.whitelist_regex (pass) -connect.rdns_access.blacklist (block) -connect.rdns_access.blacklist_regex (block) +connect.rdns\_access.whitelist (pass) +connect.rdns\_access.whitelist\_regex (pass) +connect.rdns\_access.blacklist (block) +connect.rdns\_access.blacklist\_regex (block) -Configuration connect.rdns_access.ini +Configuration connect.rdns\_access.ini ------------------------------------- General configuration file for this plugin. -* connect.rdns_access.general.deny_msg +* connect.rdns\_access.general.deny\_msg Text to send the user on reject (text). -Configuration connect.rdns_access.whitelist +Configuration connect.rdns\_access.whitelist ------------------------------------------- The whitelist is mostly to counter blacklist entries that match more than @@ -30,7 +30,7 @@ NOTE: We heavily suggest tailoring blacklist entries to be as accurate as possible and never using whitelists. Nevertheless, if you need whitelists, here they are. -Configuration connect.rdns_access.whitelist_regex +Configuration connect.rdns\_access.whitelist\_regex ------------------------------------------------- Does the same thing as the whitelist file, but each line is a regex. @@ -39,13 +39,13 @@ you. If you need to get around this restriction, you may use a '.*' at either the start or the end of your regex. This should help prevent people from writing overly permissive rules on accident. -Configuration connect.rdns_access.blacklist +Configuration connect.rdns\_access.blacklist ------------------------------------------- This file should be used for a specific IP address or rDNS hostnames, one per line, that should fail on connect. -Configuration connect.rdns_access.blacklist_regex +Configuration connect.rdns\_access.blacklist\_regex ------------------------------------------------- Does the same thing as the blacklist file, but each line is a regex. diff --git a/docs/plugins/daemonize.md b/docs/plugins/daemonize.md index e2f84843b..99e1a4d8d 100644 --- a/docs/plugins/daemonize.md +++ b/docs/plugins/daemonize.md @@ -20,12 +20,12 @@ Configuration This plugin looks for daemonize.ini in your configuration directory and the following options can be set: -- log_file (default: /var/log/haraka.log) +- log\_file (default: /var/log/haraka.log) The file that STDOUT should be redirected to. It is recommended that you use this plugin with the log.syslog plugin instead. -- pid_file (default: /var/run/haraka.pid) +- pid\_file (default: /var/run/haraka.pid) File where the master process PID should be written to. If this file cannot be locked then start-up will fail. @@ -51,7 +51,7 @@ If this is not the case on your system, then you should create the file The path to the Haraka smtp.ini configuration script -- max_open_files (default: 65535) +- max\_open\_files (default: 65535) The maximum number of open files allowed per process. If you are running Haraka using the 'cluster' module, then this is the per-child diff --git a/docs/plugins/data.rfc5322_header_checks.md b/docs/plugins/data.rfc5322_header_checks.md index 76d33261f..ab244049c 100644 --- a/docs/plugins/data.rfc5322_header_checks.md +++ b/docs/plugins/data.rfc5322_header_checks.md @@ -1,4 +1,4 @@ -data.rfc5322_header_checks +data.rfc5322\_header\_checks ========================== NOTICE: this plugin is deprecated. Use data.headers instead. diff --git a/docs/plugins/data.uribl.md b/docs/plugins/data.uribl.md index 9af3bc883..4f58d4971 100644 --- a/docs/plugins/data.uribl.md +++ b/docs/plugins/data.uribl.md @@ -28,7 +28,7 @@ The main section can contain the following options: lookups that takes longer than this will be aborted and the session will continue. -* max_uris_per_list +* max\_uris\_per\_list Default: 20 @@ -73,7 +73,7 @@ the blacklist. The following are optional for each list: -* custom_msg +* custom\_msg A custom rejection message that will be returned to the SMTP client if the list returns a positive result. If found within the string @@ -94,12 +94,12 @@ The following are optional for each list: combine multiple lists into a single zone. Using this you may specify which lists within the zone you want use. -* no_ip_lookups = 1 | true | yes | on | enabled +* no\_ip\_lookups = 1 | true | yes | on | enabled Specifies that no IP addresses should ever be check against this list. This is required for lists list dbl.spamhaus.org. -* strip_to_domain= 1 | true | yes | on | enabled +* strip\_to\_domain= 1 | true | yes | on | enabled Specifies that the list requires hostnames be stripped down to the domain boundaries prior to querying the list. This is required for diff --git a/docs/plugins/dkim_sign.md b/docs/plugins/dkim_sign.md index 2435f10c8..008b9da52 100644 --- a/docs/plugins/dkim_sign.md +++ b/docs/plugins/dkim_sign.md @@ -1,4 +1,4 @@ -dkim_sign +dkim\_sign ========= This plugin implements the DKIM Core specification found at dkimcore.org @@ -17,7 +17,7 @@ Generate DKIM selector and keys: cd /path/to/haraka/config/dkim ./dkim_key_gen.sh example.org -Peek into the dkim_key_gen.sh shell script to see the commands used to +Peek into the dkim\_key\_gen.sh shell script to see the commands used to create and format the DKIM public key. Within the config/dkim/example.org directory will be 4 files: @@ -59,7 +59,7 @@ For an alternative, see the legacy Single Domain Configuration below. Configuration ------------- -This plugin uses the configuration dkim_sign.ini in INI format. +This plugin uses the configuration dkim\_sign.ini in INI format. All configuration should appear within the 'main' block and is checked for updates on every run. @@ -67,7 +67,7 @@ checked for updates on every run. Set this to disable DKIM signing -- headers_to_sign = list, of; headers (REQUIRED) +- headers\_to\_sign = list, of; headers (REQUIRED) Set this to the list of headers that should be signed separated by either a comma, colon or semi-colon. @@ -85,7 +85,7 @@ are required. - selector = name Set this to the selector name published in DNS under the - _domainkey sub-domain of the domain referenced below. + \_domainkey sub-domain of the domain referenced below. - domain = name diff --git a/docs/plugins/dnsbl.md b/docs/plugins/dnsbl.md index 1a72e68c2..a0183f71f 100644 --- a/docs/plugins/dnsbl.md +++ b/docs/plugins/dnsbl.md @@ -18,7 +18,7 @@ dnsbl.ini - INI format with options described below: A comma or semi-colon list of zones to query. It will be merged with any lists in dnsbl.zones. -* periodic_checks +* periodic\_checks If enabled, this will check all the zones every n minutes. The minimum value that will be accepted here is 5. Any value less @@ -29,7 +29,7 @@ dnsbl.ini - INI format with options described below: disabled and will be re-checked on the next test. If a zone subsequently starts working correctly then it will be re-enabled. -* enable_stats +* enable\_stats To use this feature you must have installed the 'redis' module and have a redis server running. @@ -37,7 +37,7 @@ dnsbl.ini - INI format with options described below: When enabled, this will record several list statistics to redis. It will track the total number of queries (TOTAL) and the average - response time (AVG_RT) and the return type (e.g. LISTED or ERROR) + response time (AVG\_RT) and the return type (e.g. LISTED or ERROR) to a redis hash where the key is 'dns-list-stat:zone' and the hash field is the response type. @@ -64,7 +64,7 @@ dnsbl.ini - INI format with options described below: 6) "1" -* stats_redis_host +* stats\_redis\_host In the form of `host:port` this option allows you to specify a different host on which redis runs. diff --git a/docs/plugins/dnswl.md b/docs/plugins/dnswl.md index 045603c25..73b4c28ce 100644 --- a/docs/plugins/dnswl.md +++ b/docs/plugins/dnswl.md @@ -3,7 +3,7 @@ dnswl This plugin looks up the connecting IP address in an IP whitelist. If the host is listed, then the plugin will return OK for all hooks -up to hook_data. +up to hook\_data. IMPORTANT! The order of plugins in config/plugins is important when this plugin is used. It should be listed *before* any plugins that @@ -23,7 +23,7 @@ dnswl.ini - INI format with options described below: A comma or semi-colon list of zones to query. It will be merged with any lists in dnswl.zones. -* periodic_checks +* periodic\_checks If enabled, this will check all the zones every n minutes. The minimum value that will be accepted here is 5. Any value less @@ -34,7 +34,7 @@ dnswl.ini - INI format with options described below: disabled and will be re-checked on the next test. If a zone subsequently starts working correctly then it will be re-enabled. -* enable_stats +* enable\_stats To use this feature you must have installed the 'redis' module and have a redis server running. @@ -42,7 +42,7 @@ dnswl.ini - INI format with options described below: When enabled, this will record several list statistics to redis. It will track the total number of queries (TOTAL) and the average - response time (AVG_RT) and the return type (e.g. LISTED or ERROR) + response time (AVG\_RT) and the return type (e.g. LISTED or ERROR) to a redis hash where the key is 'dns-list-stat:zone' and the hash field is the response type. @@ -69,7 +69,7 @@ dnswl.ini - INI format with options described below: 6) "1" -* stats_redis_host +* stats\_redis\_host In the form of `host:port` this option allows you to specify a different host on which redis runs. diff --git a/docs/plugins/early_talker.md b/docs/plugins/early_talker.md index 610a6fc82..0c4d739df 100644 --- a/docs/plugins/early_talker.md +++ b/docs/plugins/early_talker.md @@ -1,4 +1,4 @@ -early_talker +early\_talker ============ Early talkers are violators of the SMTP specification, which demands that @@ -9,12 +9,12 @@ Early talker detection is handled internally by Haraka (in connection.js). At the DATA command, this plugin checks to see if an early talker was detected. -Any plugin can detect early talkers by checking connection.early_talker. +Any plugin can detect early talkers by checking connection.early\_talker. Configuration ------------- -* early_talker.pause +* early\_talker.pause Specifies a delay in milliseconds to delay before each SMTP command before sending the response, while waiting for early talkers. Default is no pause. diff --git a/docs/plugins/graph.md b/docs/plugins/graph.md index 63bcebe07..e5685af24 100644 --- a/docs/plugins/graph.md +++ b/docs/plugins/graph.md @@ -13,19 +13,19 @@ Configuration config settings are stored in config/graph.ini -* db_file +* db\_file The file name (or :memory:), where data is stored -* http_addr +* http\_addr The IP address to listen on for http. Default: `127.0.0.1`. -* http_port +* http\_port The port to listen on for http. Default: `8080`. -* ignore_re +* ignore\_re Regular expression to match plugins to ignore for logging. Default: `queue|graph|relay` diff --git a/docs/plugins/helo.checks.md b/docs/plugins/helo.checks.md index 25a35d056..c8f91392d 100644 --- a/docs/plugins/helo.checks.md +++ b/docs/plugins/helo.checks.md @@ -19,32 +19,32 @@ Configuration INI file which controls enabling of certain checks: - * check_no_dot=1 + * check\_no\_dot=1 Checks that the HELO has at least one '.' in it. - * check_raw_ip=1 + * check\_raw\_ip=1 Checks for HELO where the IP is not surrounded by square brackets. This is an RFC violation so should always be enabled. - * check_dynamic=1 + * check\_dynamic=1 Checks to see if all or part the connecting IP address appears within the HELO argument to indicate that the client has a dynamic IP address. - * check_literal_mismatch=1|2 + * check\_literal\_mismatch=1|2 Checks to see if the IP literal used matches the connecting IP address. If set to 1, the full IP must match. If set to 2, the /24 must match. - * require_valid_tld=1 + * require\_valid\_tld=1 Requires the HELO argument ends in a valid TLD if it is not an IP literal. - * skip_private_ip=1 + * skip\_private\_ip=1 - Bypasses check_no_dot, check_raw_ip, check_dynamic and require_valid_tld + Bypasses check\_no\_dot, check\_raw\_ip, check\_dynamic and require\_valid\_tld for clients within RFC1918, Loopback or APIPA IP address ranges. * [bigco] diff --git a/docs/plugins/log.syslog.md b/docs/plugins/log.syslog.md index 20975ec56..419f99cfc 100644 --- a/docs/plugins/log.syslog.md +++ b/docs/plugins/log.syslog.md @@ -63,7 +63,7 @@ chosen for you. logging the message. -* log.syslog.general.always_ok (default: false) +* log.syslog.general.always\_ok (default: false) If false, then this plugin will return with just next() allowing other plugins that have registered for the log hook to run. To speed things up, diff --git a/docs/plugins/lookup_rdns.strict.md b/docs/plugins/lookup_rdns.strict.md index 776f92186..64d05fcef 100644 --- a/docs/plugins/lookup_rdns.strict.md +++ b/docs/plugins/lookup_rdns.strict.md @@ -1,4 +1,4 @@ -lookup_rdns.strict +lookup\_rdns.strict =========== This plugin checks the reverse-DNS and compares the resulting addresses @@ -6,60 +6,60 @@ against forward DNS for a match. If there is no match it sends a DENYDISCONNECT, otherwise if it matches it sends an OK. DENYDISCONNECT messages are configurable. -Configuration lookup_rdns.strict.ini +Configuration lookup\_rdns.strict.ini -------------------------------------------- This is the general configuration file for the plugin. In it you can find ways to customize user messages, specify timeouts, and some whitelist parsing options. -* lookup_rdns.strict.general.nomatch +* lookup\_rdns.strict.general.nomatch Text to send the user if there is no reverse to forward match (text). -* lookup_rdns.strict.general.timeout +* lookup\_rdns.strict.general.timeout How long we should give this plugin before we time it out (seconds). -* lookup_rdns.strict.general.timeout_msg +* lookup\_rdns.strict.general.timeout\_msg Text to send when plugin reaches timeout (text). -* lookup_rdns.strict.forward.nxdomain +* lookup\_rdns.strict.forward.nxdomain Text to send the user if there is no forward match (text). -* lookup_rdns.strict.forward.dnserror +* lookup\_rdns.strict.forward.dnserror Text to send the user if there is some other error with the forward lookup (text). -* lookup_rdns.strict.reverse.nxdomain +* lookup\_rdns.strict.reverse.nxdomain Text to send the user if there is no reverse match (text). -* lookup_rdns.strict.reverse.dnserror +* lookup\_rdns.strict.reverse.dnserror Text to send the user if there is some other error with the reverse lookup (text). -Configuration lookup_rdns.strict.timeout +Configuration lookup\_rdns.strict.timeout ------------------------------------------------ This is how we specify to Haraka that our plugin should have a certain timeout. If you specify 0 here, then the plugin will never timeout while the connection is active. This is also required for this plugin, which needs to handle its own timeouts. To actually specify the timeout for this plugin, please see -the general config in lookup_rdns.strict.ini. +the general config in lookup\_rdns.strict.ini. -Configuration lookup_rdns.strict.whitelist +Configuration lookup\_rdns.strict.whitelist -------------------------------------------------- No matter how much you believe in checking that DNS and rDNS match, it is not @@ -70,7 +70,7 @@ providing a whitelist. This file will match exactly what you put on each line. -Configuration lookup_rdns.strict.whitelist_regex +Configuration lookup\_rdns.strict.whitelist\_regex -------------------------------------------------------- Does the same thing as the whitelist file, but each line is a regex. diff --git a/docs/plugins/mail_from.access.md b/docs/plugins/mail_from.access.md index eb8e1cc5b..47e5c042c 100644 --- a/docs/plugins/mail_from.access.md +++ b/docs/plugins/mail_from.access.md @@ -1,25 +1,25 @@ -mail_from.access +mail\_from.access =================== This plugin will evaluate the address against a set of white and black lists. The lists are applied in the following way: -mail_from.access.whitelist (pass) -mail_from.access.whitelist_regex (pass) -mail_from.access.blacklist (block) -mail_from.access.blacklist_regex (block) +mail\_from.access.whitelist (pass) +mail\_from.access.whitelist\_regex (pass) +mail\_from.access.blacklist (block) +mail\_from.access.blacklist\_regex (block) -Configuration mail_from.access.ini +Configuration mail\_from.access.ini ------------------------------------- General configuration file for this plugin. -* mail_from.access.general.deny_msg +* mail\_from.access.general.deny\_msg Text to send the user on reject (text). -Configuration mail_from.access.whitelist +Configuration mail\_from.access.whitelist ------------------------------------------- The whitelist is mostly to counter blacklist entries that match more than @@ -29,7 +29,7 @@ NOTE: We heavily suggest tailoring blacklist entries to be as accurate as possible and never using whitelists. Nevertheless, if you need whitelists, here they are. -Configuration mail_from.access.whitelist_regex +Configuration mail\_from.access.whitelist\_regex ------------------------------------------------- Does the same thing as the whitelist file, but each line is a regex. @@ -38,13 +38,13 @@ you. If you need to get around this restriction, you may use a '.*' at either the start or the end of your regex. This should help prevent people from writing overly permissive rules on accident. -Configuration mail_from.access.blacklist +Configuration mail\_from.access.blacklist ------------------------------------------- This file should be used for a specific address, one per line, that should fail on connect. -Configuration mail_from.access.blacklist_regex +Configuration mail\_from.access.blacklist\_regex ------------------------------------------------- Does the same thing as the blacklist file, but each line is a regex. diff --git a/docs/plugins/mail_from.blocklist.md b/docs/plugins/mail_from.blocklist.md index 560040b4f..cf196fa79 100644 --- a/docs/plugins/mail_from.blocklist.md +++ b/docs/plugins/mail_from.blocklist.md @@ -1,15 +1,15 @@ -mail_from.blocklist +mail\_from.blocklist =================== -This plugin blocks MAIL_FROM addresses in a list. +This plugin blocks MAIL\_FROM addresses in a list. NOTE: If all you need is to deny mail based on the exact address, this plugin will work just fine. If you want to customize the deny message, add blocks -based on a regex, or add whitelists, please use the mail_from.access plugin. +based on a regex, or add whitelists, please use the mail\_from.access plugin. Configuration ------------- -* mail_from.blocklist +* mail\_from.blocklist Contains a list of email addresses to block. diff --git a/docs/plugins/mail_from.is_resolvable.md b/docs/plugins/mail_from.is_resolvable.md index 31562f663..c07fc0d14 100644 --- a/docs/plugins/mail_from.is_resolvable.md +++ b/docs/plugins/mail_from.is_resolvable.md @@ -1,4 +1,4 @@ -mail_from.is_resolvable +mail\_from.is\_resolvable ======================= This plugin checks that the domain used in MAIL FROM is resolvable to an MX @@ -16,12 +16,12 @@ This plugin uses the INI-style file format and accepts the following options: Maximum limit in seconds for queries to complete. If the timeout is reached a TEMPFAIL is returned to the client. -* allow_mx_ip=[0|1] +* allow\_mx\_ip=[0|1] Allow MX records that return IP addresses instead of hostnames. This is not allowed as per the RFC, but some MTAs allow it. -* reject_no_mx=[0|1] +* reject\_no\_mx=[0|1] Return DENY and reject the command if no MX record is found. Otherwise a DENYSOFT (TEMPFAIL) is returned and the client will retry later. diff --git a/docs/plugins/mail_from.nobounces.md b/docs/plugins/mail_from.nobounces.md index 2e1845c35..f7b632c93 100644 --- a/docs/plugins/mail_from.nobounces.md +++ b/docs/plugins/mail_from.nobounces.md @@ -1,4 +1,4 @@ -mail_from.nobounces +mail\_from.nobounces =================== This mail blocks all bounce messages using the simple rule of checking diff --git a/docs/plugins/max_unrecognized_commands.md b/docs/plugins/max_unrecognized_commands.md index 30ac7d767..655e0ddbe 100644 --- a/docs/plugins/max_unrecognized_commands.md +++ b/docs/plugins/max_unrecognized_commands.md @@ -1,4 +1,4 @@ -max_unrecognized_commands +max\_unrecognized\_commands ========================= This plugin places a maximum limit on the number of unrecognized commands @@ -10,7 +10,7 @@ immediately (and rudely - technically an RFC violation) disconnected. Configuration ------------- -* max_unrecognized_commands +* max\_unrecognized\_commands Specifies the number of unrecognised commands to allow before disconnecting. - Default: 10. \ No newline at end of file + Default: 10. diff --git a/docs/plugins/messagesniffer.md b/docs/plugins/messagesniffer.md index c93a50be5..871ab8651 100644 --- a/docs/plugins/messagesniffer.md +++ b/docs/plugins/messagesniffer.md @@ -48,7 +48,7 @@ connection and the `[message]` section is used to specify the action to be taken Temporary directory used to write temporary message files to that are read by the SNFServer daemon. This directory and the files within need to be readable by the user that SNFServer is running as. -- gbudb_report_deny = [ true | false | 0 | 1 ] +- gbudb\_report\_deny = [ true | false | 0 | 1 ] Default: false This is an experimental option that will record a GBUdb 'bad' encounter for a connected IP address when a client @@ -56,7 +56,7 @@ connection and the `[message]` section is used to specify the action to be taken some point during the session. The idea behind this option is that it allows other Haraka plugins rejections influence GBUdb IP reputation where MessageSniffer isn't seeing the actual message because it is being rejected pre-DATA. -- tag_string +- tag\_string Default: [SPAM] String to prepend to the Subject line if the 'tag' action is applied. @@ -90,7 +90,7 @@ connection and the `[message]` section is used to specify the action to be taken Default: continue Action to take when MessageSniffer reports a 'white' result (result code: 0). -- local_white = [ accept | allow | continue | retry | tempfail | reject | quarantine | tag ] +- local\_white = [ accept | allow | continue | retry | tempfail | reject | quarantine | tag ] Default: accept Action to take when MessageSniffer reports a local whitelist result (result code: 1). @@ -117,7 +117,7 @@ connection and the `[message]` section is used to specify the action to be taken NOTE: GBUdb IP lookups during the data phase can be different than the connecting IP address if you have configured Source and DrillDown options in the Training section of SNFServer.xml. -- code_NN = [ accept | allow | continue | retry | tempfail | reject | quarantine | tag ] +- code\_NN = [ accept | allow | continue | retry | tempfail | reject | quarantine | tag ] NOTE: replace NN with the numeric MessageSniffer [result code](http://armresearch.com/support/articles/software/snfServer/core.jsp) Action to take when MessageSniffer reports a result code other than those explicitly defined above. @@ -126,7 +126,7 @@ connection and the `[message]` section is used to specify the action to be taken Defalt: reject Action to take for any non-zero result code other than those explicity defined above. This is a catch-all result that - is checked last after all other settings have been checked so you can define a code_NN value to prevent this action from + is checked last after all other settings have been checked so you can define a code\_NN value to prevent this action from being taken. Actions @@ -158,6 +158,6 @@ Actions * tag - Tag the subject with the default 'tag_string' defined in the `main` section above, this will also set X-Spam-Flag: YES in + Tag the subject with the default 'tag\_string' defined in the `main` section above, this will also set X-Spam-Flag: YES in the message headers. Once tagged, processing will continue to the next plugin. diff --git a/docs/plugins/process_title.md b/docs/plugins/process_title.md index f5e4802d1..ca9ac4407 100644 --- a/docs/plugins/process_title.md +++ b/docs/plugins/process_title.md @@ -1,4 +1,4 @@ -process_title +process\_title ============= This plugin causes the process title seen by the UNIX 'ps' command to diff --git a/docs/plugins/queue/quarantine.md b/docs/plugins/queue/quarantine.md index 82efeb186..d26dd1d44 100644 --- a/docs/plugins/queue/quarantine.md +++ b/docs/plugins/queue/quarantine.md @@ -33,7 +33,7 @@ Configuration This plugin looks for 'quarantine.ini' in the config directory. -* quarantine_path (default: /var/spool/haraka/quarantine) +* quarantine\_path (default: /var/spool/haraka/quarantine) The default base path to save the quarantine files to. It will be created if it does not already exist. diff --git a/docs/plugins/queue/smtp_forward.md b/docs/plugins/queue/smtp_forward.md index fb71a6e1a..61b2bc33d 100644 --- a/docs/plugins/queue/smtp_forward.md +++ b/docs/plugins/queue/smtp_forward.md @@ -1,4 +1,4 @@ -queue/smtp_forward +queue/smtp\_forward ================== This plugin delivers to another mail server. This is a common setup when you @@ -14,7 +14,7 @@ filtering that the ongoing mail server may provide. Configuration ------------- -* smtp_forward.ini +* smtp\_forward.ini Configuration is stored in this file in the following keys: @@ -26,7 +26,7 @@ Configuration The port to connect to. - * connect_timeout=SECONDS + * connect\_timeout=SECONDS The maximum amount of time to wait when creating a new connection to the host. Default if unspecified is 30 seconds. @@ -37,11 +37,11 @@ Configuration connection pool. This should always be less than the global plugin timeout, which should in turn be less than the connection timeout. - * max_connections=NUMBER + * max\_connections=NUMBER Maximum number of connections to create at any given time. - * enable_tls=[true|yes|1] + * enable\_tls=[true|yes|1] Enable TLS with the forward host (if supported) diff --git a/docs/plugins/queue/smtp_proxy.md b/docs/plugins/queue/smtp_proxy.md index 486f0e091..2b442954a 100644 --- a/docs/plugins/queue/smtp_proxy.md +++ b/docs/plugins/queue/smtp_proxy.md @@ -1,4 +1,4 @@ -queue/smtp_proxy +queue/smtp\_proxy ================ This plugin delivers to another mail server. This is a common setup when you @@ -15,7 +15,7 @@ have as many connections to your ongoing SMTP server as you have to Haraka. Configuration ------------- -* smtp_proxy.ini +* smtp\_proxy.ini Configuration is stored in this file in the following keys: @@ -27,7 +27,7 @@ Configuration The port to connect to. - * connect_timeout=SECONDS + * connect\_timeout=SECONDS The maximum amount of time to wait when creating a new connection to the host. Default if unspecified is 30 seconds. @@ -38,11 +38,11 @@ Configuration proxy pool. This should always be less than the global plugin timeout, which should in turn be less than the connection timeout. - * max_connections=NUMBER + * max\_connections=NUMBER Maximum number of connections to create at any given time. - * enable_tls=[true|yes|1] + * enable\_tls=[true|yes|1] Enable TLS with the forward host (if supported) diff --git a/docs/plugins/rate_limit.md b/docs/plugins/rate_limit.md index d35d90c2a..f7bb04fcc 100644 --- a/docs/plugins/rate_limit.md +++ b/docs/plugins/rate_limit.md @@ -1,4 +1,4 @@ -rate_limit +rate\_limit ========== This pluign enforces limits on connection concurrency, connection rate and @@ -19,7 +19,7 @@ hiredis and ipaddr.js packages installed via: Configuration ------------- -This plugin uses the configuration file rate_limit.ini which is checked for +This plugin uses the configuration file rate\_limit.ini which is checked for updates before each hook, so changes to this file will never require a restart and will take effect immediately after the changes are saved. @@ -27,14 +27,14 @@ The configuration options for each heading are detailed below: ### [main] -- redis_server = \[:port] *(optional)* +- redis\_server = \[:port] *(optional)* If port is missing then it defaults to 6379. If this setting is missing entirely then it defaults to 127.0.0.1:6379. Note that Redis does not currently support IPv6. -- tarpit_delay = seconds *(optional)* +- tarpit\_delay = seconds *(optional)* Set this to the length in seconds that you want to delay every SMTP response to a remote client that has exceeded the rate limits. For this @@ -117,35 +117,35 @@ per-child if you use the cluster module. IP and rDNS names are looked up by this test. This section does *not* accept an interval. It's a hard limit on the number of connections and not based on time. -### [rate_conn] +### [rate\_conn] This section limits the number of connections per interval from a given host or set of hosts. IP and rDNS names are looked up by this test. -### [rate_rcpt_host] +### [rate\_rcpt\_host] This section limits the number of recipients per interval from a given host or set of hosts. IP and rDNS names are looked up by this test. -### [rate_rcpt_sender] +### [rate\_rcpt\_sender] This section limits the number of recipients per interval from a sender or sender domain. The sender is looked up by this test. -### [rate_rcpt] +### [rate\_rcpt] This section limits the rate which a recipient or recipient domain can receive messages over an interval. Each recipient is looked up by this test. -### [rate_rcpt_null] +### [rate\_rcpt\_null] This section limits the rate at which a recipient can receive messages from a null sender (e.g. DSN, MDN etc.) over an interval. diff --git a/docs/plugins/rcpt_to.access.md b/docs/plugins/rcpt_to.access.md index 31b5fd3c3..11a886709 100644 --- a/docs/plugins/rcpt_to.access.md +++ b/docs/plugins/rcpt_to.access.md @@ -1,25 +1,25 @@ -rcpt_to.access +rcpt\_to.access =================== -This plugin blocks RCPT_TO addresses in a list or regex. -This plugin will evaluate the RCPT_TO address against a set of white and black +This plugin blocks RCPT\_TO addresses in a list or regex. +This plugin will evaluate the RCPT\_TO address against a set of white and black lists. The lists are applied in the following way: -rcpt_to.access.whitelist (pass) -rcpt_to.access.whitelist_regex (pass) -rcpt_to.access.blacklist (block) -rcpt_to.access.blacklist_regex (block) +rcpt\_to.access.whitelist (pass) +rcpt\_to.access.whitelist\_regex (pass) +rcpt\_to.access.blacklist (block) +rcpt\_to.access.blacklist\_regex (block) -Configuration rcpt_to.access.ini +Configuration rcpt\_to.access.ini ------------------------------------- General configuration file for this plugin. -* rcpt_to.access.general.deny_msg +* rcpt\_to.access.general.deny\_msg Text to send the user on reject (text). -Configuration rcpt_to.access.whitelist +Configuration rcpt\_to.access.whitelist ------------------------------------------- The whitelist is mostly to counter blacklist entries that match more than @@ -29,7 +29,7 @@ NOTE: We heavily suggest tailoring blacklist entries to be as accurate as possible and never using whitelists. Nevertheless, if you need whitelists, here they are. -Configuration rcpt_to.access.whitelist_regex +Configuration rcpt\_to.access.whitelist\_regex ------------------------------------------------- Does the same thing as the whitelist file, but each line is a regex. @@ -38,13 +38,13 @@ you. If you need to get around this restriction, you may use a '.*' at either the start or the end of your regex. This should help prevent people from writing overly permissive rules on accident. -Configuration rcpt_to.access.blacklist +Configuration rcpt\_to.access.blacklist ------------------------------------------- This file should be used for a specific address, one per line, that should fail on connect. -Configuration rcpt_to.access.blacklist_regex +Configuration rcpt\_to.access.blacklist\_regex ------------------------------------------------- Does the same thing as the blacklist file, but each line is a regex. diff --git a/docs/plugins/rcpt_to.blocklist.md b/docs/plugins/rcpt_to.blocklist.md index 8b0d1dd2a..4d4ee07ea 100644 --- a/docs/plugins/rcpt_to.blocklist.md +++ b/docs/plugins/rcpt_to.blocklist.md @@ -1,15 +1,15 @@ -rcpt_to.blocklist +rcpt\_to.blocklist =================== -This plugin blocks RCPT_TO addresses in a list. +This plugin blocks RCPT\_TO addresses in a list. NOTE: If all you need is to deny mail based on the exact address, this plugin will work just fine. If you want to customize the deny message, add blocks -based on a regex, or add whitelists, please use the rcpt_to.access plugin. +based on a regex, or add whitelists, please use the rcpt\_to.access plugin. Configuration ------------- -* rcpt_to.blocklist +* rcpt\_to.blocklist Contains a list of email addresses to block. diff --git a/docs/plugins/rcpt_to.in_host_list.md b/docs/plugins/rcpt_to.in_host_list.md index 02173657d..4c98f798b 100644 --- a/docs/plugins/rcpt_to.in_host_list.md +++ b/docs/plugins/rcpt_to.in_host_list.md @@ -1,4 +1,4 @@ -rcpt_to.in_host_list +rcpt\_to.in\_host\_list ==================== This plugin is the mainstay of an inbound Haraka server. It should list the @@ -9,11 +9,11 @@ rejected. Configuration ------------- -* host_list +* host\_list Specifies the list of hosts that are local to this server. -* host_list_regex +* host\_list\_regex Specifies the list of regexes that are local to this server. Note all these regexes are anchored with ^regex$. One can not choose not to diff --git a/docs/plugins/rcpt_to.max_count.md b/docs/plugins/rcpt_to.max_count.md index 6dcb447e6..a53b2c13f 100644 --- a/docs/plugins/rcpt_to.max_count.md +++ b/docs/plugins/rcpt_to.max_count.md @@ -1,4 +1,4 @@ -rcpt_to.max_count +rcpt\_to.max\_count ================= This plugin sets a maximum limit on RCPT TOs. Violators will be disconnected. @@ -6,6 +6,6 @@ This plugin sets a maximum limit on RCPT TOs. Violators will be disconnected. Configuration ------------- -* rcpt_to.max_count +* rcpt\_to.max\_count The maximum number of recipients. Default: 40. diff --git a/docs/plugins/rcpt_to.qmail_deliverable.md b/docs/plugins/rcpt_to.qmail_deliverable.md index c523d42c1..ca2bad95b 100644 --- a/docs/plugins/rcpt_to.qmail_deliverable.md +++ b/docs/plugins/rcpt_to.qmail_deliverable.md @@ -1,5 +1,4 @@ - -qmail_deliverable +qmail\_deliverable ============ This plugin implements a client for checking the deliverability of an email @@ -11,5 +10,5 @@ Configuration ------------- You can modify the host/port that qmail-deliverabled is listening on by -altering the contents of config/rcpt_to.qmail_deliverable.ini +altering the contents of config/rcpt\_to.qmail\_deliverable.ini diff --git a/docs/plugins/rdns.regexp.md b/docs/plugins/rdns.regexp.md index daf2722fb..bf5cffb02 100644 --- a/docs/plugins/rdns.regexp.md +++ b/docs/plugins/rdns.regexp.md @@ -2,8 +2,8 @@ rdns.regexp =========== WARNING: The services offered by this plugin, and much more, are now provided -more efficiently with the connect.rdns_access plugin. Please transition over -to using the new connect.rdns_access plugin, as this plugin is now deprecated +more efficiently with the connect.rdns\_access plugin. Please transition over +to using the new connect.rdns\_access plugin, as this plugin is now deprecated and may be removed in a future version of Haraka. This plugin checks the reverse-DNS against a list of regular expressions. Any @@ -21,12 +21,12 @@ superseded by the allow rule for generaldynamics.com. Configuration ------------- -* rdns.deny_regexps +* rdns.deny\_regexps The list of regular expressions to deny. Over broad regexes in this list can be corrected by using the allow list. -* rdns.allow_regexps +* rdns.allow\_regexps The list of regular expressions to allow. This list is always processed in favor of rules in the deny file. diff --git a/docs/plugins/relay_acl.md b/docs/plugins/relay_acl.md index d47de8ef8..e7a3cf670 100644 --- a/docs/plugins/relay_acl.md +++ b/docs/plugins/relay_acl.md @@ -1,4 +1,4 @@ -relay_acl +relay\_acl ======== This plugin makes it possible to relay outbound mails using IP based ACLs @@ -24,6 +24,6 @@ Configuration [domains] test.com = { "action": "continue" } - Please note that this config/relay_dest_domains.ini is shared with - plugins/relay_force_routing.js, which uses additional fields. + Please note that this config/relay\_dest\_domains.ini is shared with + plugins/relay\_force\_routing.js, which uses additional fields. diff --git a/docs/plugins/relay_all.md b/docs/plugins/relay_all.md index 8731fa0c8..d99ec0279 100644 --- a/docs/plugins/relay_all.md +++ b/docs/plugins/relay_all.md @@ -1,4 +1,4 @@ -relay_all +relay\_all ========= This plugin is useful in spamtraps to accept mail to any host, and to allow diff --git a/docs/plugins/relay_force_routing.md b/docs/plugins/relay_force_routing.md index 29986c655..41d1069e6 100644 --- a/docs/plugins/relay_force_routing.md +++ b/docs/plugins/relay_force_routing.md @@ -1,4 +1,4 @@ -relay_force_routing.js +relay\_force\_routing.js ======== This plugin allows you to force the next hop for the configured domains. @@ -8,10 +8,10 @@ Configuration ------------- * `config/relay_dest_domains.ini` - This config file is shared with relay_acl.js, for the basics see the - documentation provided by plugins/relay_acl.js. + This config file is shared with relay\_acl.js, for the basics see the + documentation provided by plugins/relay\_acl.js. - relay_force_routing.js adds the field "nexthop": in the JSON value + relay\_force\_routing.js adds the field "nexthop": in the JSON value of the domain. The value of "nexthop": can be hostname or IP optionally follow by :port. diff --git a/docs/plugins/reseed_rng.md b/docs/plugins/reseed_rng.md index 7e7e8ac81..6df046be9 100644 --- a/docs/plugins/reseed_rng.md +++ b/docs/plugins/reseed_rng.md @@ -1,4 +1,4 @@ -reseed_rng +reseed\_rng ========== The V8 that ships with node 0.4.x uses an unsophisticated method of @@ -12,7 +12,7 @@ problems like UUID collisions. When using the 'cluster' module, it's quite easy to observe this behavior. This plugin uses David Bao's reseed.js (see http://davidbau.com/archives/2010/01/30/random_seeds_coded_hints_and_quintillions.html) -to provide a reseedable Math.random(), and hooks the init_child event +to provide a reseedable Math.random(), and hooks the init\_child event to reseed the RNG with a sligtly better seed at spawned-process startup time. diff --git a/docs/plugins/spf.md b/docs/plugins/spf.md index 5b715d490..fbf34ff76 100644 --- a/docs/plugins/spf.md +++ b/docs/plugins/spf.md @@ -16,54 +16,54 @@ Configuration This plugin uses spf.ini for configuration and each option is documented below: -- helo_softfail_reject +- helo\_softfail\_reject Default: false Return DENY if the SPF HELO check returns SoftFail. This option should only be enabled in exceptional circumstances. -- helo_fail_reject +- helo\_fail\_reject Default: false Return DENYSOFT if the SPF HELO check returns Fail. -- helo_temperror_defer +- helo\_temperror\_defer Default: false Return DENYSOFT if the SPF HELO check returns TempError. This can cause excessive delays if a domain has a broken SPF record or any issues with their DNS configuration. -- helo_permerror_reject +- helo\_permerror\_reject Default: false Return DENY if the SPF HELO check returns Fail. This can cause false-positives if a domain has any syntax errors in their SPF record. -- mail_softfail_reject +- mail\_softfail\_reject Default: false Return DENYSOFT if the SPF MAIL check returns SoftFail. This option should only be used in exceptional circumstances. -- mail_fail_reject +- mail\_fail\_reject Default: false Return DENY if the SPF MAIL check returns Fail. -- mail_temperror_defer +- mail\_temperror\_defer Default: false Return DENYSOFT if the SPF MAIL check returns TempError. This can cause excessive delays if a domain has a broken SPF record or any issues with their DNS configuration. -- mail_permerror_reject +- mail\_permerror\_reject Default: false diff --git a/docs/plugins/toobusy.md b/docs/plugins/toobusy.md index 0f6295eac..73cba163e 100644 --- a/docs/plugins/toobusy.md +++ b/docs/plugins/toobusy.md @@ -10,7 +10,7 @@ To use this plugin you have to install the 'toobusy' module by running 'npm install toobusy' in your Haraka configuration directory. This plugin should be listed at the top of your config/plugins file so that -it runs before any other plugin that hooks lookup_rdns. +it runs before any other plugin that hooks lookup\_rdns. Configuration ------------- diff --git a/docs/tutorials/Migrating_from_v1_to_v2.md b/docs/tutorials/Migrating_from_v1_to_v2.md index 420d8fee7..67b58320f 100644 --- a/docs/tutorials/Migrating_from_v1_to_v2.md +++ b/docs/tutorials/Migrating_from_v1_to_v2.md @@ -28,7 +28,7 @@ Secondly, if you parse the mail body, attachments are now provided as a stream, rather than custom start/data/end events. To find if this is relevant for you, look for instances of `attachment_hooks` in your plugins. -Fixing data_lines plugins +Fixing data\_lines plugins ------------------------- Any plugins now working on each line of data will need to change to using a @@ -81,7 +81,7 @@ your own writable stream and then pipe the message to the stream and then extract the lines from the stream of data. See `plugins/dkim_sign.js` for an example. -Fixing attachment_hooks plugins +Fixing attachment\_hooks plugins ------------------------------- For v1.x you passed in functions to `transaction.attachment_hooks()` as diff --git a/docs/tutorials/SettingUpOutbound.md b/docs/tutorials/SettingUpOutbound.md index f64ed4136..4bab13809 100644 --- a/docs/tutorials/SettingUpOutbound.md +++ b/docs/tutorials/SettingUpOutbound.md @@ -51,7 +51,7 @@ and password: vi config/auth_flat_file.ini -See the documentation in docs/plugins/auth/flat_file.md for information about +See the documentation in docs/plugins/auth/flat\_file.md for information about what can go in that file. Now you can start Haraka. That's all the configuration you need. @@ -69,4 +69,4 @@ Watch the output of swaks and ensure no errors have occurred. Then watch the recipient email address (easiest to make this your webmail account) and see that the email arrived. -You are done! \ No newline at end of file +You are done! From e0b477a9e66d6e04a4331db2545d8ee053dc813c Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Tue, 21 Jan 2014 15:07:22 -0500 Subject: [PATCH 111/160] Fix underscores --- docs/plugins/bounces.md | 4 ++-- docs/plugins/connect.geoip.md | 6 +++--- docs/plugins/karma.md | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/plugins/bounces.md b/docs/plugins/bounces.md index cb5946242..aac61792f 100644 --- a/docs/plugins/bounces.md +++ b/docs/plugins/bounces.md @@ -6,7 +6,7 @@ This plugin provides options for bounce processing. Configuration ------------------- -- reject_all +- reject\_all Blocks all bounce messages using the simple rule of checking for `MAIL FROM:<>`. @@ -16,7 +16,7 @@ much but very few legitimate users. It is potentially bad to block all bounce messages, but unfortunately for some hosts, sometimes necessary. -- reject_invalid +- reject\_invalid -------------------- This option tries to assure the message really is a bounce. It makes sure the message has a single recipient and that the return path is diff --git a/docs/plugins/connect.geoip.md b/docs/plugins/connect.geoip.md index 99a389081..071b5aa8c 100644 --- a/docs/plugins/connect.geoip.md +++ b/docs/plugins/connect.geoip.md @@ -37,16 +37,16 @@ be the IP that Haraka is bound to, but if not you'll need to supply it. Perform the geodesic distance calculations. Calculates the distance "as the crow flies" from the remote mail server. -- public_ip: +- public\_ip: The IP address to calculate the distance from. This will typically be the public IP of your mail server. -- show_city +- show\_city show city data in logs and headers. City data is less accurate than country. -- show_region in logs and headers. Regional data are US states, Canadian +- show\_region in logs and headers. Regional data are US states, Canadian provinces and such. diff --git a/docs/plugins/karma.md b/docs/plugins/karma.md index 07920ab26..2c90de461 100644 --- a/docs/plugins/karma.md +++ b/docs/plugins/karma.md @@ -18,7 +18,7 @@ Karma can be used to craft custom connection policies such as these examples: DESCRIPTION ----------------------- Karma records the number of good, bad, and total connections. When a sender -has more bad than good connections, they are penalized for *penalty_days*. +has more bad than good connections, they are penalized for *penalty\_days*. Connections from senders in the penalty box are rejected until the penalty expires. @@ -55,13 +55,13 @@ KARMA ------------------------ When the connection ends, B records the result. Mail servers whose bad connections exceed good ones are sent to the penalty box. Servers in -the penalty box are tersely disconnected for *penalty_days*. Here is +the penalty box are tersely disconnected for *penalty\_days*. Here is an example connection from an IP in the penalty box: If only negative karma is set, desirable mailers will be penalized. For example, a Yahoo user sends an egregious spam to a user on our server. Now nobody on our server can receive email from that Yahoo server for -*penalty_days*. This will happen approximately 0% of the time if we also +*penalty\_days*. This will happen approximately 0% of the time if we also set positive karma. From d6272eb3f3366bee0e209fde791a35f51b8ad978 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Tue, 21 Jan 2014 20:09:16 +0000 Subject: [PATCH 112/160] Improve .ini parsr and add is_enabled helper function --- config.js | 5 +++++ configfile.js | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/config.js b/config.js index a9171c1b2..ac5cf1a96 100644 --- a/config.js +++ b/config.js @@ -6,6 +6,7 @@ var logger = require('./logger'); var config = exports; var config_path = process.env.HARAKA ? path.join(process.env.HARAKA, 'config') : path.join(__dirname, './config'); +var enabled = new RegExp('^(?:yes|true|enabled|ok|on|1)$', 'i'); config.get = function(name, type, cb) { if (type === 'nolog') { @@ -24,3 +25,7 @@ config.get = function(name, type, cb) { return results; } }; + +config.is_enabled = function(value) { + return enabled.match(value); +} diff --git a/configfile.js b/configfile.js index dd1646f7a..8f8ac2c31 100644 --- a/configfile.js +++ b/configfile.js @@ -9,6 +9,8 @@ var regex = { line: /^\s*(.*)\s*$/, blank: /^\s*$/, continuation: /\\[ \t]*$/, + is_integer: /^-?\d+$/, + is_float: /^-?\d+\.\d+$/, }; var cfreader = exports; @@ -132,9 +134,12 @@ cfreader.load_ini_config = function(name) { line = pre + line; pre = ''; if (match = regex.param.exec(line)) { - if (/^\d+$/.test(match[2])) { + if (regex.is_integer.test(match[2])) { current_sect[match[1]] = parseInt(match[2]); } + else if (regex.is_float.test(match[2])) { + current_sect[match[1]] = parseFloat(match[2]); + } else { current_sect[match[1]] = match[2]; } From b6e88000a8b4a60fcb8aa808151b270386059ddc Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Tue, 21 Jan 2014 20:10:38 +0000 Subject: [PATCH 113/160] Fix typo --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index ac5cf1a96..4ba8e7c76 100644 --- a/config.js +++ b/config.js @@ -27,5 +27,5 @@ config.get = function(name, type, cb) { }; config.is_enabled = function(value) { - return enabled.match(value); + return enabled.test(value); } From 912c6f57ec9afc9ab092bb72908c505deade9dbd Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Tue, 21 Jan 2014 20:23:36 +0000 Subject: [PATCH 114/160] Add parseInt(x, 10), re-use regexps and handle floats in flat_files --- configfile.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/configfile.js b/configfile.js index 8f8ac2c31..8bbcb47cb 100644 --- a/configfile.js +++ b/configfile.js @@ -74,8 +74,11 @@ cfreader.load_config = function(name, type) { result = cfreader.load_flat_config(name, type); if (result && type !== 'list' && type !== 'data') { result = result[0]; - if (/^\d+$/.test(result)) { - result = parseInt(result); + if (regex.is_integer.test(result)) { + result = parseInt(result, 10); + } + else if (regex.is_float.test(result)) { + result = parseFloat(result); } } } @@ -135,7 +138,7 @@ cfreader.load_ini_config = function(name) { pre = ''; if (match = regex.param.exec(line)) { if (regex.is_integer.test(match[2])) { - current_sect[match[1]] = parseInt(match[2]); + current_sect[match[1]] = parseInt(match[2], 10); } else if (regex.is_float.test(match[2])) { current_sect[match[1]] = parseFloat(match[2]); From d35df42315408957727ba2656a0b5e717d958cc0 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Tue, 21 Jan 2014 22:09:22 +0000 Subject: [PATCH 115/160] Updates to last commit --- config.js | 9 ++------- configfile.js | 44 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/config.js b/config.js index 4ba8e7c76..fc7ed1ad7 100644 --- a/config.js +++ b/config.js @@ -6,16 +6,15 @@ var logger = require('./logger'); var config = exports; var config_path = process.env.HARAKA ? path.join(process.env.HARAKA, 'config') : path.join(__dirname, './config'); -var enabled = new RegExp('^(?:yes|true|enabled|ok|on|1)$', 'i'); -config.get = function(name, type, cb) { +config.get = function(name, type, cb, options) { if (type === 'nolog') { type = arguments[2]; // deprecated - TODO: remove later } type = type || 'value'; var full_path = path.resolve(config_path, name); - var results = configloader.read_config(full_path, type, cb); + var results = configloader.read_config(full_path, type, cb, options); // Pass arrays by value to prevent config being modified accidentally. if (Array.isArray(results)) { @@ -25,7 +24,3 @@ config.get = function(name, type, cb) { return results; } }; - -config.is_enabled = function(value) { - return enabled.test(value); -} diff --git a/configfile.js b/configfile.js index 8bbcb47cb..70ca4c735 100644 --- a/configfile.js +++ b/configfile.js @@ -11,6 +11,7 @@ var regex = { continuation: /\\[ \t]*$/, is_integer: /^-?\d+$/, is_float: /^-?\d+\.\d+$/, + is_truth: /^(?:true|yes|ok|enabled|on|1)$/i, }; var cfreader = exports; @@ -19,20 +20,20 @@ cfreader.watch_files = true; cfreader._config_cache = {}; cfreader._watchers = {}; -cfreader.read_config = function(name, type, cb) { +cfreader.read_config = function(name, type, cb, options) { // Check cache first if (name in cfreader._config_cache) { return cfreader._config_cache[name]; } // load config file - var result = cfreader.load_config(name, type); + var result = cfreader.load_config(name, type, options); if (cfreader.watch_files) { if (name in cfreader._watchers) return result; try { cfreader._watchers[name] = fs.watch(name, {persistent: false}, function (event, filename) { - cfreader.load_config(name, type); + cfreader.load_config(name, type, options); if (typeof cb === 'function') cb(); }); } @@ -58,11 +59,11 @@ cfreader.empty_config = function(type) { } }; -cfreader.load_config = function(name, type) { +cfreader.load_config = function(name, type, options) { var result; if (type === 'ini' || /\.ini$/.test(name)) { - result = cfreader.load_ini_config(name); + result = cfreader.load_ini_config(name, options); } else if (type === 'json' || /\.json$/.test(name)) { result = cfreader.load_json_config(name); @@ -71,10 +72,13 @@ cfreader.load_config = function(name, type) { result = cfreader.load_binary_config(name, type); } else { - result = cfreader.load_flat_config(name, type); + result = cfreader.load_flat_config(name, type, options); if (result && type !== 'list' && type !== 'data') { result = result[0]; - if (regex.is_integer.test(result)) { + if (Array.isArray(options) && options['boolean'] === true) { + result = is_truth.test(result); + } + else if (regex.is_integer.test(result)) { result = parseInt(result, 10); } else if (regex.is_float.test(result)) { @@ -108,9 +112,21 @@ cfreader.load_json_config = function(name) { return result; } -cfreader.load_ini_config = function(name) { +cfreader.load_ini_config = function(name, options) { var result = cfreader.empty_config('ini'); var current_sect = result.main; + var current_sect_name = 'main'; + + // Initialize any booleans to false + if (options && Array.isArray(options.booleans)) { + for (var i=0; i Date: Mon, 20 Jan 2014 20:35:52 -0500 Subject: [PATCH 116/160] WS cleanups --- plugins/data.headers.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/data.headers.js b/plugins/data.headers.js index 387e78dc4..0d9a86c4c 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -44,20 +44,20 @@ function checkDateValid (plugin,connection) { connection.logdebug(plugin, "message date: " + msg_date); msg_date = Date.parse(msg_date); - if ( date_future_days > 0 ) { + if (date_future_days > 0) { var too_future = new Date; too_future.setHours(too_future.getHours() + 24 * date_future_days); // connection.logdebug(plugin, "too future: " + too_future); - if ( msg_date > too_future ) { + if (msg_date > too_future) { connection.loginfo(plugin, "date is newer than: " + too_future ); return "The Date header is too far in the future"; }; } - if ( date_past_days > 0 ) { + if (date_past_days > 0) { var too_old = new Date; too_old.setHours(too_old.getHours() - 24 * date_past_days); // connection.logdebug(plugin, "too old: " + too_old); - if ( msg_date < too_old ) { + if (msg_date < too_old) { connection.loginfo(plugin, "date is older than: " + too_old); return "The Date header is too old"; }; @@ -68,17 +68,17 @@ function checkDateValid (plugin,connection) { function refreshConfig(plugin) { var config = plugin.config.get('data.headers.ini'); - if ( config.main.required !== 'undefined' ) { + if (config.main.required !== 'undefined') { required_headers = config.main.required.split(','); }; - if ( config.main.singular !== 'undefined' ) { + if (config.main.singular !== 'undefined') { singular_headers = config.main.singular.split(','); }; - if ( config.main.date_future_days !== 'undefined' ) { + if (config.main.date_future_days !== 'undefined') { date_future_days = config.main.date_future_days; } - if ( config.main.date_past_days !== 'undefined' ) { + if (config.main.date_past_days !== 'undefined') { date_past_days = config.main.date_past_days; } } From c1052a38fbe819570b5626a915483efa1f242a60 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 02:21:45 -0500 Subject: [PATCH 117/160] headers: added has_invalid_header renamed some camelCase methods refactored a couple blocks into functions moved config checking/loading into functions where used --- plugins/data.headers.js | 119 ++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 46 deletions(-) diff --git a/plugins/data.headers.js b/plugins/data.headers.js index 0d9a86c4c..c0bb08512 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -1,42 +1,78 @@ - -// Enforce RFC 5322 Section 3.6 -var required_headers = ['Date', 'From']; -var singular_headers = ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', - 'Bcc', 'Message-Id', 'In-Reply-To', 'References', - 'Subject']; -var date_future_days = 2; -var date_past_days = 15; +// validate message headers and some fields exports.hook_data_post = function (next, connection) { var plugin = this; - refreshConfig(plugin); + var config = plugin.config.get('data.headers.ini'); - var header = connection.transaction.header; + var errmsg = has_missing_header(plugin, connection, config); + if (errmsg) return next(DENY, errmsg); - // Headers that MUST be present - for (var i=0,l=required_headers.length; i < l; i++) { - if (header.get_all(required_headers[i]).length === 0) - { - return next(DENY, "Required header '" + required_headers[i] + - "' missing"); - } - } + errmsg = has_duplicate_singular(plugin, connection, config); + if (errmsg) return next(DENY, errmsg); - // Headers that MUST be unique if present - for (var i=0,l=singular_headers.length; i < l; i++) { - if (header.get_all(singular_headers[i]).length > 1) { - return next(DENY, "Only one " + singular_headers[i] + - " header allowed. See RFC 5322, Section 3.6"); - } - }; + errmsg = has_invalid_date(plugin, connection, config); + if (errmsg) return next(DENY, errmsg); - var errmsg = checkDateValid(plugin,connection); + errmsg = has_invalid_header(plugin, connection); if (errmsg) return next(DENY, errmsg); return next(); } -function checkDateValid (plugin,connection) { +function has_duplicate_singular(plugin, connection, config) { + + // RFC 5322 Section 3.6, Headers that MUST be unique if present + var singular = config.main.singular !== 'undefined' + ? config.main.singular.split(',') + : ['Date', 'From', 'Sender', 'Reply-To', 'To', 'Cc', + 'Bcc', 'Message-Id', 'In-Reply-To', 'References', + 'Subject']; + + for (var i=0, l=singular.length; i < l; i++) { + if (connection.transaction.header.get_all(singular[i]).length > 1) { + return "Only one " + singular[i] + " header allowed. See RFC 5322, Section 3.6"; + } + }; + return; +}; + +function has_missing_header(plugin, connection, config) { + + // Enforce RFC 5322 Section 3.6, Headers that MUST be present + var required = config.main.required !== 'undefined' + ? config.main.required.split(',') + : ['Date', 'From']; + + for (var i=0, l=required.length; i < l; i++) { + if (connection.transaction.header.get_all(required[i]).length === 0) { + return "Required header '" + required[i] + "' missing"; + } + } + return; +}; + +function has_invalid_header(plugin, connection) { + // This tests for headers that shouldn't be present + + // RFC 5321#section-4.4 Trace Information + // A message-originating SMTP system SHOULD NOT send a message that + // already contains a Return-path header field. + + // Return-Path, aka Reverse-PATH, Envelope FROM, RFC5321.MailFrom + if (connection.relaying) { // On messages we originate + var rp = connection.transaction.header.get('Return-Path'); + if (rp) { + connection.loginfo(plugin, "invalid Return-Path!"); + return "outgoing mail must not have a Return-Path header (RFC 5321)"; + }; + }; + + // other tests here... + return; +}; + +function has_invalid_date (plugin, connection, config) { + // Assure Date header value is [somewhat] sane var msg_date = connection.transaction.header.get_all('Date'); if (!msg_date || msg_date.length === 0) return; @@ -44,6 +80,10 @@ function checkDateValid (plugin,connection) { connection.logdebug(plugin, "message date: " + msg_date); msg_date = Date.parse(msg_date); + var date_future_days = config.main.date_future_days !== 'undefined' + ? config.main.date_future_days + : 2; + if (date_future_days > 0) { var too_future = new Date; too_future.setHours(too_future.getHours() + 24 * date_future_days); @@ -53,6 +93,11 @@ function checkDateValid (plugin,connection) { return "The Date header is too far in the future"; }; } + + var date_past_days = config.main.date_past_days !== 'undefined' + ? config.main.date_past_days + : 15; + if (date_past_days > 0) { var too_old = new Date; too_old.setHours(too_old.getHours() - 24 * date_past_days); @@ -62,24 +107,6 @@ function checkDateValid (plugin,connection) { return "The Date header is too old"; }; }; + return; }; - -function refreshConfig(plugin) { - var config = plugin.config.get('data.headers.ini'); - - if (config.main.required !== 'undefined') { - required_headers = config.main.required.split(','); - }; - if (config.main.singular !== 'undefined') { - singular_headers = config.main.singular.split(','); - }; - - if (config.main.date_future_days !== 'undefined') { - date_future_days = config.main.date_future_days; - } - if (config.main.date_past_days !== 'undefined') { - date_past_days = config.main.date_past_days; - } -} - From 923dad0c702ef801564198326f3ada7e4f374c72 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 02:30:59 -0500 Subject: [PATCH 118/160] headers: strip Return-Path header in incoming mail read the comments. It's complicated. --- plugins/data.headers.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/data.headers.js b/plugins/data.headers.js index c0bb08512..17ee6932c 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -59,11 +59,19 @@ function has_invalid_header(plugin, connection) { // already contains a Return-path header field. // Return-Path, aka Reverse-PATH, Envelope FROM, RFC5321.MailFrom - if (connection.relaying) { // On messages we originate - var rp = connection.transaction.header.get('Return-Path'); - if (rp) { + var rp = connection.transaction.header.get('Return-Path'); + if (rp) { + if (connection.relaying) { // On messages we originate connection.loginfo(plugin, "invalid Return-Path!"); return "outgoing mail must not have a Return-Path header (RFC 5321)"; + } + else { + // generally, messages from the internet shouldn't have a + // Return-Path, except for when they can. Read RFC 5321, it's + // complicated. In most cases, The Right Thing to do here is to + // strip the Return-Path header. + connection.transaction.remove_header('Return-Path'); + // unless it was added by Haraka. Which at present, doesn't. }; }; From cd3912987e1f9f1eadfe95920907ad017415e401 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:49:26 -0500 Subject: [PATCH 119/160] headers: refactored each function into a data_post hook --- plugins/data.headers.js | 60 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/plugins/data.headers.js b/plugins/data.headers.js index 17ee6932c..b8efb32d7 100644 --- a/plugins/data.headers.js +++ b/plugins/data.headers.js @@ -1,25 +1,16 @@ // validate message headers and some fields -exports.hook_data_post = function (next, connection) { +exports.register = function () { var plugin = this; - var config = plugin.config.get('data.headers.ini'); - var errmsg = has_missing_header(plugin, connection, config); - if (errmsg) return next(DENY, errmsg); - - errmsg = has_duplicate_singular(plugin, connection, config); - if (errmsg) return next(DENY, errmsg); - - errmsg = has_invalid_date(plugin, connection, config); - if (errmsg) return next(DENY, errmsg); - - errmsg = has_invalid_header(plugin, connection); - if (errmsg) return next(DENY, errmsg); - - return next(); -} + this.register_hook('data_post', 'duplicate_singular'); + this.register_hook('data_post', 'missing_required'); + this.register_hook('data_post', 'invalid_date'); + this.register_hook('data_post', 'invalid'); +}; -function has_duplicate_singular(plugin, connection, config) { +exports.duplicate_singular = function(next, connection) { + var config = this.config.get('data.headers.ini'); // RFC 5322 Section 3.6, Headers that MUST be unique if present var singular = config.main.singular !== 'undefined' @@ -30,13 +21,16 @@ function has_duplicate_singular(plugin, connection, config) { for (var i=0, l=singular.length; i < l; i++) { if (connection.transaction.header.get_all(singular[i]).length > 1) { - return "Only one " + singular[i] + " header allowed. See RFC 5322, Section 3.6"; + return next(DENY, "Only one " + singular[i] + + " header allowed. See RFC 5322, Section 3.6"); } }; - return; + + return next(); }; -function has_missing_header(plugin, connection, config) { +exports.missing_required = function(next, connection) { + var config = this.config.get('data.headers.ini'); // Enforce RFC 5322 Section 3.6, Headers that MUST be present var required = config.main.required !== 'undefined' @@ -45,13 +39,14 @@ function has_missing_header(plugin, connection, config) { for (var i=0, l=required.length; i < l; i++) { if (connection.transaction.header.get_all(required[i]).length === 0) { - return "Required header '" + required[i] + "' missing"; + return next(DENY, "Required header '" + required[i] + "' missing"); } } - return; + + return next(); }; -function has_invalid_header(plugin, connection) { +exports.invalid = function(next, connection) { // This tests for headers that shouldn't be present // RFC 5321#section-4.4 Trace Information @@ -60,10 +55,11 @@ function has_invalid_header(plugin, connection) { // Return-Path, aka Reverse-PATH, Envelope FROM, RFC5321.MailFrom var rp = connection.transaction.header.get('Return-Path'); + var plugin = this; if (rp) { if (connection.relaying) { // On messages we originate connection.loginfo(plugin, "invalid Return-Path!"); - return "outgoing mail must not have a Return-Path header (RFC 5321)"; + return next(DENY, "outgoing mail must not have a Return-Path header (RFC 5321)"); } else { // generally, messages from the internet shouldn't have a @@ -75,15 +71,17 @@ function has_invalid_header(plugin, connection) { }; }; - // other tests here... - return; + // other invalid tests here... + return next(); }; -function has_invalid_date (plugin, connection, config) { +exports.invalid_date = function (next, connection) { + var plugin = this; // Assure Date header value is [somewhat] sane + var config = this.config.get('data.headers.ini'); var msg_date = connection.transaction.header.get_all('Date'); - if (!msg_date || msg_date.length === 0) return; + if (!msg_date || msg_date.length === 0) return next(); connection.logdebug(plugin, "message date: " + msg_date); msg_date = Date.parse(msg_date); @@ -98,7 +96,7 @@ function has_invalid_date (plugin, connection, config) { // connection.logdebug(plugin, "too future: " + too_future); if (msg_date > too_future) { connection.loginfo(plugin, "date is newer than: " + too_future ); - return "The Date header is too far in the future"; + return next(DENY, "The Date header is too far in the future"); }; } @@ -112,9 +110,9 @@ function has_invalid_date (plugin, connection, config) { // connection.logdebug(plugin, "too old: " + too_old); if (msg_date < too_old) { connection.loginfo(plugin, "date is older than: " + too_old); - return "The Date header is too old"; + return next(DENY, "The Date header is too old"); }; }; - return; + return next(); }; From 2fe950289d043c39aa754c63737648477f7a9343 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 01:34:02 -0500 Subject: [PATCH 120/160] geoip: added too_far setting --- config/connect.geoip.ini | 13 ++++++++----- plugins/connect.geoip.js | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/config/connect.geoip.ini b/config/connect.geoip.ini index b3ff8daa9..7ffc61150 100644 --- a/config/connect.geoip.ini +++ b/config/connect.geoip.ini @@ -1,13 +1,8 @@ -; enable distance calculations. If you don't use the distance, leave it -; disabled to save few CPU cycles. -calc_distance=0 - ; public_ip: the public IP address of *this* mail server ; if your mail server is not bound to a public IP, you'll have to provide ; this for distance calculations to work. ; public_ip= - ; show_city: show city data in logs and headers ; note: city data is less accurate than country show_city=1 @@ -15,3 +10,11 @@ show_city=1 ; show_region: show regional data (US states, CA provinces, etc..) show_region=1 +; enable distance calculations. If you don't use the distance, leave it +; disabled to save few CPU cycles. +calc_distance=0 + +; if calculating distance, an additional 'too_far' key in the geoip +; connection note can be set to true if the distance exceeds the limit. +; A suggested use for that data is the karma plugin. +;too_far=4000 diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index fa979bd11..e6089aba7 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -55,6 +55,9 @@ function calculate_distance(plugin, connection, cfg) { var gcd = haversine(local_geoip.ll[0], local_geoip.ll[1], connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); + if (cfg.main.too_far && (ParseFloat(cfg.main.too_far) < parseFloat(gcd))) { + connection.notes.geoip.too_far=1; + }; connection.notes.geoip.distance = gcd; }; From 71427a74fe6e930246f10362649976c8a8bfc791 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:26:37 -0500 Subject: [PATCH 121/160] geoip: s/1/true/, note distance units are km fixed s/ParseFloat/parseFloat/ --- config/connect.geoip.ini | 4 ++-- plugins/connect.geoip.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/connect.geoip.ini b/config/connect.geoip.ini index 7ffc61150..1077f24d0 100644 --- a/config/connect.geoip.ini +++ b/config/connect.geoip.ini @@ -15,6 +15,6 @@ show_region=1 calc_distance=0 ; if calculating distance, an additional 'too_far' key in the geoip -; connection note can be set to true if the distance exceeds the limit. -; A suggested use for that data is the karma plugin. +; connection note can be set to true if the distance exceeds the limit (in +; kilometers). A suggested use for that data is the karma plugin. ;too_far=4000 diff --git a/plugins/connect.geoip.js b/plugins/connect.geoip.js index e6089aba7..edd0d7923 100644 --- a/plugins/connect.geoip.js +++ b/plugins/connect.geoip.js @@ -55,8 +55,8 @@ function calculate_distance(plugin, connection, cfg) { var gcd = haversine(local_geoip.ll[0], local_geoip.ll[1], connection.notes.geoip.ll[0], connection.notes.geoip.ll[1]); - if (cfg.main.too_far && (ParseFloat(cfg.main.too_far) < parseFloat(gcd))) { - connection.notes.geoip.too_far=1; + if (cfg.main.too_far && (parseFloat(cfg.main.too_far) < parseFloat(gcd))) { + connection.notes.geoip.too_far=true; }; connection.notes.geoip.distance = gcd; }; From 40d6c91ca733368564b219c9545398e8863e1f41 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 24 Jan 2014 12:43:43 -0500 Subject: [PATCH 122/160] auth/flat: disallow AUTH unless private IP or TLS --- plugins/auth/flat_file.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/auth/flat_file.js b/plugins/auth/flat_file.js index da3f7e3ee..81b432ba1 100644 --- a/plugins/auth/flat_file.js +++ b/plugins/auth/flat_file.js @@ -5,6 +5,8 @@ exports.register = function () { } exports.hook_capabilities = function (next, connection) { + // don't allow AUTH unless private IP or encrypted + if (!net_utils.is_rfc1918(connection.remote_ip) && !connection.using_tls) return next(); var config = this.config.get('auth_flat_file.ini'); var methods = (config.core && config.core.methods ) ? config.core.methods.split(',') : null; if(methods && methods.length > 0) { From 7f18dbb0c0e9593fed3d7595aac3e169e3f5d107 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 23:49:55 -0500 Subject: [PATCH 123/160] Steve's delay_checks plugin renamed to delay_deny, since that's actually what it does (it doesn't delay processing, it intercepts and defers the denial) --- plugins/delay_deny.js | 170 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 plugins/delay_deny.js diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js new file mode 100644 index 000000000..3d899a973 --- /dev/null +++ b/plugins/delay_deny.js @@ -0,0 +1,170 @@ +/* +** delay_deny +** +** This plugin delays all pre-DATA 'deny' results until the recipients are sent +** and all post-DATA commands until all hook_data_post plugins have run. +** This allows relays and authenticated users to bypass pre-DATA rejections. +*/ + +var constants = require('./constants'); + +exports.hook_deny = function (next, connection, params) { + /* params + ** [0] = plugin return value (constants.deny or constants.denysoft) + ** [1] = plugin return message + */ + + var pi_name = params[2]; + var pi_function = params[3]; + var pi_params = params[4]; + var pi_hook = params[5]; + + var plugin = this; + var transaction = connection.transaction; + + // Don't delay ourselves... + if (pi_name == 'delay_deny') return next(); + + // Load config + var cfg = this.config.get('delay_deny.ini'); + var skip = cfg.main.excluded_plugins.split(/[;, ]+/); + + // See if we should skip this delay + if (skip && skip.length) { + // Skip by + if (skip.indexOf(pi_name) !== -1) { + connection.logdebug(plugin, 'not delaying excluded plugin: ' + pi_name); + return next(); + } + // Skip by : + if (skip.indexOf(pi_name + ':' + pi_hook) !== -1) { + connection.logdebug(plugin, 'not delaying excluded hook: ' + pi_hook + + ' in plugin: ' + pi_name); + return next(); + } + // Skip by :: + if (skip.indexOf(pi_name + ':' + pi_hook + ':' + pi_function) !== -1) { + connection.logdebug(plugin, 'not delaying excluded function: ' + pi_function + + ' on hook: ' + pi_hook + ' in plugin: ' + pi_name); + return next(); + } + } + + switch(pi_hook) { + // Pre-DATA connection delays + case 'lookup_rdns': + case 'connect': + case 'ehlo': + case 'helo': + if (!connection.notes.delay_deny) { + connection.notes.delay_deny_pre = []; + } + connection.notes.delay_deny_pre.push(params); + if (!connection.notes.delay_deny_pre_fail) { + connection.notes.delay_deny_pre_fail = {}; + } + connection.notes.delay_deny_pre_fail[pi_name] = 1; + return next(OK); + break; + // Pre-DATA transaction delays + case 'mail': + case 'rcpt': + case 'rcpt_ok': + if (!transaction.notes.delay_deny_pre) { + transaction.notes.delay_deny_pre = []; + } + transaction.notes.delay_deny_pre.push(params); + if (!transaction.notes.delay_deny_pre_fail) { + transaction.notes.delay_deny_pre_fail = {}; + } + transaction.notes.delay_deny_pre_fail[pi_name] = 1; + return next(OK); + break; + // Post-DATA delays + case 'data': + case 'data_post': + /* Delays disabled for now + if (!transaction.notes.delay_deny_post) { + transaction.notes.delay_deny_post = []; + } + transaction.notes.delay_deny_post.push(params); + if (!transaction.notes.delay_deny_post_fail) { + transaction.notes.delay_deny_post_fail = {}; + } + transaction.notes.delay_deny_post_fail[pi_name] = 1; + return next(OK); + break; + */ + default: + // No delays + return next(); + } +} + +exports.hook_rcpt_ok = function (next, connection, rcpt) { + var plugin = this; + var transaction = connection.transaction; + + // Bypass all pre-DATA deny for AUTH/RELAY + if (connection.relaying) { + connection.loginfo(plugin, 'bypassing all pre-DATA deny: AUTH/RELAY'); + return next(); + } + + // Apply any delayed rejections + // Check connection level pre-DATA rejections first + if (connection.notes.delay_deny_pre && + connection.notes.delay_deny_pre.length > 0) + { + for (var i=0; i 0) + { + for (var i=0; i Date: Fri, 10 Jan 2014 23:51:13 -0500 Subject: [PATCH 124/160] delay_deny: don't die if config setting is missing --- plugins/delay_deny.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index 3d899a973..b10acaa56 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -27,7 +27,10 @@ exports.hook_deny = function (next, connection, params) { // Load config var cfg = this.config.get('delay_deny.ini'); - var skip = cfg.main.excluded_plugins.split(/[;, ]+/); + var skip; + if ( cfg.main.excluded_plugins ) { + skip = cfg.main.excluded_plugins.split(/[;, ]+/); + }; // See if we should skip this delay if (skip && skip.length) { From 1ec58417c44166cafe5025ae3f94d5d6926a4f35 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 10 Jan 2014 23:52:03 -0500 Subject: [PATCH 125/160] added delay_deny.ini --- config/delay_deny.ini | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 config/delay_deny.ini diff --git a/config/delay_deny.ini b/config/delay_deny.ini new file mode 100644 index 000000000..0d63f0e9d --- /dev/null +++ b/config/delay_deny.ini @@ -0,0 +1,7 @@ + +; excluded plugins: a list of denials that are to be excluded (ie, all the immediate rejection) +; Examples: +; : +; :: +; +;excluded_plugins=spf,lookup_rdns_strict From cca377d9f38879d76d2a38529b8504e7bb1055e4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 11 Jan 2014 00:33:49 -0500 Subject: [PATCH 126/160] replace concat with push.apply (jsperf 100x faster) --- plugins/delay_deny.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index b10acaa56..847a15a9a 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -162,10 +162,10 @@ exports.hook_data = function (next, connection) { // Add a header showing all pre-DATA rejections var fails = []; if (connection.notes.delay_deny_pre_fail) { - fails = fails.concat(Object.keys(connection.notes.delay_deny_pre_fail)); + fails.push.apply(Object.keys(connection.notes.delay_deny_pre_fail)); } if (transaction.notes.delay_deny_pre_fail) { - fails = fails.concat(Object.keys(transaction.notes.delay_deny_pre_fail)); + fails.push.apply(Object.keys(transaction.notes.delay_deny_pre_fail)); } if (fails.length) transaction.add_header('X-Haraka-Fail-Pre', fails.join(' ')); From eb971042e21f2672a2007cc3ec02280ec41e1b13 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Thu, 16 Jan 2014 17:31:54 -0500 Subject: [PATCH 127/160] added doc delay_deny.md --- docs/plugins/delay_deny.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/plugins/delay_deny.md diff --git a/docs/plugins/delay_deny.md b/docs/plugins/delay_deny.md new file mode 100644 index 000000000..46ca6385e --- /dev/null +++ b/docs/plugins/delay_deny.md @@ -0,0 +1,6 @@ +# delay_deny + +Delays all pre-DATA 'deny' results until the recipients are sent +and all post-DATA commands until all hook_data_post plugins have run. +This allows relays and authenticated users to bypass pre-DATA rejections. + From 04b8db679c5ccfd616eedde7da278f877ddd51a5 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 17 Jan 2014 13:42:53 -0500 Subject: [PATCH 128/160] delay_deny: noted config file in .md --- docs/plugins/delay_deny.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/plugins/delay_deny.md b/docs/plugins/delay_deny.md index 46ca6385e..a5cdd18f8 100644 --- a/docs/plugins/delay_deny.md +++ b/docs/plugins/delay_deny.md @@ -4,3 +4,11 @@ Delays all pre-DATA 'deny' results until the recipients are sent and all post-DATA commands until all hook_data_post plugins have run. This allows relays and authenticated users to bypass pre-DATA rejections. +## Configuration + +Configuration options are in config/delay_deny.ini. + +### excluded plugins + +a list of denials that are to be excluded (ie, all the immediate rejection) + From eb34f103f294bd8a7d27c5415531ee62838c8e82 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 19:58:48 -0500 Subject: [PATCH 129/160] WS cleanup --- plugins/delay_deny.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index 847a15a9a..bf61e9b06 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -28,7 +28,7 @@ exports.hook_deny = function (next, connection, params) { // Load config var cfg = this.config.get('delay_deny.ini'); var skip; - if ( cfg.main.excluded_plugins ) { + if (cfg.main.excluded_plugins) { skip = cfg.main.excluded_plugins.split(/[;, ]+/); }; From 435ac78db7a1f36f38cd73fa57e162f1684882f8 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 00:33:23 -0500 Subject: [PATCH 130/160] spamassassin: whitespace cleanups removed lots of trailing whitespace --- plugins/spamassassin.js | 60 ++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 9be066616..ef5881f17 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -11,14 +11,13 @@ var defaults = { exports.hook_data_post = function (next, connection) { var plugin = this; - var config = this.config.get('spamassassin.ini'); - + for (var key in defaults) { config.main[key] = config.main[key] || defaults[key]; } - - ['reject_threshold', 'relay_reject_threshold', + + ['reject_threshold', 'relay_reject_threshold', 'munge_subject_threshold', 'max_size'].forEach( function (item) { if (config.main[item]) { @@ -26,30 +25,31 @@ exports.hook_data_post = function (next, connection) { } } ); - - if (config.main.max_size > 0 && - connection.transaction.data_bytes > config.main.max_size) - { - connection.loginfo(this, 'skipping: message exceeds maximum size'); - return next(); - } - + + if (config.main.max_size) { + var bytes = connection.transaction.data_bytes / (1024*1024); // to MB + var max = config.main.max_size / (1024 * 1024); + if (bytes > max) { + connection.loginfo(plugin, 'skipping, size ('+bytes+'MB) exceeds max: '+max); + return next(); + } + }; + var socket = new sock.Socket(); - if (config.main.spamd_socket.match(/\//)) { - // assume unix socket + if (config.main.spamd_socket.match(/\//)) { // assume unix socket socket.connect(config.main.spamd_socket); } else { var hostport = config.main.spamd_socket.split(/:/); socket.connect((hostport[1] || 783), hostport[0]); } - + socket.setTimeout(300 * 1000); - - var username = config.main.spamd_user || + + var username = config.main.spamd_user || connection.transaction.notes.spamd_user || 'default'; - + socket.on('timeout', function () { connection.logerror(plugin, "spamd connection timed out"); socket.end(); @@ -58,7 +58,7 @@ exports.hook_data_post = function (next, connection) { socket.on('error', function (err) { connection.logerror(plugin, "spamd connection failed: " + err); // we don't deny on error - maybe another plugin can deliver - next(); + next(); }); socket.on('connect', function () { var headers = [ @@ -75,10 +75,10 @@ exports.hook_data_post = function (next, connection) { socket.write(headers.join("\r\n")); connection.transaction.message_stream.pipe(socket); }); - + var spamd_response = {}; var state = 'line0'; - + socket.on('line', function (line) { connection.logprotocol(plugin, "Spamd C: " + line); line = line.replace(/\r?\n/, ''); @@ -105,16 +105,16 @@ exports.hook_data_post = function (next, connection) { socket.end(); } }); - + socket.on('end', function () { // Abort if the connection or transaction are gone if (!connection || (connection && !connection.transaction)) return next(); // Now we do stuff with the results... - + plugin.fixup_old_headers(config.main.old_headers_action, connection.transaction); - if (spamd_response.flag === 'Yes') { + if (spamd_response.flag === 'Yes') { connection.transaction.add_header('X-Spam-Flag', 'YES'); connection.transaction.remove_header('precedence'); connection.transaction.add_header('Precedence', 'junk'); @@ -122,7 +122,7 @@ exports.hook_data_post = function (next, connection) { connection.transaction.add_header('X-Spam-Status', spamd_response.flag + ', hits=' + spamd_response.hits + ' required=' + spamd_response.reqd + "\n\ttests=" + spamd_response.tests); - + var stars = Math.floor(spamd_response.hits); if (stars < 1) stars = 1; if (stars > 50) stars = 50; @@ -131,14 +131,14 @@ exports.hook_data_post = function (next, connection) { stars_string += '*'; } connection.transaction.add_header('X-Spam-Level', stars_string); - + connection.loginfo(plugin, "status=" + spamd_response.flag + ', hits=' + spamd_response.hits + ', required=' + spamd_response.reqd + - ", reject=" + ((connection.relaying) ? (config.main.relay_reject_threshold || config.main.reject_threshold) : - config.main.reject_threshold) + + ", reject=" + ((connection.relaying) ? (config.main.relay_reject_threshold || config.main.reject_threshold) : + config.main.reject_threshold) + ", tests=\"" + spamd_response.tests + "\""); - - if ((connection.relaying && config.main.relay_reject_threshold && (spamd_response.hits >= config.main.relay_reject_threshold)) + + if ((connection.relaying && config.main.relay_reject_threshold && (spamd_response.hits >= config.main.relay_reject_threshold)) || (config.main.reject_threshold && (spamd_response.hits >= config.main.reject_threshold))) { return next(DENY, "spam score exceeded threshold"); } From e7ce58adbc1bf63e79f40bae705371136c86e9a0 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 01:47:10 -0500 Subject: [PATCH 131/160] spamassassin: added spamassassin.ini --- config/spamassassin.ini | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 config/spamassassin.ini diff --git a/config/spamassassin.ini b/config/spamassassin.ini new file mode 100644 index 000000000..8cdf65419 --- /dev/null +++ b/config/spamassassin.ini @@ -0,0 +1,24 @@ +; How does Haraka connect to the SpamAssassin spamd daemon? +; TCP/IP: 127.0.0.1:783 +; socket: /var/run/spamd/spamd.sock +spamd_socket=127.0.0.1:783 + +; the username we tell spamd the message is to +spamd_user=`connection.transaction.notes.spamd_user` + +; messages larger than this are not scored by SA +max_size = 500000 + +; Munge the subject of messages with a score higher than.. +; munge_subject_threshold=5 +subject_prefix=*** SPAM *** + +; what to do with incoming messages with X-Spam-* headers +; options are: rename, drop, keep +old_headers_action=rename + +; Reject all messages with more than this many hits +; reject_threshold=10 + +; when a connection has relay privileges, the rejection limit +; relay_reject_threshold=7 From e88372f7fb3ba7be766d3ee61982d30aa48dbba8 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 01:48:11 -0500 Subject: [PATCH 132/160] store spamd response in connection note, added per-user SA pref support for vpopmail refactored into paragraph sized functions --- plugins/spamassassin.js | 218 ++++++++++++++++++++++++---------------- 1 file changed, 133 insertions(+), 85 deletions(-) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index ef5881f17..752f086c5 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -13,65 +13,15 @@ exports.hook_data_post = function (next, connection) { var plugin = this; var config = this.config.get('spamassassin.ini'); - for (var key in defaults) { - config.main[key] = config.main[key] || defaults[key]; - } - - ['reject_threshold', 'relay_reject_threshold', - 'munge_subject_threshold', 'max_size'].forEach( - function (item) { - if (config.main[item]) { - config.main[item] = new Number(config.main[item]); - } - } - ); - - if (config.main.max_size) { - var bytes = connection.transaction.data_bytes / (1024*1024); // to MB - var max = config.main.max_size / (1024 * 1024); - if (bytes > max) { - connection.loginfo(plugin, 'skipping, size ('+bytes+'MB) exceeds max: '+max); - return next(); - } - }; - - var socket = new sock.Socket(); - if (config.main.spamd_socket.match(/\//)) { // assume unix socket - socket.connect(config.main.spamd_socket); - } - else { - var hostport = config.main.spamd_socket.split(/:/); - socket.connect((hostport[1] || 783), hostport[0]); - } + setupDefaults(config); - socket.setTimeout(300 * 1000); + if (msgTooBig(config, connection, plugin)) return next(); - var username = config.main.spamd_user || - connection.transaction.notes.spamd_user || - 'default'; + var socket = getSpamdSocket(config, next, connection, plugin); + var username = getSpamdUsername(config, connection); + var headers = getSpamdHeaders(connection, username); - socket.on('timeout', function () { - connection.logerror(plugin, "spamd connection timed out"); - socket.end(); - next(); - }); - socket.on('error', function (err) { - connection.logerror(plugin, "spamd connection failed: " + err); - // we don't deny on error - maybe another plugin can deliver - next(); - }); socket.on('connect', function () { - var headers = [ - 'SYMBOLS SPAMC/1.3', - 'User: ' + username, - '', - 'X-Envelope-From: ' + connection.transaction.mail_from.address(), - 'X-Haraka-UUID: ' + connection.transaction.uuid, - ]; - if (connection.relaying) { - headers.push('X-Haraka-Relay: true'); - } - socket.write(headers.join("\r\n")); connection.transaction.message_stream.pipe(socket); }); @@ -110,50 +60,36 @@ exports.hook_data_post = function (next, connection) { // Abort if the connection or transaction are gone if (!connection || (connection && !connection.transaction)) return next(); - // Now we do stuff with the results... + // do stuff with the results... + connection.transaction.notes.spamassassin = spamd_response; plugin.fixup_old_headers(config.main.old_headers_action, connection.transaction); + doHeaderUpdates(connection, spamd_response); + logResults(connection, plugin, spamd_response, config); - if (spamd_response.flag === 'Yes') { - connection.transaction.add_header('X-Spam-Flag', 'YES'); - connection.transaction.remove_header('precedence'); - connection.transaction.add_header('Precedence', 'junk'); - } - connection.transaction.add_header('X-Spam-Status', spamd_response.flag + - ', hits=' + spamd_response.hits + ' required=' + spamd_response.reqd + - "\n\ttests=" + spamd_response.tests); - - var stars = Math.floor(spamd_response.hits); - if (stars < 1) stars = 1; - if (stars > 50) stars = 50; - var stars_string = ''; - for (var i = 0; i < stars; i++) { - stars_string += '*'; - } - connection.transaction.add_header('X-Spam-Level', stars_string); - - connection.loginfo(plugin, "status=" + spamd_response.flag + ', hits=' + - spamd_response.hits + ', required=' + spamd_response.reqd + - ", reject=" + ((connection.relaying) ? (config.main.relay_reject_threshold || config.main.reject_threshold) : - config.main.reject_threshold) + - ", tests=\"" + spamd_response.tests + "\""); + var hits = spamd_response.hits; + var rmax = config.main.relay_reject_threshold; + if (connection.relaying && rmax && (hits >= rmax)) { + return next(DENY, "spam score exceeded relay threshold"); + }; - if ((connection.relaying && config.main.relay_reject_threshold && (spamd_response.hits >= config.main.relay_reject_threshold)) - || (config.main.reject_threshold && (spamd_response.hits >= config.main.reject_threshold))) { + var max = config.main.reject_threshold; + if (max && (hits >= max)) { return next(DENY, "spam score exceeded threshold"); } - else if (config.main.munge_subject_threshold && (spamd_response.hits >= config.main.munge_subject_threshold)) { + + var munge = config.main.munge_subject_threshold; + if (munge && (hits >= munge)) { var subj = connection.transaction.header.get('Subject'); - // Try and prevent any double subject modifications + // Try and prevent double subject modifications var subject_re = new RegExp('^' + config.main.subject_prefix); if (!subject_re.test(subj)) { connection.transaction.remove_header('Subject'); connection.transaction.add_header('Subject', config.main.subject_prefix + " " + subj); } } - next(); + return next(); }); - }; exports.fixup_old_headers = function (action, transaction) { @@ -175,3 +111,115 @@ exports.fixup_old_headers = function (action, transaction) { break; } } + +function setupDefaults(config) { + for (var key in defaults) { + config.main[key] = config.main[key] || defaults[key]; + } + + ['reject_threshold', 'relay_reject_threshold', + 'munge_subject_threshold', 'max_size'].forEach( + function (item) { + if (config.main[item]) { + config.main[item] = new Number(config.main[item]); + } + } + ); +}; + +function doHeaderUpdates(connection, spamd_response) { + + if (spamd_response.flag === 'Yes') { + connection.transaction.add_header('X-Spam-Flag', 'YES'); + connection.transaction.remove_header('precedence'); + connection.transaction.add_header('Precedence', 'junk'); + } + + connection.transaction.add_header('X-Spam-Status', + spamd_response.flag + + ', hits=' + spamd_response.hits + + ' required=' + spamd_response.reqd + + "\n\ttests=" + spamd_response.tests); + + var stars = Math.floor(spamd_response.hits); + if (stars < 1) stars = 1; + if (stars > 50) stars = 50; + var stars_string = ''; + for (var i = 0; i < stars; i++) { + stars_string += '*'; + } + connection.transaction.add_header('X-Spam-Level', stars_string); +}; + +function getSpamdHeaders(connection, username) { + var headers = [ + 'SYMBOLS SPAMC/1.3', + 'User: ' + username, + '', + 'X-Envelope-From: ' + connection.transaction.mail_from.address(), + 'X-Haraka-UUID: ' + connection.transaction.uuid, + ]; + if (connection.relaying) { + headers.push('X-Haraka-Relay: true'); + } + return headers; +}; + +function getSpamdUsername(config, connection) { + var user = config.main.spamd_user || + connection.transaction.notes.spamd_user || + 'default'; + + if ( user === 'vpopmail' ) { + // allow per-user SA prefs to work + return connection.transaction.rcpt_to[0].address; + }; + return user; +}; + +function getSpamdSocket(config, next, connection, plugin) { + var socket = new sock.Socket(); + if (config.main.spamd_socket.match(/\//)) { // assume unix socket + socket.connect(config.main.spamd_socket); + } + else { + var hostport = config.main.spamd_socket.split(/:/); + socket.connect((hostport[1] || 783), hostport[0]); + } + + socket.setTimeout(300 * 1000); + + socket.on('timeout', function () { + connection.logerror(plugin, "spamd connection timed out"); + socket.end(); + return next(); + }); + socket.on('error', function (err) { + connection.logerror(plugin, "spamd connection failed: " + err); + // don't deny on error - maybe another plugin can deliver + return next(); + }); + return socket; +}; + +function msgTooBig(config, connection, plugin) { + if (!config.main.max_size) return false; + + var bytes = connection.transaction.data_bytes / (1024 * 1024); // to MB + var max = config.main.max_size / (1024 * 1024); + if (bytes > max) { + connection.loginfo(plugin, 'skipping, size ('+bytes+'MB) exceeds max: '+max); + return true; + } + return false; +}; + +function logResults(connection, plugin, spamd_response, config) { + connection.loginfo(plugin, "status=" + spamd_response.flag + + ', hits=' + spamd_response.hits + + ', required=' + spamd_response.reqd + + ', reject=' + ((connection.relaying) + ? (config.main.relay_reject_threshold || config.main.reject_threshold) + : config.main.reject_threshold) + + ', tests="' + spamd_response.tests + '"'); +}; From e76957cea72419af92f0e1934cd7b54872225b18 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 02:01:10 -0500 Subject: [PATCH 133/160] SA docs: fix typo & grammar error --- docs/plugins/spamassassin.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index 970707e31..30a65856e 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -33,7 +33,7 @@ spamassassin.ini Maximum size of messages (in bytes) to send to spamd. Messages over this size will be skipped. -- reject_thresold = N *optional* +- reject_threshold = N *optional* Default: none (do not reject any mail) @@ -83,5 +83,5 @@ A SpamAssassin plugin can be found in the `contrib` directory. The `Haraka.\[pm|cf\]` files should be placed in the SpamAssassin local site rules directory (/etc/mail/spamassassin on Linux), spamd should be restarted and the plugin will make spamd output the Haraka UUID as part -of it's log output to aid debugging when searching the mail logs. +of its log output to aid debugging when searching the mail logs. From 1eda2d776f5e5b7b4cfbc51c720e1998aa24615c Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Sat, 18 Jan 2014 19:16:35 -0500 Subject: [PATCH 134/160] spamassassin: pass SA headers through unaltered this allows sites to customize the presence and appearance of SA headers without touching this plugin --- docs/plugins/spamassassin.md | 25 +++++++ plugins/spamassassin.js | 127 ++++++++++++++++++++--------------- 2 files changed, 96 insertions(+), 56 deletions(-) diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index 30a65856e..82ecd6634 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -85,3 +85,28 @@ site rules directory (/etc/mail/spamassassin on Linux), spamd should be restarted and the plugin will make spamd output the Haraka UUID as part of its log output to aid debugging when searching the mail logs. + +Changes +-------------- + +In January 2014, the default X-Spam-Status string was changed from this: + + X-Spam-Status: No, hits=0.7 required=8.0 tests=ALL_TRUSTED,AWL,BAYES_... + +to this: + + X-Spam-Status: No, score=0.7 required=8.0 + +The old format is a legacy format from SpamAssassin versions < 3. This plugin +now passes the X-Spam-\* headers generated by SA through unaltered. You can +control the presence and appearance of X-Spam-\* headers by editing your SA +config. To mimic the old header format, add a line like this to local.cf: + + add_header all Status _YESNO_, hits=_SCORE_ required=_REQD_ tests=_TESTS_ + +Other headers options you might find interesting or useful are: + + add_header all Status _YESNO_, score=_SCORE_ required=_REQD_ autolearn=_AUTOLEARN_ + add_header all DCC _DCCB_: _DCCR_ + add_header all Checker-Version SpamAssassin _VERSION_ (_SUBVERSION_) on _HOSTNAME_ + add_header all Tests _TESTS_ diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 752f086c5..acc1ff78e 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -13,21 +13,22 @@ exports.hook_data_post = function (next, connection) { var plugin = this; var config = this.config.get('spamassassin.ini'); - setupDefaults(config); + setup_defaults(config); - if (msgTooBig(config, connection, plugin)) return next(); + if (msg_too_big(config, connection, plugin)) return next(); - var socket = getSpamdSocket(config, next, connection, plugin); - var username = getSpamdUsername(config, connection); - var headers = getSpamdHeaders(connection, username); + var username = get_spamd_username(config, connection); + var headers = get_spamd_headers(connection, username); + var socket = get_spamd_socket(config, next, connection, plugin); socket.on('connect', function () { socket.write(headers.join("\r\n")); connection.transaction.message_stream.pipe(socket); }); - var spamd_response = {}; + var spamd_response = { headers: {} }; var state = 'line0'; + var last_header; socket.on('line', function (line) { connection.logprotocol(plugin, "Spamd C: " + line); @@ -47,12 +48,23 @@ exports.hook_data_post = function (next, connection) { } } else { - state = 'tests'; + state = 'headers'; } } - else if (state === 'tests') { - spamd_response.tests = line; - socket.end(); + else if (state === 'headers') { + var m; + if (m = line.match(/^X-Spam-(\S*?):\s(.*)/)) { + last_header = m[1]; + spamd_response.headers[m[1]] = m[2]; + if (m[1] === 'Tests') spamd_response.tests = m[2]; + return; + }; + var fold; + if (last_header && (fold = line.match(/^(\s+.*)/))) { + spamd_response.headers[last_header] += fold[1]; + return; + }; + last_header = ''; } }); @@ -64,30 +76,14 @@ exports.hook_data_post = function (next, connection) { connection.transaction.notes.spamassassin = spamd_response; plugin.fixup_old_headers(config.main.old_headers_action, connection.transaction); - doHeaderUpdates(connection, spamd_response); - logResults(connection, plugin, spamd_response, config); + do_header_updates(connection, spamd_response); + log_results(connection, plugin, spamd_response, config); - var hits = spamd_response.hits; - var rmax = config.main.relay_reject_threshold; - if (connection.relaying && rmax && (hits >= rmax)) { - return next(DENY, "spam score exceeded relay threshold"); - }; + var exceeds_err = hits_too_high(config, connection, spamd_response); + if (exceeds_err) return next(DENY, exceeds_err); - var max = config.main.reject_threshold; - if (max && (hits >= max)) { - return next(DENY, "spam score exceeded threshold"); - } + munge_subject(connection, config, spamd_response.hits); - var munge = config.main.munge_subject_threshold; - if (munge && (hits >= munge)) { - var subj = connection.transaction.header.get('Subject'); - // Try and prevent double subject modifications - var subject_re = new RegExp('^' + config.main.subject_prefix); - if (!subject_re.test(subj)) { - connection.transaction.remove_header('Subject'); - connection.transaction.add_header('Subject', config.main.subject_prefix + " " + subj); - } - } return next(); }); }; @@ -112,7 +108,20 @@ exports.fixup_old_headers = function (action, transaction) { } } -function setupDefaults(config) { +function munge_subject(connection, config, hits) { + var munge = config.main.munge_subject_threshold; + if (!munge) return; + if (hits < munge) return; + + var subj = connection.transaction.header.get('Subject'); + var subject_re = new RegExp('^' + config.main.subject_prefix); + if (subject_re.test(subj)) return; // prevent double munge + + connection.transaction.remove_header('Subject'); + connection.transaction.add_header('Subject', config.main.subject_prefix + " " + subj); +}; + +function setup_defaults(config) { for (var key in defaults) { config.main[key] = config.main[key] || defaults[key]; } @@ -127,7 +136,7 @@ function setupDefaults(config) { ); }; -function doHeaderUpdates(connection, spamd_response) { +function do_header_updates(connection, spamd_response) { if (spamd_response.flag === 'Yes') { connection.transaction.add_header('X-Spam-Flag', 'YES'); @@ -135,25 +144,32 @@ function doHeaderUpdates(connection, spamd_response) { connection.transaction.add_header('Precedence', 'junk'); } - connection.transaction.add_header('X-Spam-Status', - spamd_response.flag - + ', hits=' + spamd_response.hits - + ' required=' + spamd_response.reqd - + "\n\ttests=" + spamd_response.tests); - - var stars = Math.floor(spamd_response.hits); - if (stars < 1) stars = 1; - if (stars > 50) stars = 50; - var stars_string = ''; - for (var i = 0; i < stars; i++) { - stars_string += '*'; - } - connection.transaction.add_header('X-Spam-Level', stars_string); + Object.keys(spamd_response.headers).forEach(function(key) { + connection.transaction.add_header('X-Spam-'+key, spamd_response.headers[key]); + }); }; -function getSpamdHeaders(connection, username) { +function hits_too_high(config, connection, spamd_response) { + var hits = spamd_response.hits; + if (connection.relaying) { + var rmax = config.main.relay_reject_threshold; + if ( rmax && (hits >= rmax)) { + return "spam score exceeded relay threshold"; + } + }; + + var max = config.main.reject_threshold; + if (max && (hits >= max)) { + return "spam score exceeded threshold"; + } + + return; +} + +function get_spamd_headers(connection, username) { + // http://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL var headers = [ - 'SYMBOLS SPAMC/1.3', + 'HEADERS SPAMC/1.3', 'User: ' + username, '', 'X-Envelope-From: ' + connection.transaction.mail_from.address(), @@ -165,19 +181,18 @@ function getSpamdHeaders(connection, username) { return headers; }; -function getSpamdUsername(config, connection) { +function get_spamd_username(config, connection) { var user = config.main.spamd_user || connection.transaction.notes.spamd_user || 'default'; - if ( user === 'vpopmail' ) { - // allow per-user SA prefs to work - return connection.transaction.rcpt_to[0].address; + if ( user === 'vpopmail' ) { // for per-user SA prefs + return connection.transaction.rcpt_to[0].address(); }; return user; }; -function getSpamdSocket(config, next, connection, plugin) { +function get_spamd_socket(config, next, connection, plugin) { var socket = new sock.Socket(); if (config.main.spamd_socket.match(/\//)) { // assume unix socket socket.connect(config.main.spamd_socket); @@ -202,7 +217,7 @@ function getSpamdSocket(config, next, connection, plugin) { return socket; }; -function msgTooBig(config, connection, plugin) { +function msg_too_big(config, connection, plugin) { if (!config.main.max_size) return false; var bytes = connection.transaction.data_bytes / (1024 * 1024); // to MB @@ -214,7 +229,7 @@ function msgTooBig(config, connection, plugin) { return false; }; -function logResults(connection, plugin, spamd_response, config) { +function log_results(connection, plugin, spamd_response, config) { connection.loginfo(plugin, "status=" + spamd_response.flag + ', hits=' + spamd_response.hits + ', required=' + spamd_response.reqd From 2bba25d89e80acbd7507782ae2a081ce0e2885df Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 16:30:18 -0500 Subject: [PATCH 135/160] spamassassin: use parseFloat for numberic compare --- plugins/spamassassin.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index acc1ff78e..113f45287 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -111,7 +111,7 @@ exports.fixup_old_headers = function (action, transaction) { function munge_subject(connection, config, hits) { var munge = config.main.munge_subject_threshold; if (!munge) return; - if (hits < munge) return; + if (parseFloat(hits) < parsefloat(munge)) return; var subj = connection.transaction.header.get('Subject'); var subject_re = new RegExp('^' + config.main.subject_prefix); @@ -127,13 +127,10 @@ function setup_defaults(config) { } ['reject_threshold', 'relay_reject_threshold', - 'munge_subject_threshold', 'max_size'].forEach( - function (item) { - if (config.main[item]) { - config.main[item] = new Number(config.main[item]); - } - } - ); + 'munge_subject_threshold', 'max_size'].forEach(function (item) { + if (!config.main[item]) return; + config.main[item] = Number(config.main[item]); + }); }; function do_header_updates(connection, spamd_response) { From ae0c7953c292320611ccbc251d5b378671f9df0a Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2014 19:59:39 -0500 Subject: [PATCH 136/160] WS cleanup --- plugins/spamassassin.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 113f45287..d4e4ac320 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -150,7 +150,7 @@ function hits_too_high(config, connection, spamd_response) { var hits = spamd_response.hits; if (connection.relaying) { var rmax = config.main.relay_reject_threshold; - if ( rmax && (hits >= rmax)) { + if (rmax && (hits >= rmax)) { return "spam score exceeded relay threshold"; } }; @@ -183,7 +183,7 @@ function get_spamd_username(config, connection) { connection.transaction.notes.spamd_user || 'default'; - if ( user === 'vpopmail' ) { // for per-user SA prefs + if (user === 'vpopmail') { // for per-user SA prefs return connection.transaction.rcpt_to[0].address(); }; return user; From 9cbad70c02f1f8f07def2e11700f0a65b63cc8fe Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 13:35:48 -0500 Subject: [PATCH 137/160] spamassassin: restrict X-Spam-(.*) to ASCII printable and comment out spamd_user in .ini --- config/spamassassin.ini | 2 +- plugins/spamassassin.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/spamassassin.ini b/config/spamassassin.ini index 8cdf65419..c992c8c6e 100644 --- a/config/spamassassin.ini +++ b/config/spamassassin.ini @@ -4,7 +4,7 @@ spamd_socket=127.0.0.1:783 ; the username we tell spamd the message is to -spamd_user=`connection.transaction.notes.spamd_user` +;spamd_user=vpopmail ; messages larger than this are not scored by SA max_size = 500000 diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index d4e4ac320..8e450e605 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -52,8 +52,8 @@ exports.hook_data_post = function (next, connection) { } } else if (state === 'headers') { - var m; - if (m = line.match(/^X-Spam-(\S*?):\s(.*)/)) { + var m; // printable ASCII: [ -~] + if (m = line.match(/^X-Spam-([ -~]+):\s(.*)/)) { last_header = m[1]; spamd_response.headers[m[1]] = m[2]; if (m[1] === 'Tests') spamd_response.tests = m[2]; From c5f3fa4076aa9acb3ca3cda1107f8483febeb73e Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:18:56 -0500 Subject: [PATCH 138/160] sa: added modern_status_syntax=1 to ini MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit escaped _ in spamassassin.md round MB to 2 decimals in “too big” log message --- config/spamassassin.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/spamassassin.ini b/config/spamassassin.ini index c992c8c6e..2822887e2 100644 --- a/config/spamassassin.ini +++ b/config/spamassassin.ini @@ -17,6 +17,11 @@ subject_prefix=*** SPAM *** ; options are: rename, drop, keep old_headers_action=rename +; use the SpamAssassin 3.0+ syntax in X-Spam-Status header +; modern: No, score=0.8 required=8.0 tests=... +; legacy: No, hits=0.8 required=8.0 tests=... +modern_status_syntax=1 + ; Reject all messages with more than this many hits ; reject_threshold=10 From a59460e22948ea7e8758bb36bcc02f6887dac2fe Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:23:01 -0500 Subject: [PATCH 139/160] sa: added modern_status_syntax=1 to ini MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit escaped _ in spamassassin.md round MB to 2 decimals in “too big” log message (missed a couple files in previous commit) --- docs/plugins/spamassassin.md | 16 +++++------ plugins/connect.asn.js | 52 ++++++++++++++++++++++++++++++++++++ plugins/spamassassin.js | 22 ++++++++++----- 3 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 plugins/connect.asn.js diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index 82ecd6634..cb94bb3d3 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -9,13 +9,13 @@ Configuration spamassassin.ini -- spamd_socket = \[host:port | /path/to/socket\] *optional* +- spamd\_socket = \[host:port | /path/to/socket\] *optional* Default: localhost:783 Host or path to socket where spamd is running. -- spamd_user = \[user\] *optional* +- spamd\_user = \[user\] *optional* Default: default @@ -26,20 +26,20 @@ spamassassin.ini `connection.transaction.notes.spamd_user` -- max_size = N *optional* +- max\_size = N *optional* Default: 500000 Maximum size of messages (in bytes) to send to spamd. Messages over this size will be skipped. -- reject_threshold = N *optional* +- reject\_threshold = N *optional* Default: none (do not reject any mail) SpamAssassin score at which the mail should be rejected. -- relay_reject_threshold = N *optional* +- relay\_reject\_threshold = N *optional* Default: none @@ -50,19 +50,19 @@ spamassassin.ini If this is not set, then the `reject_thresold` value is used. -- munge_subject_threshold = N *optional* +- munge\_subject\_threshold = N *optional* Default: none (do not munge the subject) Score at which the subject should be munged (prefixed). -- subject_prefix = \[prefix\] *optional* +- subject\_prefix = \[prefix\] *optional* Default: *** SPAM *** Prefix to use when munging the subject. -- old_headers_action = \[rename | drop | keep\] *optional* +- old\_headers\_action = \[rename | drop | keep\] *optional* Default: rename diff --git a/plugins/connect.asn.js b/plugins/connect.asn.js new file mode 100644 index 000000000..822e12008 --- /dev/null +++ b/plugins/connect.asn.js @@ -0,0 +1,52 @@ +// determine the ASN of the connecting IP + +var dns = require('dns'); +// var net = require('net'); +// var ipaddr = require('ipaddr.js'); + +exports.register = function () { + this.register_hook('lookup_rdns', 'on_connection'); +}; + +exports.on_connection = function (next, connection) { + var plugin = this; + var ip = connection.remote_ip; + + var zones = ['origin.asn.cymru.com', 'asn.routeviews.org'].forEach(function(zone) { + connection.logdebug(plugin, "zone: " + zone); + + var query = ip.split('.').reverse().join('.') + '.' + zone; + connection.logdebug(plugin, "query: " + query); + + dns.resolve(query, 'TXT', function (err, addrs) { + if (err) { + connection.logerror(plugin, "error: " + err); + return; + }; + + for (var i=0; i < addrs.length; i++) { + connection.loginfo(plugin, zone + " answer: " + addrs[i]); + if (zone === 'origin.asn.cymru.com') { + var asn = parse_cymru(addrs[i]); + } + else if (zone === 'asn.routeviews.org') { + var asn = parse_routeviews(addrs[i]); + }; + connection.loginfo(plugin, zone + " asn: " + asn); + }; + }); + }); + + return next(); +} + +function parse_routeviews(str) { + var r = str.split(/"\s+"/); + return r[0]; +}; + +function parse_cymru(str) { + var r = str.split(/\s+\|\s+/); + return r[0]; + +}; diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 8e450e605..091ace6ab 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -76,7 +76,7 @@ exports.hook_data_post = function (next, connection) { connection.transaction.notes.spamassassin = spamd_response; plugin.fixup_old_headers(config.main.old_headers_action, connection.transaction); - do_header_updates(connection, spamd_response); + do_header_updates(connection, spamd_response, config); log_results(connection, plugin, spamd_response, config); var exceeds_err = hits_too_high(config, connection, spamd_response); @@ -133,7 +133,7 @@ function setup_defaults(config) { }); }; -function do_header_updates(connection, spamd_response) { +function do_header_updates(connection, spamd_response, config) { if (spamd_response.flag === 'Yes') { connection.transaction.add_header('X-Spam-Flag', 'YES'); @@ -142,7 +142,15 @@ function do_header_updates(connection, spamd_response) { } Object.keys(spamd_response.headers).forEach(function(key) { - connection.transaction.add_header('X-Spam-'+key, spamd_response.headers[key]); + var modern = config.main.modern_status_syntax; + // connection.logdebug("modern: "+modern); + + if (key === 'Status' && (!modern || modern === undefined)) { + var legacy = spamd_response.headers[key].replace(/score/,'hits'); + connection.transaction.add_header('X-Spam-Status', legacy + ' tests=' + spamd_response.tests); + return; + }; + connection.transaction.add_header('X-Spam-' + key, spamd_response.headers[key]); }); }; @@ -217,10 +225,10 @@ function get_spamd_socket(config, next, connection, plugin) { function msg_too_big(config, connection, plugin) { if (!config.main.max_size) return false; - var bytes = connection.transaction.data_bytes / (1024 * 1024); // to MB - var max = config.main.max_size / (1024 * 1024); - if (bytes > max) { - connection.loginfo(plugin, 'skipping, size ('+bytes+'MB) exceeds max: '+max); + var msg_mb = connection.transaction.data_bytes / (1024 * 1024); // to MB + var max_mb= config.main.max_size / (1024 * 1024); + if (msg_mb > max_mb) { + connection.loginfo(plugin, 'skipping, size (' + bytes.toFixed(2) + 'MB) exceeds max: ' + max); return true; } return false; From 85189f555f95183d3db3339fb40c76a596644ad7 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:32:38 -0500 Subject: [PATCH 140/160] added spamassassin behavior change note to TODO --- TODO | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TODO b/TODO index 44a04b0e1..1d86c37ea 100644 --- a/TODO +++ b/TODO @@ -16,6 +16,11 @@ Outbound improvements - Disable deliveries for a domain - Pool connections by domain/MX +Plugin behavior changes + - in SpamAssassin plugin, change default behavior of 'legacy' status header. + Presently, when undefined, legacy is used. Legacy support should be changed to + only when requested, with a sunset date. + Remove the following deprecated plugins - rdns.regexp - data.nomsgid From 2e97ccce669bdd2d86148a6b543537460a1e69c1 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 21 Jan 2014 17:57:24 -0500 Subject: [PATCH 141/160] removed inadvertent commit of connect.asn --- plugins/connect.asn.js | 52 ----------------------------------------- plugins/spamassassin.js | 2 +- 2 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 plugins/connect.asn.js diff --git a/plugins/connect.asn.js b/plugins/connect.asn.js deleted file mode 100644 index 822e12008..000000000 --- a/plugins/connect.asn.js +++ /dev/null @@ -1,52 +0,0 @@ -// determine the ASN of the connecting IP - -var dns = require('dns'); -// var net = require('net'); -// var ipaddr = require('ipaddr.js'); - -exports.register = function () { - this.register_hook('lookup_rdns', 'on_connection'); -}; - -exports.on_connection = function (next, connection) { - var plugin = this; - var ip = connection.remote_ip; - - var zones = ['origin.asn.cymru.com', 'asn.routeviews.org'].forEach(function(zone) { - connection.logdebug(plugin, "zone: " + zone); - - var query = ip.split('.').reverse().join('.') + '.' + zone; - connection.logdebug(plugin, "query: " + query); - - dns.resolve(query, 'TXT', function (err, addrs) { - if (err) { - connection.logerror(plugin, "error: " + err); - return; - }; - - for (var i=0; i < addrs.length; i++) { - connection.loginfo(plugin, zone + " answer: " + addrs[i]); - if (zone === 'origin.asn.cymru.com') { - var asn = parse_cymru(addrs[i]); - } - else if (zone === 'asn.routeviews.org') { - var asn = parse_routeviews(addrs[i]); - }; - connection.loginfo(plugin, zone + " asn: " + asn); - }; - }); - }); - - return next(); -} - -function parse_routeviews(str) { - var r = str.split(/"\s+"/); - return r[0]; -}; - -function parse_cymru(str) { - var r = str.split(/\s+\|\s+/); - return r[0]; - -}; diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 091ace6ab..641a204c5 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -111,7 +111,7 @@ exports.fixup_old_headers = function (action, transaction) { function munge_subject(connection, config, hits) { var munge = config.main.munge_subject_threshold; if (!munge) return; - if (parseFloat(hits) < parsefloat(munge)) return; + if (parseFloat(hits) < parseFloat(munge)) return; var subj = connection.transaction.header.get('Subject'); var subject_re = new RegExp('^' + config.main.subject_prefix); From 67f240fa9d95ff5148d7b990c1f9e960338683da Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 27 Jan 2014 09:26:07 -0800 Subject: [PATCH 142/160] spamassassin: note special handling for vpopmail as the spamd_user --- docs/plugins/spamassassin.md | 4 ++++ plugins/spamassassin.js | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index cb94bb3d3..1897959bc 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -26,6 +26,10 @@ spamassassin.ini `connection.transaction.notes.spamd_user` + If the value is set to _vpopmail_, then the spamd_user will get set + to the first recipient of the message. See the get\_spamd\_user method + in the plugin for more details. + - max\_size = N *optional* Default: 500000 diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 641a204c5..91dd87d8e 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -193,6 +193,14 @@ function get_spamd_username(config, connection) { if (user === 'vpopmail') { // for per-user SA prefs return connection.transaction.rcpt_to[0].address(); + + // the alternative to choosing the first recipient is passing the + // message through SA for each recipient, and then applying the least + // strict result to the connection. That is occassionally useful when + // one user blacklists a sender that another user wants to get mail + // from, and the sender isn't courteous enough to unsubscribe the + // first recipient from his penny stock newsletter. If this is + // something you care about, fork this and submit a pull request. }; return user; }; From 44e574e297d0658506188e3ddf2e95edd9a56ea4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 27 Jan 2014 13:12:20 -0800 Subject: [PATCH 143/160] spamassassin: Steve's patch --- config/spamassassin.ini | 14 +++- docs/plugins/spamassassin.md | 58 +++++++-------- plugins/spamassassin.js | 132 ++++++++++++++++++----------------- utils.js | 7 ++ 4 files changed, 117 insertions(+), 94 deletions(-) diff --git a/config/spamassassin.ini b/config/spamassassin.ini index 2822887e2..cfaffebf8 100644 --- a/config/spamassassin.ini +++ b/config/spamassassin.ini @@ -3,11 +3,11 @@ ; socket: /var/run/spamd/spamd.sock spamd_socket=127.0.0.1:783 -; the username we tell spamd the message is to -;spamd_user=vpopmail +; the username we tell spamd the message is to (default: default) +;spamd_user= ; messages larger than this are not scored by SA -max_size = 500000 +max_size=500000 ; Munge the subject of messages with a score higher than.. ; munge_subject_threshold=5 @@ -27,3 +27,11 @@ modern_status_syntax=1 ; when a connection has relay privileges, the rejection limit ; relay_reject_threshold=7 + +; How long should we wait for SpamAssassin to answer the socket +; in seconds (default: 30) +;connect_timeout= + +; How long should we wait for a result from SpamAssassin +; in seconds (default: 300) +;results_timeout= diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index 1897959bc..4591989bb 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -24,11 +24,7 @@ spamassassin.ini You can also pass this value in dynamically by setting: - `connection.transaction.notes.spamd_user` - - If the value is set to _vpopmail_, then the spamd_user will get set - to the first recipient of the message. See the get\_spamd\_user method - in the plugin for more details. + `connection.transaction.notes.spamd_user` in another plugin. - max\_size = N *optional* @@ -47,7 +43,7 @@ spamassassin.ini Default: none - As above, except this threshold only applies to connections + As above, except this threshold only applies to connections that are relays (e.g. AUTH) where connection.relaying = true. This is used to set a *lower* thresold at which to reject mail from these hosts to prevent sending outbound spam. @@ -70,47 +66,53 @@ spamassassin.ini Default: rename - If old X-Spam-\* headers are in the email, what do we do with them? + If old X-Spam-\* headers are in the email, what do we do with them? - `rename` them to X-Old-Spam-\*. + `rename` them to X-Old-Spam-\*. - `drop` will delete them. + `drop` will delete them. - `keep` will keep them (new X-Spam-\* headers appear lower down in + `keep` will keep them (new X-Spam-\* headers appear lower down in the headers then). +- connect\_timeout = N *optional* + + Default: 30 + + Time in seconds to wait for a connection to spamd + +- results\_timeout = N *optional* + + Default: 300 + + Time in seconds to wait for results from spamd + Extras ====== -A SpamAssassin plugin can be found in the `contrib` directory. -The `Haraka.\[pm|cf\]` files should be placed in the SpamAssassin local -site rules directory (/etc/mail/spamassassin on Linux), spamd should be -restarted and the plugin will make spamd output the Haraka UUID as part +A SpamAssassin plugin can be found in the `contrib` directory. +The `Haraka.\[pm|cf\]` files should be placed in the SpamAssassin local +site rules directory (/etc/mail/spamassassin on Linux), spamd should be +restarted and the plugin will make spamd output the Haraka UUID as part of its log output to aid debugging when searching the mail logs. Changes -------------- -In January 2014, the default X-Spam-Status string was changed from this: - - X-Spam-Status: No, hits=0.7 required=8.0 tests=ALL_TRUSTED,AWL,BAYES_... - -to this: +This plugin now passes the X-Spam-\* headers generated by SA through +unaltered. You can control the presence and appearance of X-Spam-\* +headers by editing your SpamAssassin config. - X-Spam-Status: No, score=0.7 required=8.0 +The default headers added by SpamAssassin are: -The old format is a legacy format from SpamAssassin versions < 3. This plugin -now passes the X-Spam-\* headers generated by SA through unaltered. You can -control the presence and appearance of X-Spam-\* headers by editing your SA -config. To mimic the old header format, add a line like this to local.cf: - - add_header all Status _YESNO_, hits=_SCORE_ required=_REQD_ tests=_TESTS_ + add_header all Checker-Version SpamAssassin _VERSION_ (_SUBVERSION_) on _HOSTNAME_ + add_header spam Flag _YESNOCAPS_ + add_header all Level _STARS(\*)_ + add_header all Status "_YESNO_, score=_SCORE_ required=_REQD_ tests=_TESTS_ autolearn=_AUTOLEARN_ version=_VERSION_" Other headers options you might find interesting or useful are: - add_header all Status _YESNO_, score=_SCORE_ required=_REQD_ autolearn=_AUTOLEARN_ add_header all DCC _DCCB_: _DCCR_ - add_header all Checker-Version SpamAssassin _VERSION_ (_SUBVERSION_) on _HOSTNAME_ add_header all Tests _TESTS_ diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 91dd87d8e..e49194ee8 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -1,6 +1,7 @@ // Call spamassassin via spamd var sock = require('./line_socket'); +var prettySize = require('./utils').prettySize; var defaults = { spamd_socket: 'localhost:783', @@ -17,11 +18,18 @@ exports.hook_data_post = function (next, connection) { if (msg_too_big(config, connection, plugin)) return next(); - var username = get_spamd_username(config, connection); - var headers = get_spamd_headers(connection, username); - var socket = get_spamd_socket(config, next, connection, plugin); + var username = config.main.spamd_user || + connection.transaction.notes.spamd_user || + 'default'; + var headers = get_spamd_headers(connection, username); + var socket = get_spamd_socket(config, next, connection, plugin); + socket.is_connected = false; + var results_timeout = parseInt(config.main.results_timeout) || 300; socket.on('connect', function () { + this.is_connected = true; + // Reset timeout + this.setTimeout(results_timeout * 1000); socket.write(headers.join("\r\n")); connection.transaction.message_stream.pipe(socket); }); @@ -42,7 +50,8 @@ exports.hook_data_post = function (next, connection) { var matches; if (matches = line.match(/Spam: (True|False) ; (-?\d+\.\d) \/ (-?\d+\.\d)/)) { spamd_response.flag = matches[1]; - spamd_response.hits = matches[2]; + spamd_response.score = matches[2]; + spamd_response.hits = matches[2]; // backwards compat spamd_response.reqd = matches[3]; spamd_response.flag = spamd_response.flag === 'True' ? 'Yes' : 'No' } @@ -56,12 +65,11 @@ exports.hook_data_post = function (next, connection) { if (m = line.match(/^X-Spam-([ -~]+):\s(.*)/)) { last_header = m[1]; spamd_response.headers[m[1]] = m[2]; - if (m[1] === 'Tests') spamd_response.tests = m[2]; return; }; var fold; if (last_header && (fold = line.match(/^(\s+.*)/))) { - spamd_response.headers[last_header] += fold[1]; + spamd_response.headers[last_header] += "\r\n" + fold[1]; return; }; last_header = ''; @@ -69,8 +77,14 @@ exports.hook_data_post = function (next, connection) { }); socket.on('end', function () { - // Abort if the connection or transaction are gone - if (!connection || (connection && !connection.transaction)) return next(); + // Abort if the transaction is gone + if (!connection.transaction) return next(); + + // We have to strip the tests hit from the X-Spam-Status header + var tests; + if (tests = /tests=([^ ]+)/.exec(spamd_response.headers['Status'].replace(/\r?\n\t/g,''))) { + spamd_response.tests = tests[1]; + } // do stuff with the results... connection.transaction.notes.spamassassin = spamd_response; @@ -79,39 +93,45 @@ exports.hook_data_post = function (next, connection) { do_header_updates(connection, spamd_response, config); log_results(connection, plugin, spamd_response, config); - var exceeds_err = hits_too_high(config, connection, spamd_response); + var exceeds_err = score_too_high(config, connection, spamd_response); if (exceeds_err) return next(DENY, exceeds_err); - munge_subject(connection, config, spamd_response.hits); + munge_subject(connection, config, spamd_response.score); return next(); }); }; exports.fixup_old_headers = function (action, transaction) { - var headers = ['X-Spam-Flag', 'X-Spam-Status', 'X-Spam-Level']; + var headers = transaction.notes.spamassassin.headers; switch (action) { - case "keep": return; - case "drop": for (var key in headers) { transaction.remove_header(key) } - break; + case "keep": + return; + case "drop": + for (var key in headers) { + key = 'X-Spam-' + key; + transaction.remove_header(key) + } + break; case "rename": default: - for (var key in headers) { - var old_val = transaction.header.get(key); - if (old_val) { - transaction.header.remove_header(key); - transaction.header.add_header(key.replace(/X-/, 'X-Old-'), old_val); - } - } - break; + for (var key in headers) { + var key = 'X-Spam-' + key; + var old_val = transaction.header.get(key); + if (old_val) { + transaction.header.remove_header(key); + transaction.header.add_header(key.replace(/^X-/, 'X-Old-'), old_val); + } + } + break; } } -function munge_subject(connection, config, hits) { +function munge_subject(connection, config, score) { var munge = config.main.munge_subject_threshold; if (!munge) return; - if (parseFloat(hits) < parseFloat(munge)) return; + if (parseFloat(score) < parseFloat(munge)) return; var subj = connection.transaction.header.get('Subject'); var subject_re = new RegExp('^' + config.main.subject_prefix); @@ -134,37 +154,34 @@ function setup_defaults(config) { }; function do_header_updates(connection, spamd_response, config) { - if (spamd_response.flag === 'Yes') { - connection.transaction.add_header('X-Spam-Flag', 'YES'); + // X-Spam-Flag is added by SpamAssassin connection.transaction.remove_header('precedence'); connection.transaction.add_header('Precedence', 'junk'); } Object.keys(spamd_response.headers).forEach(function(key) { var modern = config.main.modern_status_syntax; - // connection.logdebug("modern: "+modern); - - if (key === 'Status' && (!modern || modern === undefined)) { - var legacy = spamd_response.headers[key].replace(/score/,'hits'); - connection.transaction.add_header('X-Spam-Status', legacy + ' tests=' + spamd_response.tests); + if (key === 'Status' && !modern) { + var legacy = spamd_response.headers[key].replace(/ score=/,' hits='); + connection.transaction.add_header('X-Spam-Status', legacy); return; }; connection.transaction.add_header('X-Spam-' + key, spamd_response.headers[key]); }); }; -function hits_too_high(config, connection, spamd_response) { - var hits = spamd_response.hits; +function score_too_high(config, connection, spamd_response) { + var score = spamd_response.score; if (connection.relaying) { var rmax = config.main.relay_reject_threshold; - if (rmax && (hits >= rmax)) { + if (rmax && (score >= rmax)) { return "spam score exceeded relay threshold"; } }; var max = config.main.reject_threshold; - if (max && (hits >= max)) { + if (max && (score >= max)) { return "spam score exceeded threshold"; } @@ -186,26 +203,8 @@ function get_spamd_headers(connection, username) { return headers; }; -function get_spamd_username(config, connection) { - var user = config.main.spamd_user || - connection.transaction.notes.spamd_user || - 'default'; - - if (user === 'vpopmail') { // for per-user SA prefs - return connection.transaction.rcpt_to[0].address(); - - // the alternative to choosing the first recipient is passing the - // message through SA for each recipient, and then applying the least - // strict result to the connection. That is occassionally useful when - // one user blacklists a sender that another user wants to get mail - // from, and the sender isn't courteous enough to unsubscribe the - // first recipient from his penny stock newsletter. If this is - // something you care about, fork this and submit a pull request. - }; - return user; -}; - function get_spamd_socket(config, next, connection, plugin) { + // TODO: support multiple spamd backends var socket = new sock.Socket(); if (config.main.spamd_socket.match(/\//)) { // assume unix socket socket.connect(config.main.spamd_socket); @@ -215,16 +214,23 @@ function get_spamd_socket(config, next, connection, plugin) { socket.connect((hostport[1] || 783), hostport[0]); } - socket.setTimeout(300 * 1000); + var connect_timeout = parseInt(config.main.connect_timeout) || 30; + socket.setTimeout(connect_timeout * 1000); socket.on('timeout', function () { - connection.logerror(plugin, "spamd connection timed out"); + if (!this.is_connected) { + connection.logerror(plugin, 'connection timed out'); + } + else { + connection.logerror(plugin, 'timeout waiting for results'); + } socket.end(); return next(); }); socket.on('error', function (err) { - connection.logerror(plugin, "spamd connection failed: " + err); - // don't deny on error - maybe another plugin can deliver + connection.logerror(plugin, 'connection failed: ' + err); + // TODO: optionally DENYSOFT + // TODO: add a transaction note return next(); }); return socket; @@ -233,10 +239,10 @@ function get_spamd_socket(config, next, connection, plugin) { function msg_too_big(config, connection, plugin) { if (!config.main.max_size) return false; - var msg_mb = connection.transaction.data_bytes / (1024 * 1024); // to MB - var max_mb= config.main.max_size / (1024 * 1024); - if (msg_mb > max_mb) { - connection.loginfo(plugin, 'skipping, size (' + bytes.toFixed(2) + 'MB) exceeds max: ' + max); + var size = connection.transaction.data_bytes; + var max = config.main.max_size; + if (size > max) { + connection.loginfo(plugin, 'skipping, size ' + prettySize(size) + ' exceeds max: ' + prettySize(max)); return true; } return false; @@ -244,7 +250,7 @@ function msg_too_big(config, connection, plugin) { function log_results(connection, plugin, spamd_response, config) { connection.loginfo(plugin, "status=" + spamd_response.flag - + ', hits=' + spamd_response.hits + + ', score=' + spamd_response.score + ', required=' + spamd_response.reqd + ', reject=' + ((connection.relaying) ? (config.main.relay_reject_threshold || config.main.reject_threshold) diff --git a/utils.js b/utils.js index 483f9af12..7774ad17c 100644 --- a/utils.js +++ b/utils.js @@ -166,3 +166,10 @@ exports.indexOfLF = function (buf, maxlength) { } return -1; } + +exports.prettySize = function (size) { + if (size === 0 || !size) return 0; + var i = Math.floor(Math.log(size)/Math.log(1024)); + var units = ['B', 'kB', 'MB', 'GB', 'TB']; + return (size/Math.pow(1024,i)).toFixed(2) * 1 + '' + units[i]; +} From bf3e40651c93866e2f27e02c766a7e196d87de50 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Mon, 27 Jan 2014 21:34:24 +0000 Subject: [PATCH 144/160] Fix typo --- plugins/spamassassin.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index e49194ee8..5d1718c12 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -120,8 +120,8 @@ exports.fixup_old_headers = function (action, transaction) { var key = 'X-Spam-' + key; var old_val = transaction.header.get(key); if (old_val) { - transaction.header.remove_header(key); - transaction.header.add_header(key.replace(/^X-/, 'X-Old-'), old_val); + transaction.remove_header(key); + transaction.add_header(key.replace(/^X-/, 'X-Old-'), old_val); } } break; From e232ee701ca1ff07eaf6159a29bea51bc3ecc4d4 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 27 Jan 2014 21:32:37 -0500 Subject: [PATCH 145/160] auth_vpopmaild: bug fix auth results weren't always valid --- plugins/auth/auth_vpopmaild.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/plugins/auth/auth_vpopmaild.js b/plugins/auth/auth_vpopmaild.js index 97b98bbfc..b28a20c3c 100644 --- a/plugins/auth/auth_vpopmaild.js +++ b/plugins/auth/auth_vpopmaild.js @@ -27,6 +27,7 @@ exports.try_auth_vpopmaild = function (connection, user, passwd, cb) { var auth_success = false; var result = ""; + var ok_count = 0; var socket = new sock.Socket(); socket.connect( ( config.main.port || 89), (config.main.host || '127.0.0.1') ); @@ -38,25 +39,31 @@ exports.try_auth_vpopmaild = function (connection, user, passwd, cb) { }); socket.on('error', function (err) { connection.logerror(plugin, "vpopmaild connection failed: " + err); + socket.end(); }); socket.on('connect', function () { - socket.write("login " + user + ' ' + passwd + "\n\r"); + // wait for server to send us +OK vvvvv }); socket.on('line', function (line) { - connection.logprotocol(plugin, 'C:' + line); - if (line.match(/^\+OK/)) { - auth_success = true; + if (line.match(/^\+OK/)) { // default server response: +OK + ok_count++; + if (ok_count === 1) { // first OK is just a 'ready' + socket.write("slogin " + user + ' ' + passwd + "\n\r"); + } + if (ok_count === 2) { // second OK is response to slogin + auth_success = true; + socket.write("quit\n\r"); + } } - if ( line.match(/^\./) ) { + if (line.match(/^\-ERR/)) { // auth failed + // socket.write("quit\n\r"); // DANGER! This returns '+OK' socket.end(); } }); - socket.on('close', function () { + // socket.on('close', function () { }); + socket.on('end', function () { connection.loginfo(plugin, 'AUTH user="' + user + '" success=' + auth_success); return cb(auth_success); }); - socket.on('end', function () { - // return cb(auth_success); - }); }; From 641d18d71806c16775764b065d9ed0000cbc4b83 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Tue, 28 Jan 2014 10:10:07 -0800 Subject: [PATCH 146/160] replace code comment with longer text comment --- plugins/auth/auth_vpopmaild.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/auth/auth_vpopmaild.js b/plugins/auth/auth_vpopmaild.js index b28a20c3c..a773df3aa 100644 --- a/plugins/auth/auth_vpopmaild.js +++ b/plugins/auth/auth_vpopmaild.js @@ -56,11 +56,12 @@ exports.try_auth_vpopmaild = function (connection, user, passwd, cb) { } } if (line.match(/^\-ERR/)) { // auth failed - // socket.write("quit\n\r"); // DANGER! This returns '+OK' - socket.end(); + // DANGER, do not say 'goodbye' to the server with "quit\n\r". The + // server will respond '+OK', which could be mis-interpreted as an + // auth response. + socket.end(); // disconnect } }); - // socket.on('close', function () { }); socket.on('end', function () { connection.loginfo(plugin, 'AUTH user="' + user + '" success=' + auth_success); return cb(auth_success); From bbed4617c482b2daccc72a1815faf6fdf8305377 Mon Sep 17 00:00:00 2001 From: Christopher Mooney Date: Wed, 29 Jan 2014 18:14:03 -0800 Subject: [PATCH 147/160] Check whitelists before checking DNS This change is a slight performance improvement, as well as a bug fix for the case where an IP address is in the whitelist. Basically, the whitelist should always be checked before any run of DNS queries. --- plugins/lookup_rdns.strict.js | 50 +++++++++++++++-------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/plugins/lookup_rdns.strict.js b/plugins/lookup_rdns.strict.js index 5b22717ed..f4dfafe76 100644 --- a/plugins/lookup_rdns.strict.js +++ b/plugins/lookup_rdns.strict.js @@ -71,18 +71,18 @@ exports.hook_lookup_rdns = function (next, connection) { var timeout = config.general && (config.general['timeout'] || 60); var timeout_msg = config.general && (config.general['timeout_msg'] || ''); + if (_in_whitelist(connection, plugin, connection.remote_ip)) { + called_next++; + next(OK, connection.remote_ip); + } + timeout_id = setTimeout(function () { if (!called_next) { connection.loginfo(plugin, 'timed out when looking up ' + connection.remote_ip + '. Disconnecting.'); called_next++; - - if (_in_whitelist(connection, plugin, connection.remote_ip)) { - next(OK, connection.remote_ip); - } else { - next(DENYDISCONNECT, '[' + connection.remote_ip + '] ' + - timeout_msg); - } + next(DENYDISCONNECT, '[' + connection.remote_ip + '] ' + + timeout_msg); } }, timeout * 1000); @@ -91,13 +91,8 @@ exports.hook_lookup_rdns = function (next, connection) { if (!called_next) { called_next++; clearTimeout(timeout_id); - - if (_in_whitelist(connection, plugin, connection.remote_ip)) { - next(OK, connection.remote_ip); - } else { - _dns_error(connection, next, err, connection.remote_ip, plugin, - rev_nxdomain, rev_dnserror); - } + _dns_error(connection, next, err, connection.remote_ip, plugin, + rev_nxdomain, rev_dnserror); } } else { // Anything this strange needs documentation. Since we are @@ -107,6 +102,15 @@ exports.hook_lookup_rdns = function (next, connection) { // on err, this helps us figure out if we still have more to check. total_checks = domains.length; + // Check whitelist before we start doing a bunch more DNS queries. + for(var i = 0; i < domains.length; i++) { + if (_in_whitelist(connection, plugin, domains[i])) { + called_next++; + clearTimeout(timeout_id); + next(OK, domains[i]); + } + } + // Now we should make sure that the reverse response matches // the forward address. Almost no one will have more than one // PTR record for a domain, however, DNS protocol does not @@ -120,13 +124,8 @@ exports.hook_lookup_rdns = function (next, connection) { if (!called_next && !total_checks) { called_next++; clearTimeout(timeout_id); - - if (_in_whitelist(connection, plugin, rdns)) { - next(OK, rdns); - } else { - _dns_error(connection, next, err, rdns, plugin, - fwd_nxdomain, fwd_dnserror); - } + _dns_error(connection, next, err, rdns, plugin, + fwd_nxdomain, fwd_dnserror); } } else { for (var i = 0; i < addresses.length ; i++) { @@ -143,13 +142,8 @@ exports.hook_lookup_rdns = function (next, connection) { if (!called_next && !total_checks) { called_next++; clearTimeout(timeout_id); - - if (_in_whitelist(connection, plugin, rdns)) { - next(OK, rdns); - } else { - next(DENYDISCONNECT, rdns + ' [' + - connection.remote_ip + '] ' + nomatch); - } + next(DENYDISCONNECT, rdns + ' [' + + connection.remote_ip + '] ' + nomatch); } } }); From fbbba8732b7b4c753856082a438a9ac7522dbbdb Mon Sep 17 00:00:00 2001 From: Christopher Mooney Date: Tue, 4 Feb 2014 15:09:50 -0800 Subject: [PATCH 148/160] This is a fix for issue #441 This fix bubbles is_dead_sender() up to smtp_client so that it can be used in smtp_client.js, plugins/queue/smtp_forward.js, and plugins/queue/smtp_proxy.js. --- plugins/queue/smtp_forward.js | 23 +++++------------------ plugins/queue/smtp_proxy.js | 10 ++++++++++ smtp_client.js | 21 ++++++++++++++++++++- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/plugins/queue/smtp_forward.js b/plugins/queue/smtp_forward.js index e5ae8f4a5..c439a86f5 100644 --- a/plugins/queue/smtp_forward.js +++ b/plugins/queue/smtp_forward.js @@ -13,7 +13,7 @@ exports.hook_queue = function (next, connection) { smtp_client.next = next; var rcpt = 0; var send_rcpt = function () { - if (_is_dead_sender(plugin, connection, smtp_client)) { + if (smtp_client.is_dead_sender(plugin, connection)) { return; } else if (rcpt < connection.transaction.rcpt_to.length) { @@ -34,14 +34,14 @@ exports.hook_queue = function (next, connection) { } smtp_client.on('data', function () { - if (_is_dead_sender(plugin, connection, smtp_client)) { + if (smtp_client.is_dead_sender(plugin, connection)) { return; } smtp_client.start_data(connection.transaction.message_stream); }); smtp_client.on('dot', function () { - if (_is_dead_sender(plugin, connection, smtp_client)) { + if (smtp_client.is_dead_sender(plugin, connection)) { return; } else if (rcpt < connection.transaction.rcpt_to.length) { @@ -54,7 +54,7 @@ exports.hook_queue = function (next, connection) { }); smtp_client.on('rset', function () { - if (_is_dead_sender(plugin, connection, smtp_client)) { + if (smtp_client.is_dead_sender(plugin, connection)) { return; } smtp_client.send_command('MAIL', @@ -62,7 +62,7 @@ exports.hook_queue = function (next, connection) { }); smtp_client.on('bad_code', function (code, msg) { - if (_is_dead_sender(plugin, connection, smtp_client)) { + if (smtp_client.is_dead_sender(plugin, connection)) { return; } smtp_client.call_next(((code && code[0] === '5') ? DENY : DENYSOFT), @@ -72,17 +72,4 @@ exports.hook_queue = function (next, connection) { }); }; -function _is_dead_sender(plugin, connection, smtp_client) { - if (!connection.transaction) { - // This likely means the sender went away on us, cleanup. - connection.logwarn( - plugin,"transaction went away, releasing smtp_client" - ); - smtp_client.release(); - return true; - } - - return false; -} - exports.hook_queue_outbound = exports.hook_queue; diff --git a/plugins/queue/smtp_proxy.js b/plugins/queue/smtp_proxy.js index c27ee04a5..aa0139bc6 100644 --- a/plugins/queue/smtp_proxy.js +++ b/plugins/queue/smtp_proxy.js @@ -6,6 +6,7 @@ var smtp_client_mod = require('./smtp_client'); exports.hook_mail = function (next, connection, params) { + var plugin = this; var config = this.config.get('smtp_proxy.ini'); connection.loginfo(this, "proxying to " + config.main.host + ":" + config.main.port); var self = this; @@ -18,6 +19,11 @@ exports.hook_mail = function (next, connection, params) { smtp_client.on('data', smtp_client.call_next); smtp_client.on('dot', function () { + if (smtp_client.is_dead_sender(plugin, connection)) { + delete connection.notes.smtp_client; + return; + } + smtp_client.call_next(OK, smtp_client.response + ' (' + connection.transaction.uuid + ')'); smtp_client.release(); delete connection.notes.smtp_client; @@ -61,6 +67,10 @@ exports.hook_queue = function (next, connection) { var smtp_client = connection.notes.smtp_client; if (!smtp_client) return next(); smtp_client.next = next; + if (smtp_client.is_dead_sender(plugin, connection)) { + delete connection.notes.smtp_client; + return; + } smtp_client.start_data(connection.transaction.message_stream); }; diff --git a/smtp_client.js b/smtp_client.js index ba4576ab1..6319b7fb0 100644 --- a/smtp_client.js +++ b/smtp_client.js @@ -208,6 +208,19 @@ SMTPClient.prototype.destroy = function () { } }; +SMTPClient.prototype.is_dead_sender = function (plugin, connection) { + if (!connection.transaction) { + // This likely means the sender went away on us, cleanup. + connection.logwarn( + plugin, "transaction went away, releasing smtp_client" + ); + this.release(); + return true; + } + + return false; +}; + // Separate pools are kept for each set of server attributes. exports.get_pool = function (server, port, host, connect_timeout, pool_timeout, max) { var port = port || 25; @@ -365,16 +378,22 @@ exports.get_client_plugin = function (plugin, connection, config, callback) { } } else { + if (smtp_client.is_dead_sender(plugin, connection)) { + return; + } smtp_client.send_command('MAIL', 'FROM:' + connection.transaction.mail_from); } }); smtp_client.on('auth', function () { + if (smtp_client.is_dead_sender(plugin, connection)) { + return; + } smtp_client.authenticated = true; smtp_client.send_command('MAIL', 'FROM:' + connection.transaction.mail_from); - }) + }); smtp_client.on('error', function (msg) { connection.logwarn(plugin, msg); From 5cf22bedcd40db3902d0bae55363fe8c91f65856 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 5 Feb 2014 13:29:42 -0500 Subject: [PATCH 149/160] delay_deny: removed ./constants (already imported) --- plugins/delay_deny.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index bf61e9b06..c963f907b 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -6,11 +6,9 @@ ** This allows relays and authenticated users to bypass pre-DATA rejections. */ -var constants = require('./constants'); - exports.hook_deny = function (next, connection, params) { /* params - ** [0] = plugin return value (constants.deny or constants.denysoft) + ** [0] = plugin return value (DENY or DENYSOFT) ** [1] = plugin return message */ @@ -145,7 +143,7 @@ exports.hook_rcpt_ok = function (next, connection, rcpt) { } // Bypass SPF temporary failures for sender_auth - if (transaction.notes.sender_auth && params[0] === constants.denysoft && params[2] === 'spf') { + if (transaction.notes.sender_auth && params[0] === DENYSOFT && params[2] === 'spf') { connection.loginfo(plugin, 'bypassing SPF temporary failure for authorized sender'); continue; } From 67833e04df04badcf6f13a5ade6ae20fb65e7635 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 5 Feb 2014 13:32:54 -0500 Subject: [PATCH 150/160] delay_deny: fixed incorrectly specified config setting delay_deny vs delay_deny_pre --- plugins/delay_deny.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index c963f907b..fe848db47 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -57,7 +57,7 @@ exports.hook_deny = function (next, connection, params) { case 'connect': case 'ehlo': case 'helo': - if (!connection.notes.delay_deny) { + if (!connection.notes.delay_deny_pre) { connection.notes.delay_deny_pre = []; } connection.notes.delay_deny_pre.push(params); From 617bcbc1f60c7f7e1a50b5b594deeaca60ed5546 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 5 Feb 2014 13:46:20 -0500 Subject: [PATCH 151/160] delay_deny: doc cleanups, remove superfluous checks --- docs/plugins/delay_deny.md | 10 ++++++---- plugins/delay_deny.js | 17 ++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/plugins/delay_deny.md b/docs/plugins/delay_deny.md index a5cdd18f8..b0db66e17 100644 --- a/docs/plugins/delay_deny.md +++ b/docs/plugins/delay_deny.md @@ -1,14 +1,16 @@ -# delay_deny +# delay\_deny Delays all pre-DATA 'deny' results until the recipients are sent -and all post-DATA commands until all hook_data_post plugins have run. +and all post-DATA commands until all hook\_data\_post plugins have run. This allows relays and authenticated users to bypass pre-DATA rejections. ## Configuration -Configuration options are in config/delay_deny.ini. +Configuration options are in config/delay\_deny.ini. ### excluded plugins -a list of denials that are to be excluded (ie, all the immediate rejection) +A comma or semicolon separated list of denials that are to be excluded. +Excluded plugins that are not bypassed and can still immediately reject +connections. diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index fe848db47..52772200c 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -28,7 +28,7 @@ exports.hook_deny = function (next, connection, params) { var skip; if (cfg.main.excluded_plugins) { skip = cfg.main.excluded_plugins.split(/[;, ]+/); - }; + } // See if we should skip this delay if (skip && skip.length) { @@ -66,7 +66,6 @@ exports.hook_deny = function (next, connection, params) { } connection.notes.delay_deny_pre_fail[pi_name] = 1; return next(OK); - break; // Pre-DATA transaction delays case 'mail': case 'rcpt': @@ -80,7 +79,6 @@ exports.hook_deny = function (next, connection, params) { } transaction.notes.delay_deny_pre_fail[pi_name] = 1; return next(OK); - break; // Post-DATA delays case 'data': case 'data_post': @@ -100,7 +98,7 @@ exports.hook_deny = function (next, connection, params) { // No delays return next(); } -} +}; exports.hook_rcpt_ok = function (next, connection, rcpt) { var plugin = this; @@ -114,9 +112,7 @@ exports.hook_rcpt_ok = function (next, connection, rcpt) { // Apply any delayed rejections // Check connection level pre-DATA rejections first - if (connection.notes.delay_deny_pre && - connection.notes.delay_deny_pre.length > 0) - { + if (connection.notes.delay_deny_pre) { for (var i=0; i 0) - { + if (transaction.notes.delay_deny_pre) { for (var i=0; i Date: Mon, 27 Jan 2014 21:56:40 -0500 Subject: [PATCH 152/160] spamassassin: added a special 'first-recipient' option to config setting spamd_user --- config/spamassassin.ini | 1 + docs/plugins/spamassassin.md | 8 +++++++- plugins/spamassassin.js | 27 +++++++++++++++++++++++---- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/config/spamassassin.ini b/config/spamassassin.ini index cfaffebf8..7ccecebdc 100644 --- a/config/spamassassin.ini +++ b/config/spamassassin.ini @@ -4,6 +4,7 @@ spamd_socket=127.0.0.1:783 ; the username we tell spamd the message is to (default: default) +;spamd_user=first-recipient (see docs) ;spamd_user= ; messages larger than this are not scored by SA diff --git a/docs/plugins/spamassassin.md b/docs/plugins/spamassassin.md index 4591989bb..288162830 100644 --- a/docs/plugins/spamassassin.md +++ b/docs/plugins/spamassassin.md @@ -24,7 +24,13 @@ spamassassin.ini You can also pass this value in dynamically by setting: - `connection.transaction.notes.spamd_user` in another plugin. + 1. `connection.transaction.notes.spamd_user` in another plugin. + + 2. The special username: _first-recipient_. The first envelope recipient + will be used as the username. + + 3. the special username _all-recipients_ may eventually be supported. See + the get_spamd_username function in the plugin. - max\_size = N *optional* diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 5d1718c12..57975e70b 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -18,9 +18,7 @@ exports.hook_data_post = function (next, connection) { if (msg_too_big(config, connection, plugin)) return next(); - var username = config.main.spamd_user || - connection.transaction.notes.spamd_user || - 'default'; + var username = get_spamd_username(config, connection); var headers = get_spamd_headers(connection, username); var socket = get_spamd_socket(config, next, connection, plugin); socket.is_connected = false; @@ -188,6 +186,27 @@ function score_too_high(config, connection, spamd_response) { return; } +function get_spamd_username(config, connection) { + + var user = connection.transaction.notes.spamd_user; // 1st priority + if (user && user !== undefined) return user; + + if (!config.main.spamd_user) return 'default'; // when not defined + user = config.main.spamd_user; + + // Enable per-user SA prefs + if (user === 'first-recipient') { // special cases + return connection.transaction.rcpt_to[0].address(); + } + if (user === 'all-recipients') { + // TODO: pass the message through SA for each recipient. Then apply + // the least strict result to the connection. That is useful when + // one user blacklists a sender that another user wants to get mail + // from. If this is something you care about, this is the spot. + } + return user; +} + function get_spamd_headers(connection, username) { // http://svn.apache.org/repos/asf/spamassassin/trunk/spamd/PROTOCOL var headers = [ @@ -201,7 +220,7 @@ function get_spamd_headers(connection, username) { headers.push('X-Haraka-Relay: true'); } return headers; -}; +} function get_spamd_socket(config, next, connection, plugin) { // TODO: support multiple spamd backends From b92589b084e5aa43fdc25ef6f44852147fbf582b Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 5 Feb 2014 13:56:48 -0500 Subject: [PATCH 153/160] spamassassin: added "throw unimplemented" --- plugins/spamassassin.js | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/spamassassin.js b/plugins/spamassassin.js index 57975e70b..394fcd73a 100644 --- a/plugins/spamassassin.js +++ b/plugins/spamassassin.js @@ -199,6 +199,7 @@ function get_spamd_username(config, connection) { return connection.transaction.rcpt_to[0].address(); } if (user === 'all-recipients') { + throw "Unimplemented"; // TODO: pass the message through SA for each recipient. Then apply // the least strict result to the connection. That is useful when // one user blacklists a sender that another user wants to get mail From e357a35f56cd0a52e079127fbadb9e033bc18435 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Wed, 5 Feb 2014 14:22:58 -0500 Subject: [PATCH 154/160] Write config get options documentation --- config.js | 19 ++++++++++++++++--- docs/Config.md | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/config.js b/config.js index fc7ed1ad7..b839d244f 100644 --- a/config.js +++ b/config.js @@ -7,11 +7,24 @@ var config = exports; var config_path = process.env.HARAKA ? path.join(process.env.HARAKA, 'config') : path.join(__dirname, './config'); +/* Ways this can be called: +config.get('thing'); +config.get('thing', type); +config.get('thing', cb); +config.get('thing', type, cb); +config.get('thing', type, options); +config.get('thing', type, cb, options); +*/ config.get = function(name, type, cb, options) { - if (type === 'nolog') { - type = arguments[2]; // deprecated - TODO: remove later + if (typeof type == 'function') { + options = cb; + cb = type; + type = 'value'; + } + if (typeof cb != 'function') { + options = cb; + cb = null; } - type = type || 'value'; var full_path = path.resolve(config_path, name); var results = configloader.read_config(full_path, type, cb, options); diff --git a/docs/Config.md b/docs/Config.md index cb32b9023..e8ecf0f2c 100644 --- a/docs/Config.md +++ b/docs/Config.md @@ -7,7 +7,7 @@ of configuration files. The API is fairly simple: // From within a plugin: - var config_item = this.config.get(name, [type='value'], [callback]); + var config_item = this.config.get(name, [type='value'], [callback], [options]); Where type can be one of: @@ -31,6 +31,8 @@ respectively then the `type` parameter can be left off. You can optionally pass in a callback function which will be called whenever an update is detected on the file. +For ini files, an `options` object is allowed. + File Formats ============ @@ -66,9 +68,24 @@ That produces the following Javascript object: The key point there is that items before any [section] marker go in the "main" section. +Note that there is some auto-conversion of values on the right hand side of +the equals: integers are converted to integers, floats are converted to +floats. + The key=value pairs also support continuation lines using the backslash "\" character. +The `options` object allows you to specify which keys are boolean: + + { booleans: ['reject','some_true_value'] } + +This ensures these values are converted to true Javascript booleans when parsed, +and supports the following options for boolean values: + + true, yes, ok, enabled, on, 1 + +Anything else is treated as false. + Flat Files ---------- From c35b99e8bbacef29e8a85c1f0ddd87f1be2ebf63 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 7 Feb 2014 00:37:18 -0500 Subject: [PATCH 155/160] auth_results: suppress if there's no results If there's no results, don't add the A-R header. Still do rename A-R to Original-A-R --- connection.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/connection.js b/connection.js index dd53bf6af..69f34244e 100644 --- a/connection.js +++ b/connection.js @@ -1238,43 +1238,42 @@ Connection.prototype.auth_results = function(message) { var has_tran = (this.transaction && this.transaction.notes) ? true : false; // initialize connection note - if ( has_conn === false) { this.notes.authentication_results = []; }; + if (has_conn === false) { this.notes.authentication_results = []; } // initialize transaction note, if possible - if ( has_tran === true && !this.transaction.notes.authentication_results ) { + if (has_tran === true && !this.transaction.notes.authentication_results) { this.transaction.notes.authentication_results = []; } // if message, store it in the appropriate note - if ( message ) { - if ( has_tran === true ) { + if (message) { + if (has_tran === true) { this.transaction.notes.authentication_results.push(message); } else { this.notes.authentication_results.push(message); } - }; + } - // format the new header - var header = [ - config.get('me'), - (has_conn === true ? this.notes.authentication_results.join('; ') : ''), - (has_tran === true ? this.transaction.notes.authentication_results.join('; ') : '') - ].join('; '); - return header; + // assemble the new header + var header = [ config.get('me') ]; + if (has_conn === true) header.push(this.notes.authentication_results.join('; ')); + if (has_tran === true) header.push(this.transaction.notes.authentication_results.join('; ')); + if (header.length === 1) return ''; // no results + return header.join('; '); }; Connection.prototype.auth_results_clean = function(conn) { // move any existing Auth-Res headers to Original-Auth-Res headers // http://tools.ietf.org/html/draft-kucherawy-original-authres-00.html var ars = conn.transaction.header.get_all('Authentication-Results'); - if ( ars.length === 0 ) { return; }; + if (ars.length === 0) return; for (var i=0; i < ars.length; i++) { conn.transaction.header.remove_header( ars[i] ); - conn.transaction.header.add_header('Original-Authentication-Results', ars[i] ); + conn.transaction.header.add_header('Original-Authentication-Results', ars[i]); } - conn.loginfo("Authentication-Results moved to Original-Authentication-Results" ); + conn.loginfo("Authentication-Results moved to Original-Authentication-Results"); }; Connection.prototype.cmd_data = function(args) { @@ -1291,8 +1290,9 @@ Connection.prototype.cmd_data = function(args) { } this.accumulate_data('Received: ' + this.received_line() + "\r\n"); - this.auth_results_clean(this); - this.transaction.add_header('Authentication-Results', this.auth_results() ); + this.auth_results_clean(this); // rename old A-R headers + var ar_field = this.auth_results(); // assemble new one + if (ar_field) this.transaction.add_header('Authentication-Results', ar_field); plugins.run_hooks('data', this); }; From 7dab8523b98a2325948ce82281bca9427962d442 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Fri, 7 Feb 2014 11:55:50 -0500 Subject: [PATCH 156/160] connection: s/conn/this/ --- connection.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/connection.js b/connection.js index 69f34244e..b647e45ff 100644 --- a/connection.js +++ b/connection.js @@ -1263,17 +1263,17 @@ Connection.prototype.auth_results = function(message) { return header.join('; '); }; -Connection.prototype.auth_results_clean = function(conn) { +Connection.prototype.auth_results_clean = function() { // move any existing Auth-Res headers to Original-Auth-Res headers // http://tools.ietf.org/html/draft-kucherawy-original-authres-00.html - var ars = conn.transaction.header.get_all('Authentication-Results'); + var ars = this.transaction.header.get_all('Authentication-Results'); if (ars.length === 0) return; for (var i=0; i < ars.length; i++) { - conn.transaction.header.remove_header( ars[i] ); - conn.transaction.header.add_header('Original-Authentication-Results', ars[i]); + this.transaction.header.remove_header( ars[i] ); + this.transaction.header.add_header('Original-Authentication-Results', ars[i]); } - conn.loginfo("Authentication-Results moved to Original-Authentication-Results"); + this.logdebug("Authentication-Results moved to Original-Authentication-Results"); }; Connection.prototype.cmd_data = function(args) { @@ -1290,7 +1290,7 @@ Connection.prototype.cmd_data = function(args) { } this.accumulate_data('Received: ' + this.received_line() + "\r\n"); - this.auth_results_clean(this); // rename old A-R headers + this.auth_results_clean(); // rename old A-R headers var ar_field = this.auth_results(); // assemble new one if (ar_field) this.transaction.add_header('Authentication-Results', ar_field); plugins.run_hooks('data', this); From 347e75786ca9ec4488bb940eda5defbe3ba9e4ba Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Fri, 7 Feb 2014 20:35:21 +0000 Subject: [PATCH 157/160] Update TLD files --- config/extra-tlds | 73 ++++- config/three-level-tlds | 127 +++++++- config/top-level-tlds | 157 ++++++++- config/two-level-tlds | 705 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 1042 insertions(+), 20 deletions(-) diff --git a/config/extra-tlds b/config/extra-tlds index a5a789056..e9553c8b4 100644 --- a/config/extra-tlds +++ b/config/extra-tlds @@ -1,4 +1,3 @@ -# update from http://rss.uribl.com/hosters/hosters.txt 110mb.com 150m.com 163.to @@ -8,6 +7,7 @@ 9k.com addr.com altervista.org +amazonaws.com angelfire.com appspot.com asso.ws @@ -20,12 +20,45 @@ bay.livefilestore.com be.tc bg.tc biz.tm +biz.ua blog.com blog.de blog.friendster.com blog.ru blog4ever.com +blogger.ca +blogger.cf +blogger.ch +blogger.co.id +blogger.co.il +blogger.com.au +blogger.com.co +blogger.com.my +blogger.com.pe +blogger.com.ph +blogger.cv +blogger.jp +blogger.pl +blogger.re +blogger.se +blogspot.ca +blogspot.co.nz +blogspot.co.uk blogspot.com +blogspot.com.ar +blogspot.com.au +blogspot.com.br +blogspot.com.es +blogspot.cv +blogspot.de +blogspot.fr +blogspot.in +blogspot.it +blogspot.jp +blogspot.mx +blogspot.pt +blogspot.re +blogspot.se blu.livefilestore.com br.tc bravehost.com @@ -40,8 +73,8 @@ ch.pn chat.ru chez.com cjb.net -clickbank.net cloud.prohosting.com +cloudfront.net cn.pn co.at.lv co.at.pn @@ -51,6 +84,7 @@ co.cc co.kg co.uk.pn com.au.pn +com.de com.sapo.pt com.vc corp.st @@ -60,6 +94,7 @@ de.lv de.pn de.tc dk.tc +dns2go.com do.sapo.pt docs.google.com dyndns-at-home.com @@ -77,9 +112,12 @@ dyndns-web.com dyndns-wiki.com dyndns-work.com dyndns.biz +dyndns.dk dyndns.info dyndns.org dyndns.tv +dyndns.ws +emltrk.com en.alibaba.com es.pn es.tc @@ -93,6 +131,7 @@ fr.tc free.fr freehostia.com freeservercity.com +fromru.su front.ru funpic.de fx.to @@ -103,6 +142,7 @@ gigazu.net gmxhome.de go.com go.ro +gob.ar gob.ve googlegroups.com googlepages.com @@ -119,6 +159,7 @@ home.ro home.sapo.pt homeip.net homepage.t-online.de +hop.clickbank.net host.sk hostevo.com hotbox.ru @@ -132,6 +173,7 @@ id.ru idoo.com iespana.es ifrance.com +in.net int.tc interia.pl interii.pl @@ -143,23 +185,30 @@ iwebsource.com jimdo.com jino-net.ru jp.pn +jpn.com kickme.to kimsufi.com kr.tc +krovatka.su kwik.to land.ru +leadpages.net livejournal.com mail.ru mail15.su mail2k.ru mail333.su mine.nu +mobile.web.tr mooo.com multiply.com mx.tc +mydyndns.org +mygbiz.com myvnc.com na.by narod.ru +nazwa.pl neostrada.pl net.tc net.vc @@ -170,6 +219,13 @@ nextmail.ru nightmail.ru ning.com nm.ru +no-ip.biz +no-ip.ca +no-ip.co.uk +no-ip.com +no-ip.info +no-ip.net +no-ip.org no.sapo.pt no.tc notlong.com @@ -185,6 +241,7 @@ perso.tc ph.tc pisem.su pl.tc +pochta.com pochta.ru pochtamt.ru pop3.ru @@ -193,19 +250,23 @@ pro.tc proboards.com profile.live.com prserv.net +qip.ru qld.edu.au rbcmail.ru +re.it redirectme.net republika.pl rm.ru ru.tc -s3.amazonaws.com sakura.ne.jp sapo.pt se.tc selfip.com selfip.net +sendgrid.org servebbs.com +servebeer.com +serveblog.net serveftp.com shop.co shutterfly.com @@ -226,6 +287,8 @@ th.tc to.it tripod.com tumblr.com +ucoz.com +ucoz.net ucoz.ru uk.pn uk.to @@ -236,6 +299,7 @@ url.st us.pn us.tc us.to +user.icpnet.pl vv.cc we.bs web-soft.ru @@ -246,11 +310,14 @@ webcindario.com webs.com weebly.com whsites.net +wix.com wordpress.com wz.cz x.fc2.com xanga.com xf.cz xorg.pl +yolasite.com z8.ru +zapto.org zmail.ru diff --git a/config/three-level-tlds b/config/three-level-tlds index 70229acd7..d6734633b 100644 --- a/config/three-level-tlds +++ b/config/three-level-tlds @@ -1,11 +1,30 @@ -# updates from http://george.surbl.org/three-level-tlds act.edu.au act.gov.au +am.gov.br bay.livefilestore.com +best.lt.ua +best.volyn.ua blog.friendster.com +blog33.fc2.com +blogger.co.id +blogger.co.il +blogger.com.au +blogger.com.co +blogger.com.my +blogger.com.pe +blogger.com.ph +blogspot.co.il +blogspot.co.nz +blogspot.co.uk +blogspot.com.ar +blogspot.com.au +blogspot.com.br +blogspot.com.es +blogspot.com.tr blu.livefilestore.com bo.nordland.no bo.telemark.no +ceeonline.co.in city.chiba.jp city.fukuoka.jp city.hiroshima.jp @@ -28,13 +47,19 @@ co.at.pn co.at.tc co.uk.pn co.uk.tc +co.uk.tt com.au.ms com.au.pn com.au.tc +com.ne.kr com.sapo.pt do.sapo.pt docs.google.com +dothome.co.kr +elitesingles.co.uk en.alibaba.com +ge.co.it +go.gov.br groups.live.com gs.aa.no gs.ah.no @@ -59,14 +84,75 @@ gs.va.no gs.vf.no heroy.more-og-romsdal.no heroy.nordland.no +home.dyndns.org home.sapo.pt homepage.t-online.de -lkd.co.im +hop.clickbank.net +hpu.edu.cn +ig.com.br +k12.ak.us +k12.al.us +k12.ar.us +k12.as.us +k12.az.us +k12.ca.us +k12.co.us +k12.ct.us +k12.dc.us +k12.de.us +k12.fl.us +k12.ga.us +k12.gu.us +k12.ia.us +k12.id.us +k12.il.us +k12.in.us +k12.ks.us +k12.ky.us +k12.la.us +k12.ma.us +k12.md.us +k12.me.us +k12.mi.us +k12.mn.us +k12.mo.us +k12.ms.us +k12.mt.us +k12.nc.us +k12.nd.us +k12.ne.us +k12.nh.us +k12.nj.us +k12.nm.us +k12.nv.us +k12.ny.us +k12.oh.us +k12.ok.us +k12.or.us +k12.pa.us +k12.pr.us +k12.ri.us +k12.sc.us +k12.tn.us +k12.tx.us +k12.ut.us +k12.va.us +k12.vi.us +k12.vt.us +k12.wa.us +k12.wi.us +k12.wv.us +k12.wy.us +lima-city.de ltd.co.im me.uk.tc metro.tokyo.jp +mg.gov.br +mobile.web.tr nes.akershus.no nes.buskerud.no +net.work.za +no-ip.co.uk no.sapo.pt nsw.edu.au nsw.gov.au @@ -78,6 +164,7 @@ os.hedmark.no os.hordaland.no pa.gov.pl paginas.sapo.pt +perso.neuf.fr perso.sfr.fr plc.co.im po.gov.pl @@ -131,22 +218,49 @@ privat.t-online.de profile.live.com qld.edu.au qld.gov.au +qzone.qq.com +rj.gov.br +rr.gov.br +s3-ap-northeast-1.amazonaws.com +s3-ap-southeast-1.amazonaws.com +s3-ap-southeast-2.amazonaws.com +s3-eu-west-1.amazonaws.com +s3-fips-us-gov-west-1.amazonaws.com +s3-sa-east-1.amazonaws.com +s3-us-gov-west-1.amazonaws.com +s3-us-west-1.amazonaws.com +s3-us-west-2.amazonaws.com +s3-website-ap-northeast-1.amazonaws.com +s3-website-ap-southeast-1.amazonaws.com +s3-website-ap-southeast-2.amazonaws.com +s3-website-eu-west-1.amazonaws.com +s3-website-sa-east-1.amazonaws.com +s3-website-us-east-1.amazonaws.com +s3-website-us-gov-west-1.amazonaws.com +s3-website-us-west-1.amazonaws.com +s3-website-us-west-2.amazonaws.com +s3.amazonaws.com sa.edu.au sa.gov.au sakura.ne.jp sande.more-og-romsdal.no sande.vestfold.no +shop.conn.tw skydrive.live.com so.gov.pl +sp.gov.br spaces.live.com spaces.msn.com sr.gov.pl starostwo.gov.pl tas.edu.au tas.gov.au +uel.ac.uk ug.gov.pl um.gov.pl upow.gov.pl +us3.list-manage2.com +user.icpnet.pl uw.gov.pl valer.hedmark.no valer.ostfold.no @@ -157,3 +271,12 @@ wa.gov.au web.aplus.net web.fc2.com web.officelive.com +wimbp.lodz.pl +win.iu.nl +wozaonline.co.za +xn--b-5ga.nordland.no +xn--b-5ga.telemark.no +xn--hery-ira.nordland.no +xn--hery-ira.xn--mre-og-romsdal-qqb.no +xn--vler-qoa.hedmark.no +xn--vler-qoa.xn--stfold-9xa.no diff --git a/config/top-level-tlds b/config/top-level-tlds index e83af4df4..57596a77f 100644 --- a/config/top-level-tlds +++ b/config/top-level-tlds @@ -1,11 +1,12 @@ -# http://data.iana.org/TLD/tlds-alpha-by-domain.txt -# Version 2011092901, Last Updated Fri Sep 30 14:07:01 2011 UTC +# Version 2014020601, Last Updated Fri Feb 7 07:07:01 2014 UTC AC +ACADEMY AD AE AERO AF AG +AGENCY AI AL AM @@ -22,67 +23,114 @@ AW AX AZ BA +BARGAINS BB BD BE +BERLIN BF BG BH BI +BIKE BIZ BJ +BLUE BM BN BO +BOUTIQUE BR BS BT +BUILD +BUILDERS +BUZZ BV BW BY BZ CA +CAB +CAMERA +CAMP +CAREERS CAT +CATERING CC CD +CENTER +CEO CF CG CH +CHEAP CI CK CL +CLEANING +CLOTHING +CLUB CM CN CO +CODES +COFFEE COM +COMMUNITY +COMPANY +COMPUTER +CONSTRUCTION +CONTRACTORS +COOL COOP CR +CRUISES CU CV +CW CX CY CZ +DANCE +DATING DE +DEMOCRAT +DIAMONDS +DIRECTORY DJ DK DM DO +DOMAINS DZ EC EDU +EDUCATION EE EG +EMAIL +ENTERPRISES +EQUIPMENT ER ES +ESTATE ET EU +EVENTS +EXPERT +EXPOSED +FARM FI FJ FK +FLIGHTS +FLORIST FM FO FR GA +GALLERY GB GD GE @@ -90,21 +138,29 @@ GF GG GH GI +GIFT GL +GLASS GM GN GOV GP GQ GR +GRAPHICS GS GT GU +GUITARS +GURU GW GY HK HM HN +HOLDINGS +HOLIDAY +HOUSE HR HT HU @@ -112,9 +168,12 @@ ID IE IL IM +IMMOBILIEN IN INFO +INSTITUTE INT +INTERNATIONAL IO IQ IR @@ -125,10 +184,14 @@ JM JO JOBS JP +KAUFEN KE KG KH KI +KIM +KITCHEN +KIWI KM KN KP @@ -137,20 +200,28 @@ KW KY KZ LA +LAND LB LC LI +LIGHTING +LIMO +LINK LK LR LS LT LU +LUXURY LV LY MA +MANAGEMENT +MARKETING MC MD ME +MENU MG MH MIL @@ -160,6 +231,8 @@ MM MN MO MOBI +MODA +MONASH MP MQ MR @@ -173,6 +246,7 @@ MX MY MZ NA +NAGOYA NAME NC NE @@ -180,6 +254,7 @@ NET NF NG NI +NINJA NL NO NP @@ -187,62 +262,97 @@ NR NU NZ OM +ONL ORG PA +PARTNERS PE PF PG PH +PHOTO +PHOTOGRAPHY +PHOTOS +PICS +PINK PK PL +PLUMBING PM PN +POST PR PRO +PROPERTIES PS PT PW PY QA RE +RECIPES +RED +RENTALS +REPAIR +REPORT +RICH RO RS RU +RUHR RW SA SB SC SD SE +SEXY SG SH +SHIKSHA +SHOES SI +SINGLES SJ SK SL SM SN SO +SOCIAL +SOLAR +SOLUTIONS SR ST SU +SUPPORT SV +SX SY +SYSTEMS SZ +TATTOO TC TD +TECHNOLOGY TEL TF TG TH +TIENDA +TIPS TJ TK TL TM TN TO +TODAY +TOKYO +TOOLS TP TR +TRAINING TRAVEL TT TV @@ -251,62 +361,83 @@ TZ UA UG UK +UNO US UY UZ VA VC VE +VENTURES VG VI +VIAJES VN +VOTING +VOYAGE VU +WANG +WATCH +WED WF +WIEN +WORKS WS -XN--0ZWM56D -XN--11B5BS3A9AJ6G +XN--3BST00M +XN--3DS443G XN--3E0B707E XN--45BRJ9C -XN--80AKHBYKNJ4F +XN--55QW42G +XN--55QX5D +XN--6FRZ82G +XN--6QQ986B3XL +XN--80AO21A +XN--80ASEHDB +XN--80ASWG XN--90A3AC -XN--9T4B11YI5A XN--CLCHC0EA0B2G2A9GCD -XN--DEBA0AD +XN--FIQ228C5HS +XN--FIQ64B XN--FIQS8S XN--FIQZ9S XN--FPCRJ9C3D XN--FZC2C9E2C -XN--G6W251D XN--GECRJ9C XN--H2BRJ9C -XN--HGBK6AJ7F53BBA -XN--HLCJ6AYA9ESC7A +XN--IO0A7I +XN--J1AMH XN--J6W193G -XN--JXALPDLP -XN--KGBECHTV XN--KPRW13D XN--KPRY57D +XN--L1ACC XN--LGBBAT1AD8J +XN--MGB9AWBF +XN--MGBA3A4F16A XN--MGBAAM7A8H XN--MGBAYH7GPA XN--MGBBH1A71E XN--MGBC0A9AZCG XN--MGBERP4A5D4AR +XN--MGBX4CD0AB +XN--NGBC5AZD XN--O3CW4H XN--OGBPF8FL XN--P1AI XN--PGBS0DH +XN--Q9JYB4C XN--S9BRJ9C +XN--UNUP4Y XN--WGBH1C XN--WGBL6A XN--XKC2AL3HYE2A XN--XKC2DL3A5EE0H XN--YFRO4I67O XN--YGBI2AMMX -XN--ZCKZAH +XN--ZFR164B XXX YE YT ZA ZM +ZONE ZW diff --git a/config/two-level-tlds b/config/two-level-tlds index 13a4beb55..92774e5f8 100644 --- a/config/two-level-tlds +++ b/config/two-level-tlds @@ -1,21 +1,48 @@ -# update from http://george.surbl.org/two-level-tlds 0.bg +0fees.net 1.bg +100megsfree5.com 110mb.com +123oferty.pl 150m.com 163.to +1accesshost.com 1blu.de +1dumb.com +1gb.ru +1x.com 2.bg 2000.hu +20fr.com +22web.org +24hr.com +25u.com +2waky.com +3-a.net 3.bg +3dn.ru +3dxtras.com +3gb.biz 4.bg +4dq.com +4mydomain.com +4pu.com +4u.com 5.bg +50webs.com +5ballov.ru +5gbfree.com +5u.com 6.bg 6a.org 6bone.pl +6te.net 7.bg +70948.com 8.bg 9.bg +96.lt +9966.org 9k.com a.bg a.se @@ -78,11 +105,13 @@ aca.pro academy.museum accident-investigation.aero accident-prevention.aero +acmetoy.com act.au ad.jp addr.com adm.br adult.ht +adultdns.net adv.br adygeya.ru ae.org @@ -131,6 +160,8 @@ alderney.gg alessandria.it alesund.no algard.no +allowed.org +almostmy.com alstahaug.no alt.na alt.za @@ -155,6 +186,7 @@ amur.ru amursk.ru amusement.aero an.it +anahuac.mx ancona.it and.museum andasuolo.no @@ -172,12 +204,14 @@ aoste.it ap.it appspot.com aq.it +aqserv.com aquarium.museum aquila.it ar.com ar.it ar.us arboretum.museum +arcadepages.com archaeological.museum archaeology.museum architecture.museum @@ -189,6 +223,7 @@ arkhangelsk.ru army.mil arna.no arq.br +arredemo.org art.br art.do art.dz @@ -200,6 +235,7 @@ artcenter.museum artdeco.museum arteducation.museum artgallery.museum +artit.com arts.co arts.museum arts.nf @@ -210,6 +246,7 @@ as.us ascoli-piceno.it ascolipiceno.it aseral.no +askadresi.net asker.no askim.no askoy.no @@ -233,6 +270,7 @@ asso.km asso.ma asso.mc asso.re +asso.st asso.ws association.aero association.museum @@ -245,10 +283,12 @@ at.pn at.tc at.tf at.tt +at.ua ath.cx atlanta.museum atm.pl ato.br +atwebpages.com au.com au.ms au.pn @@ -265,6 +305,8 @@ austin.museum australia.museum austrheim.no author.aero +authorizeddns.net +authorizeddns.us auto.pl automotive.museum av.it @@ -274,12 +316,14 @@ averoy.no aviation.museum avocat.fr avoues.fr +ax.lt axis.museum az.us b.bg b.se ba.it babia-gora.pl +backdrop.jp badaddja.no badajoz.museum baghdad.museum @@ -321,6 +365,7 @@ bearalvahki.no beardu.no beauxarts.museum bedzin.pl +bee.pl beeldengeluid.museum beiarn.no bel.tr @@ -338,6 +383,7 @@ berkeley.museum berlevag.no berlin.museum bern.museum +besaba.com beskidy.pl bg.it bg.tc @@ -351,6 +397,8 @@ bielawa.pl biella.it bieszczady.pl bievat.no +bigheadhosting.net +bij.pl bilbao.museum bill.museum bindal.no @@ -368,11 +416,13 @@ biz.ki biz.ly biz.mv biz.mw +biz.nf biz.nr biz.om biz.pk biz.pl biz.pr +biz.st biz.tj biz.tm biz.tr @@ -383,13 +433,50 @@ bj.cn bjarkoy.no bjerkreim.no bjugn.no +bl.ee bl.it bl.uk blog.br +blog.cat blog.com blog.ru blog4ever.com +blogger.ca +blogger.cf +blogger.ch +blogger.cv +blogger.jp +blogger.pl +blogger.re +blogger.se +blogspot.ae +blogspot.be +blogspot.ca +blogspot.ch +blogspot.co blogspot.com +blogspot.cv +blogspot.cz +blogspot.de +blogspot.fi +blogspot.fr +blogspot.gr +blogspot.hu +blogspot.ie +blogspot.in +blogspot.it +blogspot.jp +blogspot.kr +blogspot.mx +blogspot.nl +blogspot.pt +blogspot.re +blogspot.ro +blogspot.ru +blogspot.se +blogspot.sg +blogspot.sk +blueberrywave.com bmd.br bn.it bo.it @@ -402,6 +489,7 @@ bolzano.it bomlo.no bonn.museum boston.museum +bot.nu botanical.museum botanicalgarden.museum botanicgarden.museum @@ -417,6 +505,7 @@ brandywinevalley.museum brasil.museum bravehost.com bravepages.com +bravesites.com bremanger.no brescia.it brindisi.it @@ -438,7 +527,9 @@ bryne.no bs.it bu.no budejju.no +bugs3.com building.museum +builtfree.org burghof.museum buryatia.ru bus.museum @@ -446,6 +537,26 @@ busan.kr bushey.museum by.ru bydgoszcz.pl +byethost.com +byethost1.com +byethost10.com +byethost11.com +byethost12.com +byethost13.com +byethost14.com +byethost15.com +byethost16.com +byethost17.com +byethost18.com +byethost2.com +byethost24.com +byethost3.com +byethost4.com +byethost5.com +byethost6.com +byethost7.com +byethost8.com +byethost9.com bygland.no bykle.no bytom.pl @@ -454,6 +565,7 @@ bz.it c.bg c.la c.se +ca.im ca.it ca.na ca.pn @@ -471,6 +583,7 @@ campobasso.it can.br can.museum canada.museum +canywhere.net capebreton.museum cargo.aero carrier.museum @@ -484,10 +597,12 @@ catania.it catanzaro.it catering.aero cb.it +cba.pl cbg.ru cc.bh cc.cc cc.na +cccampaigns.net cci.fr ce.it ce.ms @@ -499,9 +614,12 @@ ch.lv ch.pn ch.tc ch.tf +ch.ua ch.vu chambagri.fr championship.aero +changeip.net +changeip.org charter.aero chat.ru chattanooga.museum @@ -512,12 +630,15 @@ cherkassy.ua chernigov.ua chernovtsy.ua chesapeakebay.museum +chez.com chiba.jp chicago.museum +chickenkiller.com chieti.it children.museum childrens.museum childrensgarden.museum +chips.jp chiropractic.museum chirurgiens-dentistes.fr chita.ru @@ -528,6 +649,7 @@ chungbuk.kr chungnam.kr chuvashia.ru cieszyn.pl +ciki.me cim.br cincinnati.museum cinema.museum @@ -541,13 +663,20 @@ civilwar.museum cjb.net ck.ua cl.it +clan.su +cleansite.us +click.org clickbank.net clinton.museum clock.museum +cloudaccess.net club.aero club.tw +cmetoy.com cmw.ru +cn.co cn.com +cn.im cn.it cn.ms cn.pn @@ -566,8 +695,10 @@ co.bw co.cc co.ci co.ck +co.cm co.cr co.cu +co.de co.dk co.ee co.fk @@ -589,10 +720,14 @@ co.lc co.ls co.ma co.me +co.mp co.mu co.mw co.mz co.na +co.nf +co.nr +co.nu co.nz co.om co.pn @@ -615,6 +750,7 @@ co.us co.uz co.ve co.vi +co.vu co.yu co.za co.zm @@ -656,10 +792,12 @@ com.bz com.cd com.ch com.ci +com.cm com.cn com.co com.cu com.cy +com.de com.dm com.do com.dz @@ -724,6 +862,7 @@ com.ng com.ni com.np com.nr +com.nu com.om com.pa com.pe @@ -769,12 +908,16 @@ com.vn com.vu com.ws com.ye +com.zm communication.museum communications.museum community.museum como.it +compress.to computer.museum computerhistory.museum +comuf.com +comyr.com conf.au conf.lv conference.aero @@ -784,8 +927,10 @@ consultant.aero consulting.aero contemporary.museum contemporaryart.museum +continent.kz control.aero convent.museum +coolpage.biz coop.br coop.ht coop.km @@ -801,10 +946,13 @@ costume.museum council.aero countryestate.museum county.museum +cp.cx cpa.pro cq.cn cr.it +crabdance.com crafts.museum +craftx.biz cranbrook.museum creation.museum cremona.it @@ -812,18 +960,19 @@ crew.aero cri.nz crimea.ua crotone.it +cry.com cs.it csiro.au ct.it ct.us cu.cc +cuccfree.com cul.na cultural.museum culturalcenter.museum culture.museum cuneo.it cv.ua -cw.cm cx.cc cyber.museum cymru.museum @@ -832,6 +981,7 @@ cz.cc cz.it cz.tc cz.tf +cz.tl czeladz.pl czest.pl d.bg @@ -841,11 +991,22 @@ daejeon.kr dagestan.ru dali.museum dallas.museum +dalnet.ca +dasfree.com database.museum davvenjarga.no davvesiida.no dc.us +ddns.info +ddns.me +ddns.me.uk +ddns.ms +ddns.name +ddns.net +ddns.us +ddns01.com ddr.museum +de.be de.com de.ki de.lv @@ -854,11 +1015,13 @@ de.net de.pn de.tc de.tf +de.tl de.tt de.us de.vu deatnu.no decorativearts.museum +dedibox.fr defense.tn delaware.museum delmenhorst.museum @@ -870,32 +1033,90 @@ design.museum detroit.museum dgca.aero dielddanuorri.no +digitalzones.com dinosaur.museum discovery.museum +diskstation.me divtasvuodna.no divttasvuotna.no dk.org dk.tc dk.tt dlugoleka.pl +dmdelivery.net dn.ua dnepropetrovsk.ua dni.us +dns-dns.com +dns-stuff.com dns.be +dns.biz +dns.info +dns.mobi +dns.ms +dns.name +dns.us +dns04.com +dns05.com +dns1.us +dns53.biz +dnsapi.info +dnsd.info +dnsd.me +dnsdynamic.com +dnsdynamic.net +dnset.com +dnsfor.me +dnsget.org +dnsrd.com +do.am dolls.museum donetsk.ua donna.no donostia.museum +dontexist.com dovre.no dp.ua dpn.br +dq.com dr.na dr.tr drammen.no drangedal.no +dreamhosters.com drobak.no +dsmtp.com dudinka.ru +dumb.com +dumb1.com durham.museum +dynalias.org +dynamic-dns.net +dynamicdns.me.uk +dynamicdns.org.uk +dyndns-at-home.com +dyndns-at-work.com +dyndns-blog.com +dyndns-free.com +dyndns-home.com +dyndns-ip.com +dyndns-mail.com +dyndns-office.com +dyndns-pics.com +dyndns-remote.com +dyndns-server.com +dyndns-web.com +dyndns-wiki.com +dyndns-work.com +dyndns.biz +dyndns.dk +dyndns.info +dyndns.org +dyndns.pro +dyndns.tv +dyndns.ws +dynet.com +dynssl.com dyroy.no e-burg.ru e-famoso.it @@ -905,13 +1126,17 @@ e12.ve e164.arpa eastafrica.museum eastcoast.museum +eb2a.com +ebatesrule.net ebiz.tw ecn.br +eco.br ed.ao ed.ci ed.cr ed.jp ed.pw +edenvale.info edu.ac edu.af edu.ai @@ -987,6 +1212,7 @@ edu.ml edu.mm edu.mn edu.mo +edu.mp edu.mt edu.mv edu.mw @@ -1043,11 +1269,13 @@ edu.ws edu.ye edu.yu edu.za +edu.zm educ.ar education.museum educational.museum educator.aero edunet.tn +efound.com egersund.no egyptian.museum ehime.jp @@ -1062,6 +1290,7 @@ ekloges.cy elblag.pl elburg.museum elk.pl +ellclassics.com elvendrell.museum elverum.no embaixada.st @@ -1069,6 +1298,8 @@ embroidery.museum emergency.aero en.it encyclopedic.museum +endofinternet.net +endsmtp.com enebakk.no eng.br eng.pro @@ -1082,11 +1313,15 @@ entertainment.aero entomology.museum environment.museum environmentalconservation.museum +envy.nu +epac.to epilepsy.museum equipment.aero ernet.in erotica.hu erotika.hu +erveuser.com +erveusers.com es.kr es.pn es.tc @@ -1096,11 +1331,15 @@ esp.br essex.museum est.pr estate.museum +esy.es etc.br ethnology.museum eti.br etne.no etnedal.no +etos.com +etradesystem.de +ettrials.com eu.com eu.im eu.int @@ -1108,21 +1347,31 @@ eu.org eu.tc eu.tf eu.tt +eu5.org eun.eg euro.tm evenassi.no evenes.no evje-og-hornnes.no +exactpages.com exchange.aero exeter.museum exhibition.museum +exidude.com experts-comptables.fr express.aero extra.hu +exvm.com +exvn.com +exxxy.biz +ez.lv +ezua.com f.bg f.se +fagms.net fam.pk family.museum +faqserv.com far.br fareast.ru farm.museum @@ -1130,10 +1379,14 @@ farmequipment.museum farmers.museum farmstead.museum farsund.no +fartit.com fauske.no fax.nr fc.it +fc2.com +fcpages.com fe.it +fe100.net fed.us federation.aero fedje.no @@ -1176,10 +1429,15 @@ fl.us fla.no flakstad.no flanders.museum +flashserv.net flatanger.no flekkefjord.no flesberg.no flight.aero +flink.com +flinkup.com +flinkup.net +flinkup.org flog.br flora.no florence.it @@ -1202,6 +1460,7 @@ forum.hu fosnes.no fot.br foundation.museum +fr.cr fr.it fr.ms fr.nf @@ -1215,9 +1474,21 @@ frankfurt.museum franziskaner.museum fredrikstad.no free.fr +freeddns.com +freehomepage.asia freehostia.com +freehostyou.com +freeiz.com freemasonry.museum +freeoda.co +freeoda.com freeservercity.com +freestuff.eu +freetcp.com +freetzi.com +freevar.com +freevnn.com +freezoy.com frei.no freiburg.museum freight.aero @@ -1226,11 +1497,17 @@ frog.museum frogn.no froland.no from.hr +fromru.su front.ru frosinone.it frosta.no froya.no fst.br +ftp.sh +ftp1.biz +ftp21.net +ftpaccess.cc +ftpserver.biz fuel.aero fukui.jp fukuoka.jp @@ -1251,6 +1528,7 @@ ga.us gaivuotna.no gallery.museum galsa.no +game-server.cc game.tw games.hu gamvik.no @@ -1319,9 +1597,12 @@ go.th go.tj go.tz go.ug +go2cloud.org +gob.ar gob.bo gob.cl gob.do +gob.ec gob.es gob.gt gob.hn @@ -1336,6 +1617,7 @@ gobiernoelectronico.ar gok.pk gol.no gon.pk +googlecode.com googlegroups.com googlepages.com gop.pk @@ -1343,6 +1625,7 @@ gorge.museum gorizia.it gorlice.pl gos.pk +got-game.org gouv.ci gouv.fr gouv.ht @@ -1365,6 +1648,7 @@ gov.bd gov.bf gov.bh gov.bm +gov.bn gov.bo gov.br gov.bs @@ -1439,6 +1723,7 @@ gov.ml gov.mm gov.mn gov.mo +gov.mp gov.mr gov.mt gov.mu @@ -1495,8 +1780,10 @@ gov.zw government.aero government.pn govt.nz +gr.com gr.it gr.jp +gr8name.biz grajewo.pl gran.no grandrapids.museum @@ -1504,6 +1791,7 @@ grane.no granvin.no gratangen.no gratishost.com +gratisphphost.info graz.museum greta.fr grimstad.no @@ -1517,6 +1805,7 @@ grp.lk grue.no gs.cn gsm.pl +gto.com gu.us gub.uy guernsey.gg @@ -1536,19 +1825,25 @@ gz.cn h.bg h.se ha.cn +ha.la ha.no habmer.no +hacked.jp hadsel.no hagebostad.no halden.no halloffame.museum halsa.no +ham-radio-op.net hamar.no hamaroy.no hamburg.museum hammarfeasta.no hammerfest.no handson.museum +hangeip.name +hangeip.net +hangeip.org hanggliding.aero hapmir.no haram.no @@ -1560,6 +1855,8 @@ hattfjelldal.no haugesund.no hawaii.museum hb.cn +hc0.me +hcp.biz he.cn health.museum health.vn @@ -1575,6 +1872,7 @@ herad.no heritage.museum hi.cn hi.us +high.com hiroshima.jp histoire.museum historical.museum @@ -1590,10 +1888,13 @@ hjelmeland.no hk.cn hk.ms hk.tc +hk.vg hl.cn hl.no hm.no hn.cn +hobby-site.com +hobby-site.org hobol.no hof.no hokkaido.jp @@ -1602,15 +1903,25 @@ hol.no hole.no holmestrand.no holtalen.no +home.kg home.pl home.ro homebuilt.aero +homedns.org +homeftp.org homeip.net +homelinux.com +homenet.org +homeunix.net honefoss.no +hopto.me hornindal.no horology.museum horten.no +hospedandofacil.info host.sk +host22.com +host56.com hostevo.com hotbox.ru hotel.hu @@ -1620,6 +1931,9 @@ house.museum hoyanger.no hoylandet.no hs.kr +http01.com +http80.info +https443.com hu.com hu.tc hu2.ru @@ -1638,6 +1952,7 @@ ia.us ibaraki.jp ibelgique.com ibestad.no +ibiz.cc iblogger.org ic.cz icnet.uk @@ -1658,16 +1973,22 @@ ie.tc iespana.es if.ua ifrance.com +igmoney.biz +ignorelist.com iim.bz +ikaba.com +ikwb.com il.im il.us ilawa.pl illustration.museum im.it imageandsound.museum +imap01.com imb.br imperia.it in.na +in.net in.rs in.th in.ua @@ -1711,14 +2032,18 @@ info.pl info.pr info.ro info.sd +info.tm info.tn info.tr info.tt info.ve info.vn +infos.st ing.pa ingatlan.hu inima.al +ino-ip.me +instanthq.com insurance.aero int.am int.ar @@ -1744,6 +2069,7 @@ intelligence.museum interactive.museum interia.pl interii.pl +inth.biz intl.tn ip6.arpa iquebec.com @@ -1752,8 +2078,18 @@ irc.pl iris.arpa irkutsk.ru iron.museum +is-a-chef.com +is-a-chef.net +is-a-chef.org +is-a-geek.com +is-a-geek.net +is-a-geek.org is.it +isa-geek.com +isa-geek.net +isa-geek.org isa.us +isecure.com isernia.it ishikawa.jp isla.pr @@ -1763,6 +2099,7 @@ it.ao it.pn it.tc it.tt +itemdb.com its.me ivano-frankivsk.ua ivanovo.ru @@ -1771,8 +2108,10 @@ ivgu.no iwate.jp iwebsource.com iwi.nz +ixth.biz iz.hr izhevsk.ru +izvaz.com j.bg jamal.ru jamison.museum @@ -1788,14 +2127,18 @@ jersey.je jerusalem.museum jessheim.no jet.uk +jetos.com jevnaker.no jewelry.museum jewish.museum jewishart.museum jfk.museum jgora.pl +jimdo.co jimdo.com jino-net.ru +jino.ru +jkub.com jl.cn jobs.tt jogasz.hu @@ -1807,6 +2150,7 @@ joshkar-ola.ru journal.aero journalism.museum journalist.aero +jp.net jp.pn jpn.com js.cn @@ -1814,6 +2158,8 @@ judaica.museum judygarland.museum juedisches.museum juif.museum +jumpingcrab.com +jungleheart.com jur.pro jus.br jx.cn @@ -1824,6 +2170,7 @@ k12.ec k12.il k12.tr k12.vi +kadm5.com kafjord.no kagawa.jp kagoshima.jp @@ -1861,6 +2208,7 @@ kherson.ua khmelnitskiy.ua khv.ru kickme.to +kicks-ass.org kids.museum kids.us kiev.ua @@ -1905,8 +2253,10 @@ kristiansand.no kristiansund.no krodsherad.no krokstadelva.no +krovatka.su ks.ua ks.us +kub.com kuban.ru kumamoto.jp kunst.museum @@ -1926,6 +2276,7 @@ kvinesdal.no kvinnherad.no kviteseid.no kvitsoy.no +kwb.com kwik.to ky.us kyonggi.kr @@ -1940,6 +2291,7 @@ labour.museum lahppi.no lajolla.museum lakas.hu +lamer.la lanarb.se lanbib.se lancashire.museum @@ -1961,6 +2313,9 @@ law.za lc.it le.it leangaviika.no +leansite.biz +leansite.info +leansite.us leasing.aero lebesby.no lebork.pl @@ -1979,6 +2334,9 @@ lesja.no levanger.no lewismiller.museum lezajsk.pl +lflink.com +lflinkup.com +lflinkup.org lg.jp lg.ua li.it @@ -1987,17 +2345,20 @@ lier.no lierne.no lillehammer.no lillesand.no +lima-city.de limanowa.pl limewebs.com lincoln.museum lindas.no lindesnes.no +linkbucks.com linz.museum lipetsk.ru livejournal.com living.museum livinghistory.museum livorno.it +lmostmy.com ln.cn lo.it loabat.no @@ -2005,10 +2366,14 @@ localhistory.museum lodi.it lodingen.no lodz.pl +loginto.me logistics.aero +lolipop.jp lom.no lomza.pl london.museum +longmusic.com +lookseekpages.com loppa.no lorenskog.no losangeles.museum @@ -2017,6 +2382,7 @@ louvre.museum lowicz.pl loyalist.museum lt.it +lt.ua ltd.cy ltd.gg ltd.gi @@ -2045,6 +2411,7 @@ m.se ma.us macerata.it mad.museum +maddsites.com madrid.museum magadan.ru magazine.aero @@ -2118,14 +2485,17 @@ media.pl medical.museum medizinhistorisches.museum meeres.museum +mefound.com meland.no meldal.no melhus.no meloy.no memorial.museum meraker.no +mericanunfinished.com mesaverde.museum messina.it +meximas.com mi.it mi.th mi.us @@ -2242,12 +2612,14 @@ modum.no molde.no moma.museum money.museum +moneyhome.biz monmouth.museum monticello.museum montreal.museum monza.it mooo.com mordovia.ru +more-loto.mobi moscow.museum mosjoen.no moskenes.no @@ -2255,16 +2627,21 @@ mosreg.ru moss.no mosvik.no motorcycle.museum +moy.su mr.no mragowo.pl +mrbasic.com +mrslove.com ms.it ms.kr ms.us +msgfocus.com msk.ru mt.it mt.us muenchen.museum muenster.museum +muf.mobi mulhouse.museum muncie.museum muni.il @@ -2284,8 +2661,29 @@ music.mobi music.museum mx.na mx.tc +my03.com +my1.ru +my3gb.com +myactivedirectory.com +myddns.com +mydomain.com +mydyndns.org +myeffect.net +myftp.biz +myftp.info +myftp.name +myftp.org +mygbiz.com +mylftv.com +mynetav.net +mynetav.org +mynumber.org +mypicture.info +mysq1.net mytis.ru myvnc.com +myvnc.org +myz.info n.bg n.se na.by @@ -2344,6 +2742,7 @@ naval.museum navigation.aero navuotna.no navy.mil +nazwa.pl nb.ca nc.us nd.us @@ -2356,6 +2755,7 @@ ne.ug ne.us nebraska.museum nedre-eiker.no +nedumb.com nel.uk neostrada.pl nesna.no @@ -2389,6 +2789,7 @@ net.cd net.ch net.ci net.ck +net.cm net.cn net.co net.cu @@ -2447,6 +2848,7 @@ net.mk net.ml net.mm net.mo +net.mp net.ms net.mt net.mu @@ -2456,6 +2858,7 @@ net.mx net.my net.na net.nc +net.net net.nf net.ng net.ni @@ -2509,9 +2912,16 @@ net.vu net.ws net.ye net.za +net.zm +net63.net +netatlantic.com +netau.net netfirms.com +netii.net +netne.net netsolhost.com neues.museum +neuf.fr new.ke newhampshire.museum newjersey.museum @@ -2528,6 +2938,7 @@ ngo.ph ngo.pl ngo.za nh.us +nhlfan.net nhs.uk nic.ar nic.im @@ -2550,10 +2961,24 @@ nls.uk nm.cn nm.ru nm.us +nmypc.biz +nmypc.info +nmypc.net +nmypc.org +nmypc.us nnov.ru +no-ip.biz +no-ip.ca +no-ip.com +no-ip.info +no-ip.net +no-ip.org no.com no.it no.tc +noads.biz +noip.me +noip.us nom.ad nom.ag nom.br @@ -2595,14 +3020,34 @@ novara.it novosibirsk.ru nowaruda.pl nrw.museum +ns-dns.com +ns-stuff.com ns.ca +ns0.it +ns01.biz +ns01.info +ns01.us +ns02.biz +ns02.info +ns02.us +ns04.com +ns05.com +ns1.name +ns1.us +ns2.us +ns3.name +ns360.info +nset.com nsk.ru nsn.us +nsrd.com +nstanthq.com nsw.au nt.au nt.ca nt.no nt.ro +ntdll.net ntr.br nu.ca nu.it @@ -2615,18 +3060,23 @@ nx.cn ny.us nyc.museum nyny.museum +nyp.com nysa.pl o.bg o.se oceanographic.museum oceanographique.museum +ocry.com od.ua odda.no +odesa.ua odessa.ua odo.br of.no off.ai +office-on-the.net og.ao +oh.info oh.us oita.jp ok.us @@ -2635,21 +3085,34 @@ okinawa.jp oksnes.no ol.no olawa.pl +ole32.com olecko.pl olkusz.pl olsztyn.pl omaha.museum omasvuotna.no +ompress.to omsk.ru on.ca one.pl +onedumb.com +oneyhome.biz +ongmusic.com +onion.to +onlc.fr online.museum +onlineslotsmania.com +onmbl.com +onmypc.net +onmypc.org ontario.museum +ontraport.net openair.museum opoczno.pl opole.pl oppdal.no oppegard.no +oppop.com or.at or.bi or.ci @@ -2762,6 +3225,7 @@ org.ml org.mm org.mn org.mo +org.mp org.mt org.mu org.mv @@ -2827,13 +3291,18 @@ org.yu org.za org.zm org.zw +orge.pl +orgfree.com oristano.it orkanger.no orkdal.no orland.no orskog.no orsta.no +ort25.biz +ortrelay.com oryol.ru +osa.pl osaka.jp osen.no oskol.ru @@ -2845,21 +3314,30 @@ ostroda.pl ostroleka.pl ostrowiec.pl ostrowwlkp.pl +ot-game.org otago.museum otc.au other.nf +oudontcare.com +ourhobby.com +ourtrap.com overhalla.no ovh.net ovre-eiker.no +ownyour.biz +ownyour.org oxford.museum oyer.no oygarden.no oystre-slidre.no +oythieves.com oz.au p.bg +p.ht p.se pa.it pa.us +pac.to pacific.museum paderborn.museum padova.it @@ -2884,10 +3362,12 @@ pavia.it pb.ao pc.it pc.pl +pcanywhere.net pd.it pe.ca pe.it pe.kr +pen.io penza.ru per.kh per.la @@ -2921,6 +3401,7 @@ pisem.su pistoia.it pisz.pl pittsburgh.museum +pl.am pl.tc pl.tf pl.ua @@ -2933,10 +3414,14 @@ plc.uk plo.ps pn.it po.it +pochta.com pochta.ru pochtamt.ru podhale.pl podlasie.pl +podzone.org +poe.com +pointto.us pol.dz pol.ht pol.tr @@ -2946,11 +3431,14 @@ poltava.ua pomorskie.pl pomorze.pl pop3.ru +populus.ch +populus.org pordenone.it porsanger.no porsangu.no porsgrunn.no port.fr +port25.biz portal.museum portland.museum portlligat.museum @@ -2986,6 +3474,8 @@ priv.hu priv.me priv.no priv.pl +privatedns.org +privatizehealthinsurance.net pro.ae pro.az pro.br @@ -3006,6 +3496,8 @@ production.aero prof.pr project.museum promocion.ar +proxy8080.com +proxydns.com prserv.net pruszkow.pl przeworsk.pl @@ -3014,6 +3506,7 @@ psi.br pskov.ru pt.it ptz.ru +pu.com pu.it pub.sa publ.pt @@ -3022,17 +3515,28 @@ pubol.museum pulawy.pl pv.it pvt.ge +pwnz.org pyatigorsk.ru pz.it q.bg qc.ca qc.com +qc.to +qdedu.net qh.cn +qhigh.com qld.au +qpoe.com qsl.br quebec.museum +quicksytes.com +quirly.info r.bg +r.im r.se +r00t.la +r8domain.biz +r8name.biz ra.it rade.no radikal.ru @@ -3051,11 +3555,14 @@ randaberg.no rauma.no ravenna.it rawa-maz.pl +rbasic.com rbcmail.ru +rbonus.com rc.it re.it re.kr realestate.pl +rebatesrule.net rec.br rec.co rec.nf @@ -3064,6 +3571,10 @@ rec.ve recreation.aero red.sv redirectme.net +reeddns.com +reetcp.com +reewww.biz +reewww.info reggio-calabria.it reggio-emilia.it reggiocalabria.it @@ -3082,9 +3593,13 @@ research.aero research.museum resistance.museum retina.ar +rface.com rg.it +rganiccrap.com ri.it ri.us +rickip.net +rickip.org rieti.it riik.ee rimini.it @@ -3105,6 +3620,7 @@ rns.tn rnu.tn ro.im ro.it +ro.lt roan.no rochester.museum rockart.museum @@ -3120,10 +3636,13 @@ rost.no rotorcraft.aero rovigo.it rovno.ua +roxydns.com royken.no royrvik.no rs.ba +rslove.com ru.com +ru.im ru.tc ru.tf rubtsovsk.ru @@ -3136,6 +3655,17 @@ rygge.no rzeszow.pl s.bg s.se +s01.biz +s01.info +s01.us +s02.biz +s02.info +s02.us +s1.name +s2.name +s3.amazonaws.com +s3.name +s37.it sa.au sa.com sa.cr @@ -3170,6 +3700,7 @@ sapporo.jp saratov.ru sark.gg sarpsborg.no +sasecret.com saskatchewan.museum sassari.it satx.museum @@ -3217,8 +3748,10 @@ sciences.museum sciencesnaturelles.museum scientist.aero scotland.museum +scrapping.cc sd.cn sd.us +sddns.info se.com se.net se.tc @@ -3229,26 +3762,41 @@ sec.ps sejny.pl sel.no selbu.no +selfip.biz selfip.com +selfip.info selfip.net selje.no seljord.no +sellclassics.com sendai.jp +sendsmtp.com seoul.kr servebbs.com +servebeer.com +serveblog.net serveftp.com +servegame.com +servehttp.com +servesarcasm.com +servetown.com +serveuser.com services.aero settlement.museum settlers.museum sex.hu sex.pl +sexidude.com +sexxxy.biz sf.no sg.tf sh.cn +sharepoint.com shell.museum sherbrooke.museum shiga.jp shimane.jp +shit.la shizuoka.jp shop.co shop.ht @@ -3256,6 +3804,7 @@ shop.hu shop.ms shop.pl shop.tc +shop.tm show.aero shutterfly.com si.it @@ -3269,6 +3818,11 @@ simbirsk.ru siracusa.it sirdal.no site.tc +site40.net +site90.net +sitebr.net +sivit.org +sixth.biz sk.ca skanit.no skanland.no @@ -3287,6 +3841,7 @@ skoczow.pl skodje.no skole.museum skydiving.aero +sl443.org slask.pl slattum.no sld.cu @@ -3294,8 +3849,11 @@ sld.do sld.pa slg.br slupsk.pl +sm.ua smola.no smolensk.ru +smtp.biz +smtp.com smtp.ru sn.cn snaase.no @@ -3336,16 +3894,21 @@ soundandvision.museum southcarolina.museum southwest.museum sp.it +sp.ru space.museum spb.ru spjelkavik.no sport.hu spy.museum spydeberg.no +sql01.com square.museum +squirly.info sr.it srv.br ss.it +ssh01.com +ssh22.net sshn.se st.no stadt.museum @@ -3384,10 +3947,13 @@ storfjord.no stpetersburg.museum strand.no stranda.no +strangled.net stryn.no student.aero +stufftoread.com stuttgart.museum stv.ru +sub.es suedtirol.it suisse.museum sula.no @@ -3413,6 +3979,7 @@ swinoujscie.pl sx.cn sydney.museum sykkylven.no +sytes.net syzran.ru szczecin.pl szczytno.pl @@ -3421,6 +3988,8 @@ szkola.pl szm.com t.bg t.se +t15.org +t28.net t3.to t35.com t35.me @@ -3430,6 +3999,7 @@ takamatsu.jp tambov.ru tana.no tananger.no +tank.jp tank.museum taranto.it targi.pl @@ -3446,16 +4016,21 @@ tel.no tel.nr tel.tr telecom.na +telecomitalia.it telekommunikation.museum telememo.au television.museum +temdb.com +tempors.com teramo.it terni.it ternopil.ua test.ru texas.museum textile.museum +tftpd.net tgory.pl +th.ht th.tc theater.museum time.museum @@ -3488,6 +4063,7 @@ tn.it tn.us to.it tochigi.jp +toh.info tokke.no tokushima.jp tokyo.jp @@ -3495,7 +4071,9 @@ tolga.no tom.ru tomsk.ru tonsberg.no +topica.com topology.museum +tor2web.org torino.it torino.museum torsken.no @@ -3506,8 +4084,11 @@ tourism.pl tourism.tn town.museum toyama.jp +toythieves.com tozsde.hu tp.it +tp1.biz +tpserver.biz tr.it tr.no trader.aero @@ -3525,6 +4106,8 @@ tree.museum trentino.it trento.it treviso.it +trickip.net +trickip.org trieste.it tripod.com troandin.no @@ -3537,10 +4120,16 @@ trust.museum trustee.museum trysil.no ts.it +tsaol.com tsaritsyn.ru tsk.ru +ttl60.com +ttl60.org +ttps443.net +ttps443.org tula.ru tumblr.com +tur.ar tur.br tur.cu turek.pl @@ -3556,6 +4145,8 @@ tv.sd tvedestrand.no tver.ru tw.cn +twilightparadox.com +twomini.com tx.us tychy.pl tydal.no @@ -3564,17 +4155,37 @@ tysfjord.no tysnes.no tysvar.no tyumen.ru +tzo.com u.bg u.se ua.tc uba.ar +ucoz.ae +ucoz.com +ucoz.de +ucoz.es +ucoz.hu +ucoz.lv +ucoz.net +ucoz.org +ucoz.pl +ucoz.ro ucoz.ru +ucoz.ua ud.it udine.it udm.ru udmurtia.ru +ueuo.co +ueuo.com +ufcfan.org +uhostall.com uhren.museum +uilfordschools.org uk.com +uk.ht +uk.im +uk.mn uk.net uk.pn uk.tc @@ -3586,11 +4197,15 @@ ullensvang.no ulm.museum ulsan.kr ulvik.no +umb1.com unam.na unbi.ba unblog.fr undersea.museum +undo.it +ungleheart.com uni.cc +uni.me uni7.net union.aero uniti.al @@ -3599,16 +4214,22 @@ unjarga.no unlugar.com unsa.ba upt.al +urhobby.com uri.arpa +url.ph url.st urn.arpa us.com +us.im us.ms us.na +us.org +us.pn us.tc us.tf us.to us.tt +usa.cc usa.museum usantiques.museum usarts.museum @@ -3616,30 +4237,39 @@ uscountryestate.museum usculture.museum usdecorativearts.museum usenet.pl +user32.com usgarden.museum ushistory.museum ushuaia.museum uslivinghistory.museum +ustdied.com ustka.pl ut.us utah.museum utazas.hu +uthorizeddns.net +uthorizeddns.org +uthorizeddns.us utsira.no utsunomiya.jp uu.mt +uv.ro uvic.museum uy.com uz.ua uzhgorod.ua v.bg +v22v.net va.it va.no va.us vaapste.no +vacau.com vadso.no vaga.no vagan.no vagsoy.no +vai.la vaksdal.no valle.no valley.museum @@ -3693,8 +4323,10 @@ vindafjord.no vinnica.ua virginia.museum virtual.museum +virtue.nu virtuel.museum viterbo.it +vk.me vlaanderen.museum vladikavkaz.ru vladimir.ru @@ -3702,10 +4334,12 @@ vladivostok.ru vlog.br vn.ua voagat.no +voip01.com volda.no volgograd.ru volkenkunde.museum vologda.ru +volyn.ua voronezh.ru voss.no vossevangen.no @@ -3713,6 +4347,7 @@ vr.it vrn.ru vt.it vt.us +vuturevx.com vv.cc vv.it vyatka.ru @@ -3721,6 +4356,8 @@ w.se wa.au wa.us wakayama.jp +wakwak.info +waky.com walbrzych.pl wales.museum wallonie.museum @@ -3744,8 +4381,15 @@ web.tj web.tr web.ve web.za +web44.net +webatu.com webcindario.com +webcrow.jp +webeden.co.uk +webege.com +webgarden.es webs.com +webuda.com weebly.com wegrow.pl western.museum @@ -3754,56 +4398,104 @@ whaling.museum whsites.net wi.us wielun.pl +wikaba.com wiki.br wildlife.museum williamsburg.museum windmill.museum +wix.com wlocl.pl wloclawek.pl wodzislaw.pl wolomin.pl wordpress.com workinggroup.aero +workisboring.com works.aero workshop.museum +wow64.net wroc.pl wroclaw.pl +ws.gy ws.na wv.us +ww1.biz +wwhost.biz www.ro +www1.biz +wwwhost.biz wy.us wz.cz x.bg x.se +x24hr.com +x64.me xanga.com xf.cz xj.cn xn--aroport-bya.ci +xn--c1avg.xn--p1ai xn--drbak-wua.no +xn--e1apq.xn--p1ai +xn--j1aef.xn--p1ai xn--leagaviika-52b.no xn--ostery-fya.no xn--tysvr-vra.no xn--unjrga-rta.no xn--vegrshei-c0a.no +xns01.com xorg.pl +xp3.biz +xuz.com +xxuz.com +xxxy.biz +xxy.biz +xxy.info xz.cn y.bg y.se +y03.com yakutia.ru +yalta.ua yamagata.jp yamaguchi.jp yamal.ru yamanashi.jp yaroslavl.ru +ydad.info +yddns.com +ye.vc yekaterinburg.ru +yftp.info +yftp.name yk.ca +ylftv.com +ymom.info yn.cn +ynamic-dns.net +ynamicdns.biz +yndns.pro +ynetav.net +ynetav.org +yns.com +ynssl.com +ynumber.org yokohama.jp +yolasite.com york.museum yorkshire.museum yosemite.museum +youdontcare.com +yourtrap.com youth.museum +ypicture.info +ypop3.net +ypop3.org +ysecondarydns.com yuzhno-sakhalinsk.ru +ywww.biz +yz.info +yzi.me z.bg z.se z8.ru @@ -3813,9 +4505,12 @@ za.org za.pl zachpomor.pl zagan.pl +zaghost.com zakopane.pl zaporizhzhe.ua +zapto.org zarow.pl +ze.cx zgora.pl zgorzelec.pl zgrad.ru @@ -3823,7 +4518,13 @@ zhitomir.ua zj.cn zlg.br zmail.ru +zone.be zoological.museum zoology.museum zp.ua zt.ua +zua.com +zux.com +zxq.net +zyns.com +zzux.com From 817161c698b655574d840cc0950af29bc62406b5 Mon Sep 17 00:00:00 2001 From: Steve Freegard Date: Fri, 7 Feb 2014 20:39:42 +0000 Subject: [PATCH 158/160] Remove redundent code --- plugins/delay_deny.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/plugins/delay_deny.js b/plugins/delay_deny.js index 52772200c..57ba74f46 100644 --- a/plugins/delay_deny.js +++ b/plugins/delay_deny.js @@ -115,14 +115,6 @@ exports.hook_rcpt_ok = function (next, connection, rcpt) { if (connection.notes.delay_deny_pre) { for (var i=0; i Date: Fri, 7 Feb 2014 14:01:46 -0800 Subject: [PATCH 159/160] karma: fix typo s/threshold/thresholds/ --- plugins/karma.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/karma.js b/plugins/karma.js index e91d5a485..9454bf8fb 100644 --- a/plugins/karma.js +++ b/plugins/karma.js @@ -206,7 +206,7 @@ exports.karma_onDisconnect = function (next, connection) { var key = 'karma|' + connection.remote_ip; var history = k.history; - if (config.threshold) { + if (config.thresholds) { var pos_lim = config.thresholds.positive || 2; if (k.connection > pos_lim) { From ff38515f2bf28ff596fd2d6f8d4d434894582cf8 Mon Sep 17 00:00:00 2001 From: Matt Sergeant Date: Fri, 7 Feb 2014 17:10:39 -0500 Subject: [PATCH 160/160] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 62c0d9cbf..ad5799fc9 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "name": "Haraka", "description": "An SMTP Server project.", "keywords": ["haraka", "smtp", "server", "email"], - "version": "2.2.8", + "version": "2.3.0", "homepage": "https://github.com/baudehlo/Haraka/", "repository": { "type": "git",