Merge pull request #4657 from fatalbanana/rbl_matchers

[Feature] rbl: support use of different matchers for return codes

conf/modules.d/rbl.conf

@@ -105,6 +105,7 @@ rbl {
       ipv6 = true;
       checks = ['from', 'received'];
       is_whitelist = true;
+      returncodes_matcher = "luapattern";
       whitelist_exception = "RCVD_IN_DNSWL";
       whitelist_exception = "RCVD_IN_DNSWL_NONE";
       whitelist_exception = "RCVD_IN_DNSWL_LOW";
@@ -152,6 +153,7 @@ rbl {
       rbl = "dwl.dnswl.org";
       checks = ['dkim'];
       ignore_whitelist = true;
+      returncodes_matcher = "luapattern";
       unknown = false;
 
       returncodes {
@@ -222,6 +224,7 @@ rbl {
       selector = "specific_urls_filter_map('surbl_hashbl_map', {limit = 10}).apply_methods('get_host', 'get_path').join_tables('/')",
       hash = 'md5';
       hash_len = 32;
+      returncodes_matcher = "luapattern";
       returncodes = {
         SURBL_HASHBL_PHISH = "127.0.0.8";
         SURBL_HASHBL_MALWARE = "127.0.0.16";

lualib/plugins/rbl.lua

@@ -130,6 +130,7 @@ local rule_schema_tbl = {
   return_codes = return_codes_schema:is_optional(),
   returnbits = return_bits_schema:is_optional(),
   returncodes = return_codes_schema:is_optional(),
+  returncodes_matcher = ts.one_of { "equality", "glob", "luapattern", "radix", "regexp" }:is_optional(),
   selector = ts.one_of { ts.string, ts.table }:is_optional(),
   selector_flatten = ts.boolean:is_optional(),
   symbol = ts.string:is_optional(),
@@ -199,6 +200,21 @@ local function convert_checks(rule)
     rule.from = true
   end
 
+  if rule.returncodes and not rule.returncodes_matcher then
+    for _, v in pairs(rule.returncodes) do
+      for _, e in ipairs(v) do
+        if e:find('%', 1, true) then
+          rspamd_logger.info(rspamd_config, 'implicitly enabling luapattern returncodes_matcher for rule %s', rule.symbol)
+          rule.returncodes_matcher = 'luapattern'
+          break
+        end
+      end
+      if rule.returncodes_matcher then
+        break
+      end
+    end
+  end
+
   return rule
 end
 

src/plugins/lua/rbl.lua

@@ -215,7 +215,43 @@ local function gen_check_rcvd_conditions(rbl, received_total)
   end
 end
 
-local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt)
+local matchers = {}
+
+matchers.radix = function(_, _, real_ip, map)
+  return map and map:get_key(real_ip) or false
+end
+
+matchers.equality = function(codes, to_match)
+  if type(codes) ~= 'table' then return codes == to_match end
+  for _, ip in ipairs(codes) do
+    if to_match == ip then
+      return true
+    end
+  end
+  return false
+end
+
+matchers.luapattern = function(codes, to_match)
+  if type(codes) ~= 'table' then
+    return string.find(to_match, '^' .. codes .. '$') and true or false
+  end
+  for _, pattern in ipairs(codes) do
+    if string.find(to_match, '^' .. pattern .. '$') then
+      return true
+    end
+  end
+  return false
+end
+
+matchers.regexp = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
+end
+
+matchers.glob = function(_, to_match, _, map)
+  return map and map:get_key(to_match) or false
+end
+
+local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_table_elt, match)
   local function make_option(ip, label)
     if ip then
       return string.format('%s:%s:%s',
@@ -274,6 +310,8 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl
     return
   end
 
+  local returncodes_maps = rbl.returncodes_maps or {}
+
   for _, result in ipairs(results) do
     local ipstr = result:to_string()
     lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
@@ -292,12 +330,10 @@ local function rbl_dns_process(task, rbl, to_resolve, results, err, resolve_tabl
       end
     elseif rbl.returncodes then
       for s, codes in pairs(rbl.returncodes) do
-        for _, v in ipairs(codes) do
-          if string.find(ipstr, '^' .. v .. '$') then
-            foundrc = true
-            insert_results(s)
-            break
-          end
+        local res = match(codes, ipstr, result, returncodes_maps[s])
+        if res then
+          foundrc = true
+          insert_results(s)
         end
       end
     end
@@ -858,14 +894,19 @@ local function gen_rbl_callback(rule)
     description[#description + 1] = 'selector'
   end
 
+  if not rule.returncodes_matcher then
+    rule.returncodes_matcher = 'equality'
+  end
+  local match = matchers[rule.returncodes_matcher]
+
   local callback_f = function(task)
     -- DNS requests to issue (might be hashed afterwards)
     local dns_req = {}
     local whitelist = task:cache_get('rbl_whitelisted') or {}
 
     local function gen_rbl_dns_callback(resolve_table_elt)
       return function(_, to_resolve, results, err)
-        rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt)
+        rbl_dns_process(task, rule, to_resolve, results, err, resolve_table_elt, match)
       end
     end
 
@@ -972,6 +1013,12 @@ local function gen_rbl_callback(rule)
   return callback_f, string.format('checks: %s', table.concat(description, ','))
 end
 
+local map_match_types = {
+  glob = true,
+  radix = true,
+  regexp = true,
+}
+
 local function add_rbl(key, rbl, global_opts)
   if not rbl.symbol then
     rbl.symbol = key:upper()
@@ -1049,6 +1096,19 @@ local function add_rbl(key, rbl, global_opts)
         def_type, rbl.symbol)
   end
 
+  local match_type = rbl.returncodes_matcher
+  if match_type and rbl.returncodes and map_match_types[match_type] then
+    if not rbl.returncodes_maps then
+      rbl.returncodes_maps = {}
+    end
+    for label, v in pairs(rbl.returncodes) do
+      if type(v) ~= 'table' then
+        v = {v}
+      end
+      rbl.returncodes_maps[label] = lua_maps.map_add_from_ucl(v, match_type, string.format('%s_%s RBL returncodes', label, rbl.symbol))
+    end
+  end
+
   if rbl.url_compose_map then
     local lua_urls_compose = require "lua_urls_compose"
     rbl.url_compose_map = lua_urls_compose.add_composition_map(rspamd_config, rbl.url_compose_map)

test/functional/configs/merged-override.conf

@@ -269,9 +269,10 @@ rbl {
         from = 'FAKE_RBL',
       }
       unknown = true;
+      returncodes_matcher = "regexp";
       returncodes = {
-        "CODE_2" = "127.0.0.2";
-        "CODE_3" = "127.0.0.3";
+        "CODE_2" = '^127\.0\.0\.2$';
+        "CODE_3" = '^127\.0\.0\.3$';
       }
     }
     fake_whitelist {
@@ -283,9 +284,10 @@ rbl {
       rbl = "fake.wl";
       symbol = "FAKE_WL_RBL_UNKNOWN";
       unknown = true;
+      #returncodes_matcher = "luapattern";
       returncodes = {
-        "FAKE_WL_RBL_CODE_2" = "127.0.0.2";
-        "FAKE_WL_RBL_CODE_3" = "127.0.0.3";
+        "FAKE_WL_RBL_CODE_2" = "127%.0%.0%.2";
+        "FAKE_WL_RBL_CODE_3" = "127%.0%.0%.3";
       }
     }
     RSPAMD_EMAILBL {
@@ -294,8 +296,9 @@ rbl {
       ignore_defaults = true;
       emails = true;
       emails_domainonly = true
+      returncodes_matcher = "radix";
       returncodes = {
-        RSPAMD_EMAILBL = "127.0.0.2";
+        RSPAMD_EMAILBL = "127.0.0.2/32";
       }
     }
     URIBL_NUMERIC {
@@ -307,10 +310,14 @@ rbl {
       images = true;
       rbl = "test9.uribl";
     }
-    URIBL_NUMERIC_CONTENT {
+    UNKNOWN_URIBL_NUMERIC_CONTENT {
       checks = ["numeric_urls"];
       content_urls = true;
       rbl = "test9.uribl";
+      returncodes_matcher = "glob";
+      returncodes = {
+        URIBL_NUMERIC_CONTENT = "*.*.*.*";
+      }
     }
     URIBL_NUMERIC_EVERYTHING {
       checks = ["numeric_urls"];