[Minor] lua_scanners - fix need_check, lua warnings #2, use lua_util

lualib/lua_scanners/clamav.lua

@@ -167,7 +167,9 @@ local function clamav_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, clamav_check_uncached) then
+    return
+  else
     clamav_check_uncached()
   end
 

lualib/lua_scanners/common.lua

@@ -186,14 +186,16 @@ local function dynamic_scan(task, rule)
   end
 end
 
-local function check_cache(task, digest, rule, fn)
+local function need_check(task, content, rule, digest, fn)
+
+  local uncached = true
   local key = digest
 
   local function redis_av_cb(err, data)
     if data and type(data) == 'string' then
       -- Cached
-      data = rspamd_str_split(data, '\t')
-      local threat_string = rspamd_str_split(data[1], '\v')
+      data = lua_util.str_split(data, '\t')
+      local threat_string = lua_util.str_split(data[1], '\v')
       local score = data[2] or rule.default_score
       if threat_string[1] ~= 'OK' then
         lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s',
@@ -203,12 +205,28 @@ local function check_cache(task, digest, rule, fn)
         lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
           rule.log_prefix, key, threat_string[1])
       end
+      uncached = false
     else
       if err then
         rspamd_logger.errx(task, 'got error checking cache: %s', err)
       end
-      return true
     end
+
+    local f_message_not_too_large = message_not_too_large(task, content, rule) or true
+    local f_message_not_too_small = message_not_too_small(task, content, rule) or true
+    local f_message_min_words = message_min_words(task, rule) or true
+    local f_dynamic_scan = dynamic_scan(task, rule) or true
+
+    if uncached and
+      f_message_not_too_large and
+      f_message_not_too_small and
+      f_message_min_words and
+      f_dynamic_scan then
+
+      fn()
+
+    end
+
   end
 
   if rule.redis_params then
@@ -228,14 +246,7 @@ local function check_cache(task, digest, rule, fn)
   end
 
   return false
-end
 
-local function need_check(task, content, rule, digest)
-  return check_cache(task, digest, rule) and
-    message_not_too_large(task, content, rule) and
-    message_not_too_small(task, content, rule) and
-    message_min_words(task, rule) and
-    dynamic_scan(task, rule)
 end
 
 local function save_cache(task, digest, rule, to_save, dyn_weight)
@@ -248,8 +259,8 @@ local function save_cache(task, digest, rule, to_save, dyn_weight)
       rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
           rule.detection_category, to_save, key, err)
     else
-      lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s',
-        rule.log_prefix, key, to_save, dyn_weight)
+      lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s',
+        rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
     end
   end
 
@@ -322,7 +333,7 @@ end
 -- ext is the last extension, LOWERCASED
 -- ext2 is the one before last extension LOWERCASED
 local function gen_extension(fname)
-  local filename_parts = rspamd_str_split(fname, '.')
+  local filename_parts = lua_util.str_split(fname, '.')
 
   local ext = {}
   for n = 1, 2 do

lualib/lua_scanners/dcc.lua

@@ -298,7 +298,9 @@ local function dcc_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, dcc_check_uncached) then
+    return
+  else
     dcc_check_uncached()
   end
 

lualib/lua_scanners/fprot.lua

@@ -160,7 +160,9 @@ local function fprot_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, fprot_check_uncached) then
+    return
+  else
     fprot_check_uncached()
   end
 

lualib/lua_scanners/icap.lua

@@ -112,7 +112,7 @@ local function icap_check(task, content, digest, rule)
 
     local function icap_callback(err, conn)
 
-      local function icap_requery(error, info)
+      local function icap_requery(err_m, info)
         -- set current upstream to fail because an error occurred
         upstream:fail()
 
@@ -123,7 +123,7 @@ local function icap_check(task, content, digest, rule)
 
           lua_util.debugm(rule.name, task,
               '%s: %s Request Error: %s - retries left: %s',
-              rule.log_prefix, info, error, retransmits)
+              rule.log_prefix, info, err_m, retransmits)
 
           -- Select a different upstream!
           upstream = rule.upstreams:get_upstream_round_robin()
@@ -144,15 +144,14 @@ local function icap_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
-            'exceed - err: %s', rule.log_prefix, error)
-          common.yield_result(task, rule, 'failed - err: ' .. error, 0.0, 'fail')
+            'exceed - error: %s', rule.log_prefix, err_m or '')
+          common.yield_result(task, rule, 'failed - error: ' .. err_m or '', 0.0, 'fail')
         end
       end
 
       local function get_respond_query()
-        table.insert(respond_headers, 1,
-            'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/'
-          .. rule.scheme .. ' ICAP/1.0\r\n')
+        table.insert(respond_headers, 1, string.format(
+            'RESPMOD icap://%s:%s/%s ICAP/1.0\r\n', addr:to_string(), addr:get_port(), rule.scheme))
         table.insert(respond_headers, '\r\n')
         table.insert(respond_headers, size .. '\r\n')
         table.insert(respond_headers, content)
@@ -161,7 +160,9 @@ local function icap_check(task, content, digest, rule)
       end
 
       local function add_respond_header(name, value)
-        table.insert(respond_headers, name .. ': ' .. value .. '\r\n' )
+        if name and value then
+          table.insert(respond_headers, string.format('%s: %s\r\n', name, value))
+        end
       end
 
       local function icap_result_header_table(result)
@@ -242,7 +243,7 @@ local function icap_check(task, content, digest, rule)
               '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID'])
 
           if string.find(icap_headers['X-Virus-ID'], ', ') then
-            local vnames = lua_util.rspamd_str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {}
+            local vnames = lua_util.str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {}
 
             for _,v in ipairs(vnames) do
               table.insert(threat_string, v)
@@ -267,7 +268,7 @@ local function icap_check(task, content, digest, rule)
               rule.log_prefix, infection_name, infected_filename)
 
           if string.find(infection_name, ', ') then
-            local vnames = lua_util.rspamd_str_split(infection_name, ',') or {}
+            local vnames = lua_util.str_split(infection_name, ',') or {}
 
             for _,v in ipairs(vnames) do
               table.insert(threat_string, v)
@@ -285,9 +286,9 @@ local function icap_check(task, content, digest, rule)
         end
       end
 
-      local function icap_r_respond_cb(error, data, connection)
-        if error or connection == nil then
-          icap_requery(err, "icap_r_respond_cb")
+      local function icap_r_respond_cb(err_m, data, connection)
+        if err_m or connection == nil then
+          icap_requery(err_m, "icap_r_respond_cb")
         else
           local result = tostring(data)
           conn:close()
@@ -315,17 +316,17 @@ local function icap_check(task, content, digest, rule)
         end
       end
 
-      local function icap_w_respond_cb(error, connection)
-        if error or connection == nil then
-          icap_requery(err, "icap_w_respond_cb")
+      local function icap_w_respond_cb(err_m, connection)
+        if err_m or connection == nil then
+          icap_requery(err_m, "icap_w_respond_cb")
         else
           connection:add_read(icap_r_respond_cb, '\r\n\r\n')
         end
       end
 
-      local function icap_r_options_cb(error, data, connection)
-        if error or connection == nil then
-          icap_requery(err, "icap_r_options_cb")
+      local function icap_r_options_cb(err_m, data, connection)
+        if err_m or connection == nil then
+          icap_requery(err_m, "icap_r_options_cb")
         else
           local icap_headers = icap_result_header_table(tostring(data))
 
@@ -343,7 +344,7 @@ local function icap_check(task, content, digest, rule)
                 local from = task:get_from('mime')
                 local rcpt_to = task:get_principal_recipient()
                 local client = task:get_from_ip()
-                add_respond_header('X-Client-IP', client:to_string())
+                if client then add_respond_header('X-Client-IP', client:to_string()) end
                 add_respond_header('X-Mail-From', from[1].addr)
                 add_respond_header('X-Rcpt-To', rcpt_to)
               end
@@ -384,7 +385,9 @@ local function icap_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, icap_check_uncached) then
+    return
+  else
     icap_check_uncached()
   end
 

lualib/lua_scanners/kaspersky_av.lua

@@ -178,7 +178,9 @@ local function kaspersky_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, kaspersky_check_uncached) then
+    return
+  else
     kaspersky_check_uncached()
   end
 

lualib/lua_scanners/oletools.lua

@@ -304,7 +304,9 @@ local function oletools_check(task, content, digest, rule)
 
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, oletools_check_uncached) then
+    return
+  else
     oletools_check_uncached()
   end
 

lualib/lua_scanners/savapi.lua

@@ -247,7 +247,9 @@ local function savapi_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, savapi_check_uncached) then
+    return
+  else
     savapi_check_uncached()
   end
 

lualib/lua_scanners/sophos.lua

@@ -167,7 +167,9 @@ local function sophos_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, sophos_check_uncached) then
+    return
+  else
     sophos_check_uncached()
   end
 

lualib/lua_scanners/spamassassin.lua

@@ -180,7 +180,7 @@ local function spamassassin_check(task, content, digest, rule)
             common.save_cache(task, digest, rule, symbols, spam_score)
           else
             local symbols_table = {}
-            symbols_table = rspamd_str_split(symbols, ",")
+            symbols_table = lua_util.str_split(symbols, ",")
             lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
 
             common.yield_result(task, rule, symbols_table, spam_score)
@@ -202,7 +202,9 @@ local function spamassassin_check(task, content, digest, rule)
     })
   end
 
-  if common.need_check(task, content, rule, digest) then
+  if common.need_check(task, content, rule, digest, spamassassin_check_uncached) then
+    return
+  else
     spamassassin_check_uncached()
   end