ioc2rpz is a place where threat intelligence meets DNS

ChangeLog.md

@@ -3,6 +3,7 @@
 ## 2019-07-19 v0.9.5.0
 - Bug fixes related to IXFR zone update and transfer
 - [CB] Source IXFR update "from" time will be keept the same until we get "non zero" update.
+- Retry for unavailable sources (see ioc2rpz.hrl)
 
 ## 2019-06-13 v0.9.4.0
 - Fixed bugs:

include/ioc2rpz.hrl

@@ -25,6 +25,9 @@
 -define(DefConf,"./cfg/ioc2rpz.conf"). %Default configuration
 -define(DefDB,"./db"). %Default DB location
 
+-define(Src_Retry,3). %# of retries if a source is not available
+-define(Src_Retry_TimeOut,3). %timeout between retries in seconds
+
 %-define(logTS, true). % Log timestamps (comment or uncomment)
 -define(debug, true). % Log debug messages
 -define(ioc2rpzSampleRPZ,"sample-zone.ioc2rpz"). %Default DB location

src/ioc2rpz_conn.erl

@@ -19,7 +19,7 @@
 -export([get_ioc/3,clean_feed_bin/2,clean_feed/2]).
 
 get_ioc(URL,REGEX,Source) ->
-  case get_ioc(URL) of
+  case get_ioc(URL,?Src_Retry) of
     {ok, Bin} ->
       ioc2rpz_fun:logMessage("Source: ~p, size: ~s (~p), MD5: ~p ~n",[Source#source.name, ioc2rpz_fun:conv_to_Mb(byte_size(Bin)),byte_size(Bin), ioc2rpz_fun:bin_to_hexstr(crypto:hash(md5,Bin))]), %TODO debug
       %Uncomment next 2 lines in case of limited memory. REGEX must be prepared for lowcase sources
@@ -68,29 +68,37 @@ p_clean_feed(IOC,REGEX,Max,Count)  ->
 
 
 %reads IOCs from a local file
-get_ioc(<<"file:",Filename/binary>> = _URL) ->
+get_ioc(<<"file:",Filename/binary>> = URL, Retry) ->
   case file:read_file(Filename) of
     {ok, Bin} ->
       {ok, Bin};
-    {error, Reason} ->
+	  {error,Reason} when Retry > 0 ->
+	    ioc2rpz_fun:logMessage("Error downloading feed ~p reason ~p. Try ~n",[URL, Reason, (?Src_Retry-Retry)]), %TODO timeout and add retry
+			timer:sleep(?Src_Retry_TimeOut*1000),
+			get_ioc(URL, Retry-1);
+    {error, Reason}  when Retry == 0->
       ioc2rpz_fun:logMessage("Error reading file ~p reason ~p ~n",[Filename, Reason]), 
       {error, Reason}
   end;
 
 %IOCs are provided by a local script
-get_ioc(<<"shell:",CMD/binary>> = _URL) ->
+get_ioc(<<"shell:",CMD/binary>> = _URL, Retry) ->
   {ok, list_to_binary(os:cmd(binary_to_list(CMD)))};
 
 %download IOCs from http/https/ftp
-get_ioc(<<Proto:5/bytes,_/binary>> = URL) when Proto == <<"http:">>;Proto == <<"https">>;Proto == <<"ftp:/">> ->
+get_ioc(<<Proto:5/bytes,_/binary>> = URL, Retry) when Proto == <<"http:">>;Proto == <<"https">>;Proto == <<"ftp:/">> ->
 	httpc:set_options([{cookies,enabled}]),
   case httpc:request(get,{binary_to_list(URL),[{"User-Agent", "Mozilla"}]},[],[{body_format,binary},{sync,true}]) of %,{socket_opts,[{cookies,enabled}]}
   {ok,{{_,200,_},_,Response}} ->
     {ok,Response};
   {ok,{{_,Code,_},Headers,Response}} ->
     ioc2rpz_fun:logMessage("Unexpected response code ~p, headers ~p ~n",[Code, Headers]), 
 		{ok,<<>>};
-  {error,Reason} ->
+  {error,Reason} when Retry > 0 ->
+    ioc2rpz_fun:logMessage("Error downloading feed ~p reason ~p. Try ~n",[URL, Reason, (?Src_Retry-Retry)]), %TODO timeout and add retry
+		timer:sleep(?Src_Retry_TimeOut*1000),
+		get_ioc(URL, Retry-1);
+  {error,Reason} when Retry == 0 ->
     ioc2rpz_fun:logMessage("Error downloading feed ~p reason ~p ~n",[URL, Reason]), %TODO timeout and add retry
     {error,Reason}
   end.