IoC lookup REST API call was introduced.

ChangeLog.md

@@ -1,6 +1,19 @@
 # ioc2rpz change log
 [CB] - Changed Behaviour
+## 2019-12-11 v1.1.1.3
+- [CB] IoC lookup REST API call. The submitted indicator converted to lowcase before the lookups.
+
+## 2019-12-11 v1.1.1.2
+- [CB] IoC lookup REST API call output was modified
+
+## 2019-12-11 v1.1.1.1
+- [CB] Regex expressions were updated to match any type of newline string chars "{newline, any}"
+
+## 2019-12-10 v1.1.1.0
+- IoC lookup REST API call
+
 ## 2019-12-05 v1.1.0.2
+- Bug #20. Whitelists didn't work.
 
 ## 2019-11-25 v1.1.0.1
 - Bug with updating zones (broken packets after AXFR and wildcard rule after IXFR). It is recommended to upgrade to the newest release.

README.md

@@ -111,6 +111,7 @@ API requests:
 - GET ``/api/v1.0/mgmt/update_tkeys`` - update TSIG keys.
 - GET ``/api/v1.0/mgmt/terminate`` - shutdown ioc2rpz server.
 - GET ``/api/v1.0/feed/:rpz`` - get content (indicators) of ``:rpz`` feed. 
+- GET ``/api/v1.0/ioc/:ioc?tkey=:tkey`` - check if indicator is blocked by RPZ feeds. An optional param ``:tkey`` allows to limit validation to a specific TSIG Key. W/o it the search will be done among all feeds. 
 
 ## Configuration file
 The configuration is an Erlang file. Every configuration option is an Erlang term so the configuration must comply with Erlang syntax. ioc2rpz does not check the configuration file for possible errors, typos etc.

TODO.md

@@ -1,8 +1,9 @@
 ## Bugs
-- [ ] wildcard rule is generated
+- [x] wildcard rule is generated
 
 ## Core / DNS
-- [ ] check IoC in the RPZs feeds "What's in your DNS?"
+- [x] check IoC in the RPZs feeds "What's in your DNS?"
+- [ ] simple permissions model
 - [ ] REST API rate limiting
 - [ ] DNS requests rate limiting
 - [ ] HotCache optimization if refresh time less than hotcache storage time

include/ioc2rpz.hrl

@@ -48,7 +48,7 @@
 %%%%%%
 %%%%%% Do not modify any settings below the line
 %%%%%%
--define(ioc2rpz_ver, "1.1.0.2-2019120501").
+-define(ioc2rpz_ver, "1.1.1.3-2019127001").
 
 -define(ZNameZip,16#c00c:16). %Zone name/original fqdn from a request is always at byte 10 in the response
 -define(ZNameZipN,16#c00c). % Offset in bytes - Zone name/original fqdn from a request is always at byte 10 in the response

src/ioc2rpz.erl

@@ -657,7 +657,9 @@ send_zone(<<"true">>,Socket,{Questions,DNSId,OptB,OptE,RH,Rest,Zone,?T_IXFR,NSSe
   IOCnew=[ {X,Z} || [X,_Y,Z] <- ioc2rpz_db:read_db_record(Zone,SOA#dns_SOA_RR.serial,new)],
 %  ioc2rpz_fun:logMessage("Serial ~p /= Serial IXFR ~p Zone ~p Expired IOC ~p, New IOC ~p ~n",[Zone#rpz.serial,Zone#rpz.serial_ixfr,Zone#rpz.zone_str,IOCexp,IOCnew]),
 
-  {ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})$"), %
+% {ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})$"), %
+	{ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(\\/[0-9]{1,3})?)$|(:)"),
+
   PktHLen = 12+byte_size(Questions),
 %  T_ZIP_L=ets:new(label_zip_table, [{read_concurrency, true}, {write_concurrency, true}, set, private]), % нужны ли {read_concurrency, true}, {write_concurrency, true} ???
 	T_ZIP_L=init_T_ZIP_L(Zone),
@@ -1006,7 +1008,7 @@ gen_rpzrule(Domain,RPZ,TTL,<<"false">>,{Action,LocData},_,PktHLen,T_ZIP_L) when
     {error, _} ->
       {ok,0,[],[]};
     {_,BDomain} -> %ok, zip
-     {_,LocDataZ} = domstr_to_bin_zip({ok,LocData},0,PktHLen+10+byte_size(BDomain),T_ZIP_L),
+     {_,LocDataZ} = domstr_to_bin_zip({ok,[LocData]},0,PktHLen+10+byte_size(BDomain),T_ZIP_L),
 %     {_,LocDataZ} = domstr_to_bin(LocData,0), 
       ELDS=byte_size(LocDataZ),
       {ok,1,[list_to_binary([BDomain,<<?T_CNAME:16,?C_IN:16, TTL:32,ELDS:16>>,LocDataZ])],[list_to_binary([<<?T_CNAME:16,?C_IN:16, TTL:32,ELDS:16>>,LocDataZ])]}

src/ioc2rpz_conn.erl

@@ -22,14 +22,7 @@ get_ioc(URL,REGEX,Source) ->
   case get_ioc(URL,?Src_Retry) of
     {ok, Bin} ->
       ioc2rpz_fun:logMessage("Source: ~p, size: ~s (~p), MD5: ~p ~n",[Source#source.name, ioc2rpz_fun:conv_to_Mb(byte_size(Bin)),byte_size(Bin), ioc2rpz_fun:bin_to_hexstr(crypto:hash(md5,Bin))]), %TODO debug
-      %Uncomment next 2 lines in case of limited memory. REGEX must be prepared for lowcase sources
-      %BinLow=ioc2rpz_fun:bin_to_lowcase(Bin),
-      %L=clean_feed(ioc2rpz_fun:split_tail(BinLow,<<"\n">>),REGEX),
-
-      %methods used below consume more memory. It is not possible to run ioc2rpz with 1M indicator on AWS free tier
-      %L=clean_feed_bin(ioc2rpz_fun:split_tail(Bin,<<"\n">>),REGEX),
-      %Comment next 1 line in case of limited memory. REGEX must be prepared for lowcase sources
-
+
       %TODO spawn cleanup
       CTime=ioc2rpz_fun:curr_serial_60(),
       %L=[ {ioc2rpz_fun:bin_to_lowcase(X),Y} || {X,Y} <- clean_feed(ioc2rpz_fun:split_tail(Bin,<<"\n">>),REGEX) ],
@@ -164,13 +157,13 @@ clean_feed(IOC,none) ->
 %Default REFEX
 %Extract IOCs,remove unsupported chars using standard REGEX. Expiration date is not supported;
 clean_feed(IOC,[]) ->
-  {ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$"),
+  {ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$",[{newline, any}]),
   [ X || X <- clean_feed(IOC,[],MP), X /= <<>>];
 
 
 %Extract IOCs,remove unsupported chars using user's defined REGEX. Expiration date is supported. First value - IOC, second - Exp. Date;
 clean_feed(IOC,REX) -> %REX - user's regular expression
-  {ok,MP} = re:compile(REX),
+  {ok,MP} = re:compile(REX,[{newline, any}]),
   [ X || X <- clean_feed(IOC,[],MP), X /= <<>>].
 
 clean_feed([Head|Tail],CleanIOC,REX) ->
@@ -190,11 +183,11 @@ clean_feed_bin(IOC,none) ->
   [ {X,0} || X <- IOC, X /= <<>>];
 
 clean_feed_bin(IOC,[]) ->
-  {ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$"),
+  {ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$",[{newline, any}]), 
   [ X || X <- clean_feed_bin(IOC,<<>>,MP), X /= <<>>];
 
 clean_feed_bin(IOC,REX) -> %REX - user's regular expression
-  {ok,MP} = re:compile(REX),
+  {ok,MP} = re:compile(REX,[{newline, any}]), 
   [ X || X <- clean_feed_bin(IOC,<<>>,MP), X /= <<>>].
 
 clean_feed_bin([Head|Tail],CleanIOC,REX) ->

src/ioc2rpz_db.erl

@@ -17,7 +17,7 @@
 -module(ioc2rpz_db).
 -include_lib("ioc2rpz.hrl").
 -export([init_db/3,db_table_info/2,read_db_pkt/1,write_db_pkt/2,delete_db_pkt/1,read_db_record/3,write_db_record/3,delete_old_db_record/1,saveZones/0,loadZones/0,loadZones/1,
-        get_zone_info/2,clean_DB/1,save_zone_info/1,get_allzones_info/2]).
+        get_zone_info/2,clean_DB/1,save_zone_info/1,get_allzones_info/2, lookup_db_record/2]).
 
 
 init_db(ets,DBDir,PID) ->
@@ -149,7 +149,7 @@ write_db_record(ets,Zone,IOCs,ixfr) when IOCs == [] ->
 write_db_record(mnesia,Zone,IOCs,ixfr) ->
 	{ok,0};
 
-write_db_record(_DBStorage,_Zone,_IOCs,_XFR) ->
+write_db_record(_DBStorage,_Zone,_IOCs,_XFR) ->	
 	{ok,0}. %non cached zones
 
 update_db_record(ets, Zone, Serial, IOC, IOCExp, [], CTime) when IOCExp > 0,IOCExp =< CTime ->
@@ -171,6 +171,42 @@ update_db_record(ets, Zone, Serial, IOC, IOCExp, Update, CTime) -> %ok; %not new
 
 update_db_record(mnesia, Zone, Serial, IOC, IOCExp, Update, CTime) -> ok.
 
+%%%
+%%% Lookup if an indicator is in the DB.
+%%% Recurs - validate hosts/fqdns if they are blocked by a wildcard rule or a subnet. 
+%%%
+lookup_db_record(IOC, Recurs) ->
+	lookup_db_record(?DBStorage, IOC, Recurs).
+
+lookup_db_record(ets, IOC, false) ->
+	{ok,[{IOC,ets:select(rpz_ixfr_table,[{{{ioc,'$0',IOC},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]}; 
+
+lookup_db_record(mnesia, IOC, false) ->
+	{ok,[{IOC,[]}]};
+
+lookup_db_record(ets, IOC, true) ->
+% check IP or domain
+			%ioc2rpz_fun:logMessage("Checking IOC ~s ~n",[IOC]),
+			{ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(\\/[0-9]{1,3})?)$|(:)"),
+      case re:run(IOC,MP,[global,notempty,{capture,[1],binary}]) of
+        {match,_} -> {ok,[{IOC,ets:select(rpz_ixfr_table,[{{{ioc,'$0',IOC},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]};
+        _ ->  lookup_db_record(ets,IOC,<<"">>,ioc2rpz_fun:rsplit_tail(IOC, <<".">>),[])
+      end; 
+
+lookup_db_record(mnesia, IOC, true) ->
+	{ok,[{IOC,[]}]}.
+
+lookup_db_record(ets,IOC, FQDN, [], Result) ->
+  %ioc2rpz_fun:logMessage("Result: ~p\n\n",[{ok,Result}]),
+	FResult = [{IOC2,ARR} || {IOC2,ARR} <-Result, ((IOC == IOC2) or (ARR /= []))],
+	{ok,FResult};
+
+lookup_db_record(ets,IOC, FQDN, [Label|REST], Result) ->
+	NFQDN = if FQDN == <<"">> -> Label; true ->  <<Label/binary,".",FQDN/binary>> end,
+  %ioc2rpz_fun:logMessage("Checking ~p ~n",[NFQDN]),
+	lookup_db_record(ets, IOC, NFQDN, REST, Result ++ [{NFQDN,ets:select(rpz_ixfr_table,[{{{ioc,'$0',NFQDN},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]).
+
+
 delete_old_db_record(Zone) ->
   delete_old_db_record(?DBStorage,Zone).
 

src/ioc2rpz_fun.erl

@@ -17,7 +17,7 @@
 -module(ioc2rpz_fun).
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("ioc2rpz.hrl").
--export([logMessage/2,logMessageCEF/2,strs_to_binary/1,curr_serial/0,curr_serial_60/0,constr_ixfr_url/3,ip_to_bin/1,read_local_actions/1,split_bin_bytes/2,split_tail/2,
+-export([logMessage/2,logMessageCEF/2,strs_to_binary/1,curr_serial/0,curr_serial_60/0,constr_ixfr_url/3,ip_to_bin/1,read_local_actions/1,split_bin_bytes/2,split_tail/2,rsplit_tail/2,
          bin_to_lowcase/1,ip_in_list/2,intersection/2,bin_to_hexstr/1,conv_to_Mb/1,q_class/1,q_type/1,split/2,msg_CEF/1,base64url_decode/1]).
 
 logMessage(Message, Vars) ->
@@ -164,6 +164,14 @@ split_tail(String, Pattern) ->
 		[] -> []
 	end.
 
+rsplit_tail(String, Pattern) ->
+%  ioc2rpz_fun:logMessage("z_split ~p ~p ~n",[String, Pattern]),
+	case binary:split(String, Pattern) of %binary:split
+		[First, Second] -> rsplit_tail(Second, Pattern) ++ [First];
+		[First] -> [First];
+		[] -> []
+	end.
+
 bin_to_lowcase(A) ->
  << << (b_to_lowcase(C)) >> || << C >> <= A >>.
 % << << C >> || << C >> <= A >>.
@@ -289,3 +297,13 @@ base64url_decode_test() -> [
  ?assert(base64url_decode(<<"AAABAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE">>) =:= {ok,<<0,0,1,0,0,1,0,0,0,0,0,0,7,101,120,97,109,112,108,101,3,99,111,109,0,0,1,0,1>>}),
  ?assert(base64url_decode(<<"AAABAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE==">>) =:= {error,<<>>})
 ].
+
+bin_to_lowcase_test() ->[
+	?assert(bin_to_lowcase(<<"fC00::01">>) =:= <<"fc00::01">>),
+	?assert(bin_to_lowcase(<<"Aaaaaa">>) =:= <<"aaaaaa">>),
+	?assert(bin_to_lowcase(<<"bBbBbB">>) =:= <<"bbbbbb">>),
+	?assert(bin_to_lowcase(<<"ccC">>) =:= <<"ccc">>),
+	?assert(bin_to_lowcase(<<"D">>) =:= <<"d">>),
+	?assert(bin_to_lowcase(<<"f">>) =:= <<"f">>),
+	?assert(bin_to_lowcase(<<"eeeeeeeeeeeeeeeeeeeeeee">>) =:= <<"eeeeeeeeeeeeeeeeeeeeeee">>)
+].

src/ioc2rpz_rest.erl

@@ -199,6 +199,8 @@ srv_mgmt(Req, State, Format) when State#state.op == get_rpz -> % Get RPZ
 %
 %% need to validate TSIG on the access to the feeds
 %
+% ioc2rpz_db:lookup_db_record(<<"baddomain1.com">>,no).
+% ioc2rpz_db:lookup_db_record(<<"99.98.61.5">>,no).
 % DB
 %11> ets:select(rpz_ixfr_table,[{{{ioc,'$0',<<"99.98.61.5">>},'$2','$3'},[],[{{'$0','$2','$3'}}]}]).   
 %[{<<5,108,111,99,97,108,7,105,111,99,50,114,112,122,0>>,
@@ -215,6 +217,21 @@ srv_mgmt(Req, State, Format) when State#state.op == get_rpz -> % Get RPZ
 %	{Body, Req0, State};
 
 
+srv_mgmt(Req, State, Format) when State#state.op == get_ioc -> % check IoC
+	#{peer := {IP, Port}} = Req,
+    ioc2rpz_fun:logMessageCEF(ioc2rpz_fun:msg_CEF(230),[ioc2rpz:ip_to_str(IP), Port, cowboy_req:path(Req), ""]),
+		IOC = ioc2rpz_fun:bin_to_lowcase(cowboy_req:binding(ioc, Req)),
+		TKEY = try
+				maps:get(tkey,cowboy_req:match_qs([tkey],Req)) %%%%% parse_qs
+			catch _:_ ->
+				<<"">>
+		end,
+		{Recur, Zones} = get_tkey_zones(TKEY),
+    %ioc2rpz_fun:logMessage("Recursion: ~p\nZones: ~p\n\n",[Recur,Zones]),
+		Body=format_ioc(ioc2rpz_db:lookup_db_record(IOC,Recur),{IOC,TKEY, Zones},Format),
+		{Body, Req, State};
+
+
 srv_mgmt(Req, State, Format) when State#state.op == catch_all -> % Catch all unsupported requests from authenticated users
 	#{peer := {IP, Port}} = Req,
     ioc2rpz_fun:logMessageCEF(ioc2rpz_fun:msg_CEF(137),[ioc2rpz:ip_to_str(IP), Port, cowboy_req:path(Req), ""]),
@@ -232,6 +249,64 @@ srv_mgmt(Req, State, Format) when State#state.op == catch_all -> % Catch all uns
 rest_terminate(Req, State) ->
 	ok.
 
+format_ioc({ok,Results},Req,Format) ->
+	format_ioc(Results,Req,Format,"");
+
+format_ioc({error,_Results},{IOC,_TKEY,_Zones},json) ->
+	io_lib:format("{\"status\":\"error\", \"ioc\": ~p}",[IOC]);
+
+format_ioc({error,_Results},{IOC,_TKEY,_Zones},txt) ->
+	io_lib:format("status: error\nIOC: ~p\n",[IOC]).
+
+format_ioc([],{IOC,TKEY,_Zones},json, Result) ->
+ io_lib:format("{\"ioc\":\"~s\", \"tkey\":\"~s\", \"data\":[~s]}\n\n",[IOC,TKEY,Result]);
+
+format_ioc([{El,Feeds}|Results],Req,json,"") ->
+	Ind=io_lib:format("{\"ioc\": \"~s\", \"feeds\": ~s}",[El, parse_feeds(Feeds,Req,"",json)]),
+	format_ioc(Results,Req,json, Ind);
+
+format_ioc([{El,Feeds}|Results],Req,Format,Result) ->
+	Ind=io_lib:format("{\"ioc\": \"~s\", \"feeds\": ~s}",[El, parse_feeds(Feeds,Req,"",json)]),
+	format_ioc(Results,Req,json, Result ++","++ Ind).
+
+
+parse_feeds([],_Req,Result,json) ->
+	"["++Result++"]";
+
+parse_feeds([{Feed, Serial, Exp}|REST],{_IOC,_TKEY,Zones}=Req,"",json) ->
+	Memb=maps:is_key(Feed,Zones),
+	Feed_Str=if (Memb) -> {FN,TY,WC}=maps:get(Feed,Zones), io_lib:format("{\"feed\":~p, \"wildcard\":~s, \"type\":~p, \"rpz_serial\": ~p, \"ioc_expiration\": ~p}",[FN, WC, binary_to_list(TY), Serial, Exp]); true -> "" end,
+	parse_feeds(REST,Req,Feed_Str,json);
+
+parse_feeds([{Feed, Serial, Exp}|REST],{_IOC,_TKEY,Zones}=Req,Result,json) ->
+	Memb=maps:is_key(Feed,Zones),
+	Feed_Str=if (Memb) -> {FN,TY,WC}=maps:get(Feed,Zones), ","++io_lib:format("{\"feed\":~p, \"wildcard\":~s, \"type\":~p, \"rpz_serial\": ~p, \"ioc_expiration\": ~p}",[FN, WC, binary_to_list(TY), Serial, Exp]); true -> "" end,
+	parse_feeds(REST,Req,Result++Feed_Str,json).
+
+%%%
+%%% Get zones availble for TKey
+%%%
+get_tkey_zones(TKey) ->
+	{ok, TKeyBin} = ioc2rpz:domstr_to_bin(TKey,0),
+	Groups = [ X || [X,Y] <- ets:match(cfg_table,{[key_group,'$1',TKeyBin],'$3'}) ],
+	get_tkey_zones(TKeyBin, Groups, [ X || [X] <- ets:match(cfg_table,{[rpz,'_'],'_','$4'}) ], []). %{X#rpz.zone, X#rpz.zone_str, X#rpz.wildcards, X#rpz.akeys, X#rpz.ioc_type, X#rpz.key_groups}
+
+get_tkey_zones(TKeyBin, _Groups,[], Zones) ->
+	Recur = [ X || {_,{_,_,X}} <- Zones, X == <<"true">> ] /= [],
+%	ZNames = [ X || {X,{_,_,_}} <- Zones ],
+	{Recur, maps:from_list(lists:flatten(Zones))};
+
+get_tkey_zones(TKeyBin, Groups, [RPZ|Rest], Zones) ->
+	KZ = lists:member(TKeyBin, RPZ#rpz.akeys),
+	GZ = [X || X <- Groups, lists:member(X,RPZ#rpz.key_groups)],
+	AZ=case {KZ,GZ,TKeyBin} of
+		{true,_,_} -> [{RPZ#rpz.zone, {RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
+		{_,Gr,_} when Gr /= [] -> [{RPZ#rpz.zone, {RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
+		{_,_,<<0,0>>} -> [{RPZ#rpz.zone,{RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
+		_Else -> []
+	end,
+	get_tkey_zones(TKeyBin, Groups, Rest, Zones ++ AZ).
+
 gen_rpz_stats() ->
 	[ [{"name",X#rpz.zone_str},{"rule_count",X#rpz.rule_count},{"ioc_count",X#rpz.ioc_count},{"serial",X#rpz.serial},{"serial_ixfr",X#rpz.serial_ixfr},{"update_time",X#rpz.update_time},{"ixfr_update_time",X#rpz.ixfr_update_time},{"ixfr_nz_update_time",X#rpz.ixfr_nz_update_time}] || [X]  <- ets:match(cfg_table,{[rpz,'_'],'_','$2'}), X#rpz.rule_count /= undefined].