Skip to content
Permalink
Browse files

IoC lookup REST API call was introduced.

  • Loading branch information
Homas committed Dec 28, 2019
2 parents 331c126 + 5700de0 commit b39e007373f83d550a550ead99a24b6671d6dd6f
Showing with 159 additions and 20 deletions.
  1. +13 −0 ChangeLog.md
  2. +1 −0 README.md
  3. +3 −2 TODO.md
  4. +1 −1 include/ioc2rpz.hrl
  5. +4 −2 src/ioc2rpz.erl
  6. +5 −12 src/ioc2rpz_conn.erl
  7. +38 −2 src/ioc2rpz_db.erl
  8. +19 −1 src/ioc2rpz_fun.erl
  9. +75 −0 src/ioc2rpz_rest.erl
@@ -1,6 +1,19 @@
# ioc2rpz change log
[CB] - Changed Behaviour
## 2019-12-11 v1.1.1.3
- [CB] IoC lookup REST API call. The submitted indicator converted to lowcase before the lookups.

## 2019-12-11 v1.1.1.2
- [CB] IoC lookup REST API call output was modified

## 2019-12-11 v1.1.1.1
- [CB] Regex expressions were updated to match any type of newline string chars "{newline, any}"

## 2019-12-10 v1.1.1.0
- IoC lookup REST API call

## 2019-12-05 v1.1.0.2
- Bug #20. Whitelists didn't work.

## 2019-11-25 v1.1.0.1
- Bug with updating zones (broken packets after AXFR and wildcard rule after IXFR). It is recommended to upgrade to the newest release.
@@ -111,6 +111,7 @@ API requests:
- GET ``/api/v1.0/mgmt/update_tkeys`` - update TSIG keys.
- GET ``/api/v1.0/mgmt/terminate`` - shutdown ioc2rpz server.
- GET ``/api/v1.0/feed/:rpz`` - get content (indicators) of ``:rpz`` feed.
- GET ``/api/v1.0/ioc/:ioc?tkey=:tkey`` - check if indicator is blocked by RPZ feeds. An optional param ``:tkey`` allows to limit validation to a specific TSIG Key. W/o it the search will be done among all feeds.
## Configuration file
The configuration is an Erlang file. Every configuration option is an Erlang term so the configuration must comply with Erlang syntax. ioc2rpz does not check the configuration file for possible errors, typos etc.
@@ -1,8 +1,9 @@
## Bugs
- [ ] wildcard rule is generated
- [x] wildcard rule is generated

## Core / DNS
- [ ] check IoC in the RPZs feeds "What's in your DNS?"
- [x] check IoC in the RPZs feeds "What's in your DNS?"
- [ ] simple permissions model
- [ ] REST API rate limiting
- [ ] DNS requests rate limiting
- [ ] HotCache optimization if refresh time less than hotcache storage time
@@ -48,7 +48,7 @@
%%%%%%
%%%%%% Do not modify any settings below the line
%%%%%%
-define(ioc2rpz_ver, "1.1.0.2-2019120501").
-define(ioc2rpz_ver, "1.1.1.3-2019127001").

-define(ZNameZip,16#c00c:16). %Zone name/original fqdn from a request is always at byte 10 in the response
-define(ZNameZipN,16#c00c). % Offset in bytes - Zone name/original fqdn from a request is always at byte 10 in the response
@@ -657,7 +657,9 @@ send_zone(<<"true">>,Socket,{Questions,DNSId,OptB,OptE,RH,Rest,Zone,?T_IXFR,NSSe
IOCnew=[ {X,Z} || [X,_Y,Z] <- ioc2rpz_db:read_db_record(Zone,SOA#dns_SOA_RR.serial,new)],
% ioc2rpz_fun:logMessage("Serial ~p /= Serial IXFR ~p Zone ~p Expired IOC ~p, New IOC ~p ~n",[Zone#rpz.serial,Zone#rpz.serial_ixfr,Zone#rpz.zone_str,IOCexp,IOCnew]),

{ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})$"), %
% {ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})$"), %
{ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(\\/[0-9]{1,3})?)$|(:)"),

PktHLen = 12+byte_size(Questions),
% T_ZIP_L=ets:new(label_zip_table, [{read_concurrency, true}, {write_concurrency, true}, set, private]), % нужны ли {read_concurrency, true}, {write_concurrency, true} ???
T_ZIP_L=init_T_ZIP_L(Zone),
@@ -1006,7 +1008,7 @@ gen_rpzrule(Domain,RPZ,TTL,<<"false">>,{Action,LocData},_,PktHLen,T_ZIP_L) when
{error, _} ->
{ok,0,[],[]};
{_,BDomain} -> %ok, zip
{_,LocDataZ} = domstr_to_bin_zip({ok,LocData},0,PktHLen+10+byte_size(BDomain),T_ZIP_L),
{_,LocDataZ} = domstr_to_bin_zip({ok,[LocData]},0,PktHLen+10+byte_size(BDomain),T_ZIP_L),
% {_,LocDataZ} = domstr_to_bin(LocData,0),
ELDS=byte_size(LocDataZ),
{ok,1,[list_to_binary([BDomain,<<?T_CNAME:16,?C_IN:16, TTL:32,ELDS:16>>,LocDataZ])],[list_to_binary([<<?T_CNAME:16,?C_IN:16, TTL:32,ELDS:16>>,LocDataZ])]}
@@ -22,14 +22,7 @@ get_ioc(URL,REGEX,Source) ->
case get_ioc(URL,?Src_Retry) of
{ok, Bin} ->
ioc2rpz_fun:logMessage("Source: ~p, size: ~s (~p), MD5: ~p ~n",[Source#source.name, ioc2rpz_fun:conv_to_Mb(byte_size(Bin)),byte_size(Bin), ioc2rpz_fun:bin_to_hexstr(crypto:hash(md5,Bin))]), %TODO debug
%Uncomment next 2 lines in case of limited memory. REGEX must be prepared for lowcase sources
%BinLow=ioc2rpz_fun:bin_to_lowcase(Bin),
%L=clean_feed(ioc2rpz_fun:split_tail(BinLow,<<"\n">>),REGEX),

%methods used below consume more memory. It is not possible to run ioc2rpz with 1M indicator on AWS free tier
%L=clean_feed_bin(ioc2rpz_fun:split_tail(Bin,<<"\n">>),REGEX),
%Comment next 1 line in case of limited memory. REGEX must be prepared for lowcase sources


%TODO spawn cleanup
CTime=ioc2rpz_fun:curr_serial_60(),
%L=[ {ioc2rpz_fun:bin_to_lowcase(X),Y} || {X,Y} <- clean_feed(ioc2rpz_fun:split_tail(Bin,<<"\n">>),REGEX) ],
@@ -164,13 +157,13 @@ clean_feed(IOC,none) ->
%Default REFEX
%Extract IOCs,remove unsupported chars using standard REGEX. Expiration date is not supported;
clean_feed(IOC,[]) ->
{ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$"),
{ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$",[{newline, any}]),
[ X || X <- clean_feed(IOC,[],MP), X /= <<>>];


%Extract IOCs,remove unsupported chars using user's defined REGEX. Expiration date is supported. First value - IOC, second - Exp. Date;
clean_feed(IOC,REX) -> %REX - user's regular expression
{ok,MP} = re:compile(REX),
{ok,MP} = re:compile(REX,[{newline, any}]),
[ X || X <- clean_feed(IOC,[],MP), X /= <<>>].

clean_feed([Head|Tail],CleanIOC,REX) ->
@@ -190,11 +183,11 @@ clean_feed_bin(IOC,none) ->
[ {X,0} || X <- IOC, X /= <<>>];

clean_feed_bin(IOC,[]) ->
{ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$"),
{ok,MP} = re:compile("^([A-Za-z0-9][A-Za-z0-9\-\._]+)[^A-Za-z0-9\-\._]*.*$",[{newline, any}]),
[ X || X <- clean_feed_bin(IOC,<<>>,MP), X /= <<>>];

clean_feed_bin(IOC,REX) -> %REX - user's regular expression
{ok,MP} = re:compile(REX),
{ok,MP} = re:compile(REX,[{newline, any}]),
[ X || X <- clean_feed_bin(IOC,<<>>,MP), X /= <<>>].

clean_feed_bin([Head|Tail],CleanIOC,REX) ->
@@ -17,7 +17,7 @@
-module(ioc2rpz_db).
-include_lib("ioc2rpz.hrl").
-export([init_db/3,db_table_info/2,read_db_pkt/1,write_db_pkt/2,delete_db_pkt/1,read_db_record/3,write_db_record/3,delete_old_db_record/1,saveZones/0,loadZones/0,loadZones/1,
get_zone_info/2,clean_DB/1,save_zone_info/1,get_allzones_info/2]).
get_zone_info/2,clean_DB/1,save_zone_info/1,get_allzones_info/2, lookup_db_record/2]).


init_db(ets,DBDir,PID) ->
@@ -149,7 +149,7 @@ write_db_record(ets,Zone,IOCs,ixfr) when IOCs == [] ->
write_db_record(mnesia,Zone,IOCs,ixfr) ->
{ok,0};

write_db_record(_DBStorage,_Zone,_IOCs,_XFR) ->
write_db_record(_DBStorage,_Zone,_IOCs,_XFR) ->
{ok,0}. %non cached zones

update_db_record(ets, Zone, Serial, IOC, IOCExp, [], CTime) when IOCExp > 0,IOCExp =< CTime ->
@@ -171,6 +171,42 @@ update_db_record(ets, Zone, Serial, IOC, IOCExp, Update, CTime) -> %ok; %not new

update_db_record(mnesia, Zone, Serial, IOC, IOCExp, Update, CTime) -> ok.

%%%
%%% Lookup if an indicator is in the DB.
%%% Recurs - validate hosts/fqdns if they are blocked by a wildcard rule or a subnet.
%%%
lookup_db_record(IOC, Recurs) ->
lookup_db_record(?DBStorage, IOC, Recurs).

lookup_db_record(ets, IOC, false) ->
{ok,[{IOC,ets:select(rpz_ixfr_table,[{{{ioc,'$0',IOC},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]};

lookup_db_record(mnesia, IOC, false) ->
{ok,[{IOC,[]}]};

lookup_db_record(ets, IOC, true) ->
% check IP or domain
%ioc2rpz_fun:logMessage("Checking IOC ~s ~n",[IOC]),
{ok,MP} = re:compile("^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(\\/[0-9]{1,3})?)$|(:)"),
case re:run(IOC,MP,[global,notempty,{capture,[1],binary}]) of
{match,_} -> {ok,[{IOC,ets:select(rpz_ixfr_table,[{{{ioc,'$0',IOC},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]};
_ -> lookup_db_record(ets,IOC,<<"">>,ioc2rpz_fun:rsplit_tail(IOC, <<".">>),[])
end;

lookup_db_record(mnesia, IOC, true) ->
{ok,[{IOC,[]}]}.

lookup_db_record(ets,IOC, FQDN, [], Result) ->
%ioc2rpz_fun:logMessage("Result: ~p\n\n",[{ok,Result}]),
FResult = [{IOC2,ARR} || {IOC2,ARR} <-Result, ((IOC == IOC2) or (ARR /= []))],
{ok,FResult};

lookup_db_record(ets,IOC, FQDN, [Label|REST], Result) ->
NFQDN = if FQDN == <<"">> -> Label; true -> <<Label/binary,".",FQDN/binary>> end,
%ioc2rpz_fun:logMessage("Checking ~p ~n",[NFQDN]),
lookup_db_record(ets, IOC, NFQDN, REST, Result ++ [{NFQDN,ets:select(rpz_ixfr_table,[{{{ioc,'$0',NFQDN},'$2','$3'},[],[{{'$0','$2','$3'}}]}])}]).


delete_old_db_record(Zone) ->
delete_old_db_record(?DBStorage,Zone).

@@ -17,7 +17,7 @@
-module(ioc2rpz_fun).
-include_lib("eunit/include/eunit.hrl").
-include_lib("ioc2rpz.hrl").
-export([logMessage/2,logMessageCEF/2,strs_to_binary/1,curr_serial/0,curr_serial_60/0,constr_ixfr_url/3,ip_to_bin/1,read_local_actions/1,split_bin_bytes/2,split_tail/2,
-export([logMessage/2,logMessageCEF/2,strs_to_binary/1,curr_serial/0,curr_serial_60/0,constr_ixfr_url/3,ip_to_bin/1,read_local_actions/1,split_bin_bytes/2,split_tail/2,rsplit_tail/2,
bin_to_lowcase/1,ip_in_list/2,intersection/2,bin_to_hexstr/1,conv_to_Mb/1,q_class/1,q_type/1,split/2,msg_CEF/1,base64url_decode/1]).

logMessage(Message, Vars) ->
@@ -164,6 +164,14 @@ split_tail(String, Pattern) ->
[] -> []
end.

rsplit_tail(String, Pattern) ->
% ioc2rpz_fun:logMessage("z_split ~p ~p ~n",[String, Pattern]),
case binary:split(String, Pattern) of %binary:split
[First, Second] -> rsplit_tail(Second, Pattern) ++ [First];
[First] -> [First];
[] -> []
end.

bin_to_lowcase(A) ->
<< << (b_to_lowcase(C)) >> || << C >> <= A >>.
% << << C >> || << C >> <= A >>.
@@ -289,3 +297,13 @@ base64url_decode_test() -> [
?assert(base64url_decode(<<"AAABAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE">>) =:= {ok,<<0,0,1,0,0,1,0,0,0,0,0,0,7,101,120,97,109,112,108,101,3,99,111,109,0,0,1,0,1>>}),
?assert(base64url_decode(<<"AAABAAABAAAAAAAAB2V4YW1wbGUDY29tAAABAAE==">>) =:= {error,<<>>})
].

bin_to_lowcase_test() ->[
?assert(bin_to_lowcase(<<"fC00::01">>) =:= <<"fc00::01">>),
?assert(bin_to_lowcase(<<"Aaaaaa">>) =:= <<"aaaaaa">>),
?assert(bin_to_lowcase(<<"bBbBbB">>) =:= <<"bbbbbb">>),
?assert(bin_to_lowcase(<<"ccC">>) =:= <<"ccc">>),
?assert(bin_to_lowcase(<<"D">>) =:= <<"d">>),
?assert(bin_to_lowcase(<<"f">>) =:= <<"f">>),
?assert(bin_to_lowcase(<<"eeeeeeeeeeeeeeeeeeeeeee">>) =:= <<"eeeeeeeeeeeeeeeeeeeeeee">>)
].
@@ -199,6 +199,8 @@ srv_mgmt(Req, State, Format) when State#state.op == get_rpz -> % Get RPZ
%
%% need to validate TSIG on the access to the feeds
%
% ioc2rpz_db:lookup_db_record(<<"baddomain1.com">>,no).
% ioc2rpz_db:lookup_db_record(<<"99.98.61.5">>,no).
% DB
%11> ets:select(rpz_ixfr_table,[{{{ioc,'$0',<<"99.98.61.5">>},'$2','$3'},[],[{{'$0','$2','$3'}}]}]).
%[{<<5,108,111,99,97,108,7,105,111,99,50,114,112,122,0>>,
@@ -215,6 +217,21 @@ srv_mgmt(Req, State, Format) when State#state.op == get_rpz -> % Get RPZ
% {Body, Req0, State};


srv_mgmt(Req, State, Format) when State#state.op == get_ioc -> % check IoC
#{peer := {IP, Port}} = Req,
ioc2rpz_fun:logMessageCEF(ioc2rpz_fun:msg_CEF(230),[ioc2rpz:ip_to_str(IP), Port, cowboy_req:path(Req), ""]),
IOC = ioc2rpz_fun:bin_to_lowcase(cowboy_req:binding(ioc, Req)),
TKEY = try
maps:get(tkey,cowboy_req:match_qs([tkey],Req)) %%%%% parse_qs
catch _:_ ->
<<"">>
end,
{Recur, Zones} = get_tkey_zones(TKEY),
%ioc2rpz_fun:logMessage("Recursion: ~p\nZones: ~p\n\n",[Recur,Zones]),
Body=format_ioc(ioc2rpz_db:lookup_db_record(IOC,Recur),{IOC,TKEY, Zones},Format),
{Body, Req, State};


srv_mgmt(Req, State, Format) when State#state.op == catch_all -> % Catch all unsupported requests from authenticated users
#{peer := {IP, Port}} = Req,
ioc2rpz_fun:logMessageCEF(ioc2rpz_fun:msg_CEF(137),[ioc2rpz:ip_to_str(IP), Port, cowboy_req:path(Req), ""]),
@@ -232,6 +249,64 @@ srv_mgmt(Req, State, Format) when State#state.op == catch_all -> % Catch all uns
rest_terminate(Req, State) ->
ok.

format_ioc({ok,Results},Req,Format) ->
format_ioc(Results,Req,Format,"");

format_ioc({error,_Results},{IOC,_TKEY,_Zones},json) ->
io_lib:format("{\"status\":\"error\", \"ioc\": ~p}",[IOC]);

format_ioc({error,_Results},{IOC,_TKEY,_Zones},txt) ->
io_lib:format("status: error\nIOC: ~p\n",[IOC]).

format_ioc([],{IOC,TKEY,_Zones},json, Result) ->
io_lib:format("{\"ioc\":\"~s\", \"tkey\":\"~s\", \"data\":[~s]}\n\n",[IOC,TKEY,Result]);

format_ioc([{El,Feeds}|Results],Req,json,"") ->
Ind=io_lib:format("{\"ioc\": \"~s\", \"feeds\": ~s}",[El, parse_feeds(Feeds,Req,"",json)]),
format_ioc(Results,Req,json, Ind);

format_ioc([{El,Feeds}|Results],Req,Format,Result) ->
Ind=io_lib:format("{\"ioc\": \"~s\", \"feeds\": ~s}",[El, parse_feeds(Feeds,Req,"",json)]),
format_ioc(Results,Req,json, Result ++","++ Ind).


parse_feeds([],_Req,Result,json) ->
"["++Result++"]";

parse_feeds([{Feed, Serial, Exp}|REST],{_IOC,_TKEY,Zones}=Req,"",json) ->
Memb=maps:is_key(Feed,Zones),
Feed_Str=if (Memb) -> {FN,TY,WC}=maps:get(Feed,Zones), io_lib:format("{\"feed\":~p, \"wildcard\":~s, \"type\":~p, \"rpz_serial\": ~p, \"ioc_expiration\": ~p}",[FN, WC, binary_to_list(TY), Serial, Exp]); true -> "" end,
parse_feeds(REST,Req,Feed_Str,json);

parse_feeds([{Feed, Serial, Exp}|REST],{_IOC,_TKEY,Zones}=Req,Result,json) ->
Memb=maps:is_key(Feed,Zones),
Feed_Str=if (Memb) -> {FN,TY,WC}=maps:get(Feed,Zones), ","++io_lib:format("{\"feed\":~p, \"wildcard\":~s, \"type\":~p, \"rpz_serial\": ~p, \"ioc_expiration\": ~p}",[FN, WC, binary_to_list(TY), Serial, Exp]); true -> "" end,
parse_feeds(REST,Req,Result++Feed_Str,json).

%%%
%%% Get zones availble for TKey
%%%
get_tkey_zones(TKey) ->
{ok, TKeyBin} = ioc2rpz:domstr_to_bin(TKey,0),
Groups = [ X || [X,Y] <- ets:match(cfg_table,{[key_group,'$1',TKeyBin],'$3'}) ],
get_tkey_zones(TKeyBin, Groups, [ X || [X] <- ets:match(cfg_table,{[rpz,'_'],'_','$4'}) ], []). %{X#rpz.zone, X#rpz.zone_str, X#rpz.wildcards, X#rpz.akeys, X#rpz.ioc_type, X#rpz.key_groups}

get_tkey_zones(TKeyBin, _Groups,[], Zones) ->
Recur = [ X || {_,{_,_,X}} <- Zones, X == <<"true">> ] /= [],
% ZNames = [ X || {X,{_,_,_}} <- Zones ],
{Recur, maps:from_list(lists:flatten(Zones))};

get_tkey_zones(TKeyBin, Groups, [RPZ|Rest], Zones) ->
KZ = lists:member(TKeyBin, RPZ#rpz.akeys),
GZ = [X || X <- Groups, lists:member(X,RPZ#rpz.key_groups)],
AZ=case {KZ,GZ,TKeyBin} of
{true,_,_} -> [{RPZ#rpz.zone, {RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
{_,Gr,_} when Gr /= [] -> [{RPZ#rpz.zone, {RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
{_,_,<<0,0>>} -> [{RPZ#rpz.zone,{RPZ#rpz.zone_str, RPZ#rpz.ioc_type, RPZ#rpz.wildcards}}];
_Else -> []
end,
get_tkey_zones(TKeyBin, Groups, Rest, Zones ++ AZ).

gen_rpz_stats() ->
[ [{"name",X#rpz.zone_str},{"rule_count",X#rpz.rule_count},{"ioc_count",X#rpz.ioc_count},{"serial",X#rpz.serial},{"serial_ixfr",X#rpz.serial_ixfr},{"update_time",X#rpz.update_time},{"ixfr_update_time",X#rpz.ixfr_update_time},{"ixfr_nz_update_time",X#rpz.ixfr_nz_update_time}] || [X] <- ets:match(cfg_table,{[rpz,'_'],'_','$2'}), X#rpz.rule_count /= undefined].

0 comments on commit b39e007

Please sign in to comment.
You can’t perform that action at this time.