ioc2rpz is a place where threat intelligence meets DNS

ChangeLog.md

@@ -1,7 +1,7 @@
# ioc2rpz change log
[CB] - Changed Behaviour

## 2019-03-01 v0.9.3.0
## 2019-03-10 v0.9.3.0
- REST API
- added rebar3 to manage dependencies

Dockerfile

@@ -18,18 +18,22 @@ FROM erlang:alpine
MAINTAINER Vadim Pavlov<ioc2rpz@gmail.com>
WORKDIR /opt/ioc2rpz

RUN mkdir /opt/ioc2rpz/ebin /opt/ioc2rpz/cfg /opt/ioc2rpz/db /opt/ioc2rpz/include /opt/ioc2rpz/src /opt/ioc2rpz/scripts /opt/ioc2rpz/log && apk add bind-tools curl python3
ADD ebin/ioc2rpz.app /opt/ioc2rpz/ebin/
#RUN mkdir /opt/ioc2rpz/ebin /opt/ioc2rpz/cfg /opt/ioc2rpz/db /opt/ioc2rpz/include /opt/ioc2rpz/src /opt/ioc2rpz/scripts /opt/ioc2rpz/log && apk add bind-tools curl python3
#ADD ebin/ioc2rpz.app /opt/ioc2rpz/ebin/
#ADD scripts/* /opt/ioc2rpz/scripts/
#ADD ioc2rpz_app.config  /opt/ioc2rpz/
#RUN erlc -I include/ -o ebin/ src/*.erl
#ENTRYPOINT ["erl", "-noshell", "-pa", "./ebin", "-sname", "ioc2rpz", "-eval", "application:start(ioc2rpz,permanent)", "-config", "ioc2rpz_app"]
#CMD ["/bin/sh", "/opt/ioc2rpz/scripts/run_ioc2rpz.sh"]

RUN mkdir /opt/ioc2rpz/cfg /opt/ioc2rpz/db /opt/ioc2rpz/include /opt/ioc2rpz/src /opt/ioc2rpz/log && apk add bind-tools curl python3
ADD src/* /opt/ioc2rpz/src/
ADD include/* /opt/ioc2rpz/include/
ADD scripts/* /opt/ioc2rpz/scripts/
ADD ioc2rpz_app.config  /opt/ioc2rpz/

RUN erlc -I include/ -o ebin/ src/*.erl
RUN rebar3 release -d false

VOLUME ["/opt/ioc2rpz/cfg", "/opt/ioc2rpz/db"]

EXPOSE 53/tcp 53/udp 853/tcp

#ENTRYPOINT ["erl", "-noshell", "-pa", "./ebin", "-sname", "ioc2rpz", "-eval", "application:start(ioc2rpz,permanent)", "-config", "ioc2rpz_app"]
CMD ["/bin/sh", "/opt/ioc2rpz/scripts/run_ioc2rpz.sh"]
ENTRYPOINT ["CD=/opt/ioc2rpz/", "DB=/opt/ioc2rpz/db", "/opt/ioc2rpz/_build/default/rel/ioc2rpz/bin/ioc2rpz", "foreground"]

TODO.md

@@ -1,33 +1,63 @@
#TODO
- [ ] Dockerfile because of rebar3

## Core
- [ ] Rebar3 update
  - [x] update Dockerfile and startup script
  - [x] node name
  - [ ] remove ioc2rpz_app.config, ebin/ioc2rpz.app, scripts/run_ioc2rpz.sh
  - [x] add config/*


## Core / DNS
- [ ] DoH https://tools.ietf.org/html/rfc8484
- [ ] DoD https://tools.ietf.org/html/draft-ietf-dprive-dnsodtls-06
- [ ] Redo AXFR logs
- [ ] Mnesia for storage (and auto creation)
- [ ] Distributed configuration
- [ ] Wait while a remote server confirms receiving a notification
- [ ] Access to the hotcache and the cfg_table via FUNs
- [ ] (1) Clean up the code & add comments
- [ ] (2) EDNS0 Support: DNS Cookie, edns-tcp-keepalive, NSID
- [ ] (3) Memory optimization for huge zones (erl -pa ebin +MEas bf ?????)
- [ ] (*) saveZones - doesn't correctly save zones if there a lot of updates. Save strategy based on update size and time and currently running updates.
- [ ] Logs level startup config
- [ ] Check delete in ioc2rpz: rpz_hotcache_table/pkthotcache
- [ ] (1) Terminate updating zones during config reload

## Sources
- [ ] Source: ioc type, max # of IOCs, max file size, RPZ action, NS type, lowcase optimization option, spawn processes
- [ ] ioc type
- [x] max # of IOCs
- [ ] max file size
- [ ] RPZ action
- [ ] NS type
- [ ] lowcase optimization option
  - [ ] (1) IOC to lowercase - check memory usage impact (in ioc2rpz_conn)
- [ ] spawn processes
- [ ] Hot cache optimization depending on RPZ refresh time and source usage in multiple feeds
- [ ] Cache optimization for huge zones
- [ ] Statistics table
- [ ] Max # of IOCs
- [ ] Add script for RPZ via "shell:"
- [ ] Add source PostreSQL, MySQL via "shell:"
- [ ] RPZ action per source
- [ ] Max IOCs, current IOCs
- [ ] (2) Source based on files check by mod.date and size -> read by chunks
- [ ] Retry if source is not available
- [ ] Simultanious source downloads


## RPZ
- [ ] RPZ only by source intersections
- [ ] RPZ by source intersection
- [ ] Max # of IOCs
- [ ] Catalog zones
- [ ] (2) FDateTime,ToDateTime,FDateTimeZ,ToDateTimeZ + support them for AXFR  
[:FDateTime:] = "2017-10-13 13:13:13", [:FDateTimeZ:] = "2017-10-13T13:13:13Z"  
[:ToDateTime:] = "2017-10-13 13:13:13", [:ToDateTimeZ:] = "2017-10-13T13:13:13Z"
- [ ] Statistics per zone (# records, last update, # AXFR, # IXFR, last axfr update time, avg axfr update time, last ixfr update time, avg ixfr update time)
- [ ] RPZ behavior: ignore unreachable sources, use old data for unreachable sources, do not update the zone
- [ ] Additional local records: ptr, srv, mx etc
- [ ] An action per source: {"",action,locdata} //default action ,{"source_name",action,locdata}
- [ ] RPZ transfer rate limiting


## Servers
- [ ] Max # of IOCs
- [ ] Enforcement max # of IOCs
- [ ] Secondary DNS via MNESIA

## REST
@@ -43,15 +73,11 @@

## Management
- [ ] DNS health check requests
- [ ] (2) MGMT via DNS move to a separate port/IP
- [ ] Statistics per zone (# records, last update, # AXFR, # IXFR, last axfr update time, avg axfr update time, last ixfr update time, avg ixfr update time)
- [ ] By default disable MGMT via DNS (update ioc2rpz.gui first)


## Unsorted
- [x] (1) http/https/ftp errors handling - source status in the record. If a source is not available - work w/o it
  - [ ] RPZ behavior: ignore unreachable sources, use old data for unreachable sources, do not update the zone
  - [ ] (2) Source based on files check by mod.date and size -> read by chunks
  - [ ] Retry if source is not available
- [ ] Performance testing vs bind:
  - [ ] 1 core/8GB RAM: start time, zone transfer time, zone size, CPU, Memory
    - [ ] 100k rules
@@ -61,26 +87,10 @@
    - [ ] 100k rules
    - [ ] 1M rules
    - [ ] 10M rules
- [ ] Mnesia for storage (and auto creation)
- [ ] Distributed configuration
- [ ] Wait while a remote server confirms receiving a notification
- [ ] Additional local records: ptr, srv, mx etc
- [ ] An action per source: {"",action,locdata} //default action ,{"source_name",action,locdata}
- [ ] Switch from IXFR cache to Sources cache. IXFR cache allows you to support less zone updates but IOCs can be stored multiple times. Sources cache will contain duplicate IOCs from the same source but RPZs will be updated more frequently (looks like it is not bad).
  - [ ] (3) Share IOC between the feeds in IXFR table (do not forget about different whitelists)
- [ ] Access to the hotcache and the cfg_table via FUNs
- [ ] DNS over TLS https://tools.ietf.org/html/rfc8310
- [ ] Simultanious source downloads

## Other/optimization TODO
- [ ] (1) Do not cache expired IOCs if ExpDateTime<Serial_IXFR / update ExpDateTime if exists
- [ ] (1) Check zones IXFR update from multiple sources
- [ ] (1) Clean up the code & add comments
- [ ] (2) EDNS0 Support: DNS Cookie, edns-tcp-keepalive, NSID
- [ ] (1) IOC to lowercase - check memory usage impact (in ioc2rpz_conn)
- [ ] (2) UDP & TableMGMT under supervisors
- [ ] (3) Memory optimization for huge zones (erl -pa ebin +MEas bf ?????)
- [ ] (*) saveZones - doesn't correctly save zones if there a lot of updates. Save strategy based on update size and time and currently running updates.
- [ ] Logs level startup config
- [ ] Check delete in ioc2rpz: rpz_hotcache_table/pkthotcache
- [ ] (1) Terminate updating zones during config reload
- [x] (2) UDP & TableMGMT under supervisors

config/sys.config.src

@@ -0,0 +1 @@
[{ioc2rpz,[{ipv4,"${IPv4}"},{ipv6,"${IPv6}"},{conf_file,"${CONF}"},{db_dir,"${DB}"},{cd,"${CD}"}]}].

config/vm.args

@@ -0,0 +1,8 @@
## Name of the node
#-name {{release_name}}@127.0.0.1
#-sname {{release_name}}
-sname ${NODE_NAME}
-proto_dist inet_tls

## Cookie for distributed erlang
#-setcookie {{release_name}}

include/ioc2rpz.hrl

@@ -22,6 +22,8 @@
-define(PortTLS,853). %DoT Port
-define(PortREST,8443). %REST Port
-define(TTL,900). %Default record TTL
-define(DefConf,"./cfg/ioc2rpz.conf"). %Default configuration
-define(DefDB,"./db"). %Default DB location

%-define(logTS, true). % Log timestamps (comment or uncomment)
-define(debug, true). % Log debug messages

rebar.config

@@ -1,7 +1,21 @@
{minimum_otp_vsn, "21.0"}.
{erl_opts, [debug_info]}.
{deps, [{cowboy, "2.5.0"}]}.

{shell, [
  % {config, "config/sys.config"},
    {sys_config_src, "config/sys.config.src"},
    {vm_args_src, "config/vm.args"},
    {apps, [ioc2rpz]}
]}.


{relx, [
  {release, {ioc2rpz, {cmd, "grep ioc2rpz_ver include/ioc2rpz.hrl | awk -F'\"' '{printf $2}'"}}, [ioc2rpz]},

  {dev_mode, true},
  {include_erts, false},
  {sys_config_src, "config/sys.config.src"},
  {vm_args_src, "config/vm.args"},

  {extended_start_script, true}
]}.

src/ioc2rpz.app.src

@@ -33,8 +33,10 @@
  {applications, [
                  kernel,
                  stdlib,
                  inets,
                  ssl,
                  sasl,
                  cowboy
                 ]},
  {mod, { ioc2rpz_app, []}} %, %["","./cfg/ioc2rpz.conf","./db"]
  %{env, []}
  {mod, { ioc2rpz_app, []}}
 ]}.

src/ioc2rpz_app.erl

@@ -16,13 +16,26 @@
-module(ioc2rpz_app).
-behaviour(application).
-export([start/2, stop/1]).
-include_lib("ioc2rpz.hrl").

start(_StartType, _Start_Args) ->
    IPv4=application:get_env(ioc2rpz, ipv4, ""),
    IPv6=application:get_env(ioc2rpz, ipv6, ""),
    Conf_File=application:get_env(ioc2rpz, conf_file, "./cfg/ioc2rpz.conf"),
    DB=application:get_env(ioc2rpz, db_dir, "./db"),
    IPv4=get_env(ioc2rpz, ipv4, ""),
    IPv6=get_env(ioc2rpz, ipv6, ""),
    Conf_File=get_env(ioc2rpz, conf_file, ?DefConf),
    DB=get_env(ioc2rpz, db_dir, ?DefDB),
    {ok, CWD} = file:get_cwd(),
    Dir=get_env(ioc2rpz, cd, CWD),
    file:set_cwd(Dir),
    io:format("Env ip4: ~p ip6: ~p conf: ~p db: ~p cwd: ~p ~n",[IPv4,IPv6,Conf_File,DB,Dir]),
    ioc2rpz_sup:start_ioc2rpz_sup([IPv4,IPv6,Conf_File,DB]).

stop(_State) ->
    ok.
    ok.


get_env(App, Param, Default) ->
  case application:get_env(App, Param) of
    undefined   ->  Default;
    {ok, []}    ->  Default;
    {ok, X}     ->  X
  end.