Permalink
Browse files

Merge PR #1183 (v2017.08 release) into master

  • Loading branch information...
eugeneia committed Aug 30, 2017
2 parents 8960b0f + cc8c0ac commit 707db7f02921431b770ce7307234476d667eb93d
Showing with 7,775 additions and 487 deletions.
  1. +0 −1 .gitignore
  2. +1 −1 .version
  3. +5 −0 lib/ljsyscall/syscall/linux/ioctl.lua
  4. +1 −1 src/Makefile
  5. +6 −0 src/apps/intel_mp/README.md
  6. +21 −8 src/apps/intel_mp/intel_mp.lua
  7. +119 −0 src/apps/ipfix/README.md
  8. +3,618 −0 src/apps/ipfix/ipfix-information-elements.inc
  9. +575 −0 src/apps/ipfix/ipfix.lua
  10. +383 −0 src/apps/ipfix/template.lua
  11. +59 −0 src/apps/ipv4/README.md
  12. +268 −0 src/apps/ipv4/arp.lua
  13. +13 −0 src/apps/ipv6/README.md
  14. +78 −47 src/apps/ipv6/nd_light.lua
  15. +0 −153 src/apps/lwaftr/arp.lua
  16. +8 −1 src/apps/lwaftr/fragmentv4_test.lua
  17. +1 −72 src/apps/lwaftr/ipv4_apps.lua
  18. +1 −0 src/apps/lwaftr/ipv6_apps.lua
  19. +6 −0 src/apps/lwaftr/nh_fwd.lua
  20. +5 −6 src/apps/socket/unix.lua
  21. +56 −2 src/apps/tap/README.md
  22. +139 −15 src/apps/tap/tap.lua
  23. +60 −26 src/apps/vlan/README.md
  24. +131 −63 src/apps/vlan/vlan.lua
  25. +5 −2 src/apps/wall/l7fw.lua
  26. +4 −0 src/doc/genbook.sh
  27. +20 −5 src/lib/ctable.lua
  28. +92 −77 src/lib/ipc/shmem/iftable_mib.lua
  29. +4 −0 src/lib/ipsec/esp.lua
  30. +53 −0 src/lib/lpm/README.md
  31. +27 −0 src/lib/lpm/build_fixtures.pl
  32. +131 −0 src/lib/lpm/ip4.lua
  33. +116 −0 src/lib/lpm/ip6.lua
  34. +140 −0 src/lib/lpm/lpm.lua
  35. +264 −0 src/lib/lpm/lpm4.lua
  36. +12 −0 src/lib/lpm/lpm4_248.c
  37. +91 −0 src/lib/lpm/lpm4_248.lua
  38. +31 −0 src/lib/lpm/lpm4_dxr.c
  39. +183 −0 src/lib/lpm/lpm4_dxr.lua
  40. +194 −0 src/lib/lpm/lpm4_poptrie.lua
  41. +351 −0 src/lib/lpm/lpm4_trie.lua
  42. +44 −0 src/lib/lpm/random.dasl
  43. +1 −0 src/lib/pmu.lua
  44. +18 −0 src/lib/pmu_x86.dasl
  45. +2 −1 src/lib/protocol/README.md
  46. +3 −0 src/lib/protocol/gre.lua
  47. +4 −1 src/lib/protocol/header.lua
  48. +1 −0 src/lib/protocol/icmp/nd/header.lua
  49. +2 −0 src/lib/protocol/icmp/nd/options/tlv.lua
  50. +1 −0 src/lib/protocol/ipv4.lua
  51. +3 −0 src/lib/protocol/ipv6.lua
  52. +3 −0 src/lib/protocol/keyed_ipv6_tunnel.lua
  53. +1 −0 src/lib/protocol/tcp.lua
  54. +6 −0 src/program/ipfix/README
  55. +1 −0 src/program/ipfix/README.inc
  56. +25 −0 src/program/ipfix/README.md
  57. +18 −0 src/program/ipfix/ipfix.lua
  58. +32 −0 src/program/ipfix/probe/README
  59. +1 −0 src/program/ipfix/probe/README.inc
  60. +198 −0 src/program/ipfix/probe/probe.lua
  61. +43 −0 src/program/ipfix/tests/bench.sh
  62. +57 −0 src/program/ipfix/tests/collector-test.sh
  63. +33 −0 src/program/ipfix/tests/generate_packets.py
  64. +6 −5 src/program/lwaftr/setup.lua
View
@@ -20,4 +20,3 @@ __pycache__
/src/programs.inc
.images
/lib/luajit/usr
/src/core/version.lua
View
@@ -1 +1 @@
2017.07
2017.08
@@ -189,6 +189,11 @@ local ioctl = strflag {
SIOCDELRT = 0x890C,
SIOCRTMSG = 0x890D,
SIOCGIFFLAGS = 0x8913,
SIOCSIFFLAGS = 0x8914,
SIOCGIFMTU = 0x8921,
SIOCSIFMTU = 0x8922,
SIOCGIFHWADDR = 0x8927,
SIOCGIFINDEX = 0x8933,
SIOCDARP = 0x8953,
View
@@ -135,7 +135,7 @@ $(PFLUAOBJ): obj/%_lua.o: ../lib/pflua/src/%.lua Makefile
$(COBJ): obj/%_c.o: %.c $(CHDR) Makefile | $(OBJDIR)
$(E) "C $@"
$(Q) $(CC) $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
$(Q) $(CC) $(DEBUG) -O3 -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
obj/arch/avx2_c.o: arch/avx2.c Makefile
$(E) "C(AVX2) $@"
@@ -49,6 +49,12 @@ light or not. The default is `false`.
*Optional* Number of seconds `new` waits for the device to come up. The default
is 120.
— Key **linkup_wait_recheck**
*Optional* If the `linkup_wait` option is true, the number of seconds
to sleep between checking the link state again. The default is 0.1
seconds.
— Key **mtu**
*Optional* The maximum packet length sent or received, excluding the trailing
@@ -256,6 +256,7 @@ Intel = {
rxq = {},
mtu = {default=9014},
linkup_wait = {default=120},
linkup_wait_recheck = {default=0.1},
wait_for_link = {default=false},
master_stats = {default=true},
run_stats = {default=false}
@@ -289,6 +290,8 @@ function Intel:new (conf)
rxq = conf.rxq,
mtu = conf.mtu or self.config.mtu.default,
linkup_wait = conf.linkup_wait or self.config.linkup_wait.default,
linkup_wait_recheck =
conf.linkup_wait_recheck or self.config.linkup_wait_recheck.default,
wait_for_link = conf.wait_for_link
}
@@ -351,6 +354,15 @@ end
function Intel:disable_interrupts ()
self.r.EIMC(0xffffffff)
end
function Intel:wait_linkup (timeout)
if timeout == nil then timeout = self.linkup_wait end
if self:link_status() then return true end
for i=1,math.max(math.floor(timeout/self.linkup_wait_recheck), 1) do
C.usleep(math.floor(self.linkup_wait_recheck * 1e6))
if self:link_status() then return true end
end
return false
end
function Intel:init_rx_q ()
if not self.rxq then return end
assert((self.rxq >=0) and (self.rxq < self.max_q),
@@ -755,11 +767,7 @@ function Intel1g:init ()
self.r.CTRL_EXT:set( bits { AutoSpeedDetect = 12, DriverLoaded = 28 })
self.r.RLPML(self.mtu + 4) -- mtu + crc
self:unlock_sw_sem()
for i=1, math.floor(self.linkup_wait/2) do
if self:link_status() then break end
if not self.wait_for_link then break end
C.usleep(2000000)
end
if self.wait_for_link then self:wait_linkup() end
end
function Intel1g:link_status ()
@@ -865,7 +873,12 @@ function Intel82599:init ()
pci.set_bus_master(self.pciaddress, true)
pci.disable_bus_master_cleanup(self.pciaddress)
for i=1,math.floor(self.linkup_wait/2) do
-- The 82599 devices sometimes just don't come up, especially when
-- there is traffic already on the link. If 2s have passed and the
-- link is still not up, loop and retry.
local reset_timeout = math.max(self.linkup_wait_recheck, 2)
local reset_count = math.max(math.floor(self.linkup_wait / reset_timeout), 1)
for i=1,reset_count do
self:disable_interrupts()
local reset = bits{ LinkReset=3, DeviceReset=26 }
self.r.CTRL(reset)
@@ -880,9 +893,9 @@ function Intel82599:init ()
self.r.AUTOC2(0)
self.r.AUTOC2:set(bits { tenG_PMA_PMD_Serial = 17 })
self.r.AUTOC:set(bits{restart_AN=12})
C.usleep(2000000)
if self:link_status() then break end
if not self.wait_for_link then break end
if self:wait_linkup(reset_timeout) then break end
end
-- 4.6.7
View
@@ -0,0 +1,119 @@
# IPFIX and NetFlow apps
## IPFIX (apps.ipfix.ipfix)
The `IPFIX` app implements an RFC 7011 IPFIX "meter" and "exporter"
that records the flows present in incoming traffic and sends exported
UDP packets describing those flows to an external collector (not
included). The exporter can produce output in either the standard RFC
7011 IPFIX format, or the older NetFlow v9 format from RFC 3954.
DIAGRAM: IPFIX
+-----------+
input ---->* IPFIX *----> output
+-----------+
See the `snabb ipfix probe` command-line interface for a program built
using this app.
### Configuration
The `IPFIX` app accepts a table as its configuration argument. The
following keys are defined:
— Key **idle_timeout**
*Optional*. Number of seconds after which a flow should be considered
idle and available for expiry. The default is 300 seconds.
— Key **active_timeout**
*Optional*. Period at which an active, non-idle flow should produce
export records. The default is 120 seconds.
— Key **cache_size**
*Optional*. Initial size of flow tables, in terms of number of flows.
The default is 20000.
— Key **template_refresh_interval**
*Optional*. Period at which to send template records over UDP. The
default is 600 seconds.
— Key **ipfix_version**
*Optional*. Version of IPFIX to export. 9 indicates legacy NetFlow
v9; 10 indicates RFC 7011 IPFIX. The default is 10.
— Key **mtu**
*Optional*. MTU for exported UDP packets. The default is 512.
— Key **observation_domain**
*Optional*. Observation domain tag to attach to all exported packets.
The default is 256.
— Key **exporter_ip**
*Required*, sadly. The IPv4 address from which to send exported UDP
packets.
— Key **collector_ip**
*Required*. The IPv4 address to which to send exported UDP packets.
— Key **collector_port**
*Required*. The port on which the collector is listening for UDP
packets.
— Key **templates**
*Optional*. The templates for flows being collected. See the source
code for more information.
### To-do list
Some ideas for things to hack on are below.
#### Limit the number of flows
As it is, if an attacker can create millions of flows, then our flow
set will expand to match (and never shrink). Perhaps we should cap
the total size of the flow table.
#### Look up multiple keys in parallel
For large ctables, we can only do 7 or 8 million lookups per second if
we look up one key after another. However if we do lookups in
parallel, then we can get 15 million or so, which would allow us to
reach 10Gbps line rate on 64-byte packets.
#### YANG schema to define IPFIX app configuration
We should try to model the configuration of the IPFIX app with a YANG
schema. See RFC 6728 for some inspiration.
#### Use special-purpose internal links
The links that we use as internal buffers between parts of the IPFIX
app have some overhead as they have to update counters. Perhaps we
should use a special-purpose data structure.
#### Use a monotonic timer
Currently internal flow start and end times use UNIX time. This isn't
great for timers, but it does match what's specified in RFC 7011.
Could we switch to monotonic time?
#### Allow export to IPv6 collectors
We can collect IPv6 flows of course, but we only export to collectors
over IPv4 for the moment.
#### Allow packets to count towards multiple templates
Right now, routing a packet towards a flow set means no other flow set
can measure that packet. Perhaps this should change.
Oops, something went wrong.

0 comments on commit 707db7f

Please sign in to comment.