Permalink
Browse files

throttling per dhcp server rather than per client

Summary:
In light of what we have recently seen in production I think we should change throttling to be per DHCP KEA server rather than per client connection to the LB.
The throttling is made to protect KEA servers after all.

Reviewed By: mkaczanowski

Differential Revision: D4021501

fbshipit-source-id: 59845bc162808c282be03181b025c235c69e010e
  • Loading branch information...
pallotron authored and Facebook Github Bot committed Oct 16, 2016
1 parent 21754ef commit fefe29675aae850f8e2591c2b698c9c09486b9bc
Showing with 31 additions and 28 deletions.
  1. +2 −2 config.json
  2. +7 −5 docs/getting-started.md
  3. +3 −3 lib/config.go
  4. +17 −16 lib/handler.go
  5. +2 −2 lib/server.go
View
@@ -11,7 +11,7 @@
"rc_ratio": 0,
"throttle_cache_size": 1024,
"throttle_cache_rate": 128,
"throttle_rate_per_conn": 256
"throttle_rate": 256
},
"v6": {
"version": 6,
@@ -25,6 +25,6 @@
"rc_ratio": 0,
"throttle_cache_size": 1024,
"throttle_cache_rate": 128,
"throttle_rate_per_conn": 256
"throttle_rate": 256
}
}
View
@@ -19,7 +19,7 @@ Configuration is provided to the program via a JSON file
"rc_ratio": 0, // what percentage of requests should go to RC servers
"throttle_cache_size": 1024, // cache size for number of throttling objects for unique clients
"throttle_cache_rate": 128, // rate value for throttling cache invalidation (per second)
"throttle_rate_per_conn": 256 // rate value for request per client (per second)
"throttle_rate": 256 // rate value for request per second
},
... (same options for "v6") ...
```
@@ -59,13 +59,15 @@ servers returned by the `GetServersFromTier(tier string)` function of the
Throttling
----------
`dhcplb` keeps track of the request rate per second for each client.
It can be set through `throttle_rate_per_conn` configuration parameter.
`dhcplb` keeps track of the request rate per second for each backend DHCP
server.
It can be set through `throttle_rate` configuration parameter.
Requests exceeding this limit will be logged and dropped. For 0 or negative
values no throttling will be done, and no cache will be created.
An LRU cache is used to keep track of rate information for each client. Cache
size can be set through `throttle_cache_size`. To prevent fast cache
An LRU cache is used to keep track of rate information for each backend DHCP
server.
Cache size can be set through `throttle_cache_size`. To prevent fast cache
invalidation from malicious clients, `dhcplb` also keeps track of the number of
new clients being added to the cache (per second). This behavior can be set
through `throttle_cache_rate` configuration parameter. For 0 or negative values
View
@@ -43,7 +43,7 @@ type Config struct {
Extras interface{}
TCacheSize int
TCacheRate int
TRatePerConn int
TRate int
}
// Override represents the dhcp server or the group of dhcp servers (tier) we
@@ -181,7 +181,7 @@ type configSpec struct {
Extras json.RawMessage `json:"extras"`
TCacheSize int `json:"throttle_cache_size"`
TCacheRate int `json:"throttle_cache_rate"`
TRatePerConn int `json:"throttle_rate_per_conn"`
TRate int `json:"throttle_rate"`
}
type combinedconfigSpec struct {
@@ -278,7 +278,7 @@ func newConfig(spec *configSpec, overrides map[string]Override, provider ConfigP
Extras: extras,
TCacheSize: spec.TCacheSize,
TCacheRate: spec.TCacheRate,
TRatePerConn: spec.TRatePerConn,
TRate: spec.TRate,
}, nil
}
View
@@ -45,14 +45,6 @@ func handleConnection(conn *net.UDPConn, config *Config, logger loggerHelper, bu
return
}
// Check for connection rate per IP address
ok, err := throttle.OK(peer.IP.String())
if !ok {
bufPool.Put(buffer)
logger.LogErr(time.Now(), nil, nil, peer, ErrConnRate, err)
return
}
go func() {
defer func() {
// always release this routine's buffer back to the pool
@@ -68,9 +60,9 @@ func handleConnection(conn *net.UDPConn, config *Config, logger loggerHelper, bu
}()
if config.Version == 4 {
handleRawPacketV4(logger, config, buffer[:bytesRead], peer)
handleRawPacketV4(logger, config, buffer[:bytesRead], peer, throttle)
} else if config.Version == 6 {
handleRawPacketV6(logger, config, buffer[:bytesRead], peer)
handleRawPacketV6(logger, config, buffer[:bytesRead], peer, throttle)
}
}()
}
@@ -161,8 +153,17 @@ func handleTierOverride(config *Config, tier string, message *DHCPMessage) (*DHC
return server, nil
}
func sendToServer(logger loggerHelper, start time.Time, server *DHCPServer, packet []byte, peer *net.UDPAddr) error {
err := server.sendTo(packet)
func sendToServer(logger loggerHelper, start time.Time, server *DHCPServer, packet []byte, peer *net.UDPAddr, throttle Throttle) error {
// Check for connection rate
ok, err := throttle.OK(server.Address.String())
if !ok {
glog.Errorf("Error writing to server %s, drop due to throttling", server.Hostname)
logger.LogErr(time.Now(), server, packet, peer, ErrConnRate, err)
return err
}
err = server.sendTo(packet)
if err != nil {
glog.Errorf("Error writing to server %s, drop due to %s", server.Hostname, err)
logger.LogErr(start, server, packet, peer, ErrWrite, err)
@@ -177,7 +178,7 @@ func sendToServer(logger loggerHelper, start time.Time, server *DHCPServer, pack
return nil
}
func handleRawPacketV4(logger loggerHelper, config *Config, buffer []byte, peer *net.UDPAddr) {
func handleRawPacketV4(logger loggerHelper, config *Config, buffer []byte, peer *net.UDPAddr, throttle Throttle) {
// runs in a separate go routine
start := time.Now()
var message DHCPMessage
@@ -199,10 +200,10 @@ func handleRawPacketV4(logger loggerHelper, config *Config, buffer []byte, peer
return
}
sendToServer(logger, start, server, packet, peer)
sendToServer(logger, start, server, packet, peer, throttle)
}
func handleRawPacketV6(logger loggerHelper, config *Config, buffer []byte, peer *net.UDPAddr) {
func handleRawPacketV6(logger loggerHelper, config *Config, buffer []byte, peer *net.UDPAddr, throttle Throttle) {
// runs in a separate go routine
start := time.Now()
packet := Packet6(buffer)
@@ -255,7 +256,7 @@ func handleRawPacketV6(logger loggerHelper, config *Config, buffer []byte, peer
}
relayMsg := packet.Encapsulate(peer.IP)
sendToServer(logger, start, server, relayMsg, peer)
sendToServer(logger, start, server, relayMsg, peer, throttle)
}
func handleV6RelayRepl(logger loggerHelper, start time.Time, packet Packet6, peer *net.UDPAddr) {
View
@@ -86,8 +86,8 @@ func NewServer(config *Config, version int, personalizedLogger PersonalizedLogge
}
glog.Infof("Setting up throttle: Cache Size: %d - Cache Rate: %d - Request Rate: %d",
config.TCacheSize, config.TCacheRate, config.TRatePerConn)
throttle, err := NewThrottle(config.TCacheSize, config.TCacheRate, config.TRatePerConn)
config.TCacheSize, config.TCacheRate, config.TRate)
throttle, err := NewThrottle(config.TCacheSize, config.TCacheRate, config.TRate)
if err != nil {
return nil, err
}

0 comments on commit fefe296

Please sign in to comment.