Permalink
Browse files

implement bandwidth quota

Change-Id: I0a32181bcf94fe97e5ea13bba41fccfad4e73548
  • Loading branch information...
1 parent 459d12d commit 3cce20517a65aaec8209b8ad17c6fe49a746853f @andyzh andyzh committed Aug 17, 2012
@@ -219,6 +219,10 @@ def self.convert_limit_request(args)
when "disk"
request = Protocol::LimitDiskRequest.new(attributes)
request.byte = Integer(args.shift) unless args.empty?
+ when "bandwidth"
+ request = Protocol::LimitBandwidthRequest.new(attributes)
+ request.rate = Integer(args.shift) unless args.empty?
+ request.burst = Integer(args.shift) unless args.empty?
else
raise "Unknown limit: #{limit}"
end
@@ -234,6 +238,10 @@ def self.convert_limit_disk_response(response)
response.byte
end
+ def self.convert_limit_bandwidth_response(response)
+ "rate: #{response.rate} burst: #{response.burst}"
+ end
+
def self.convert_ping_request(args)
request = Protocol::PingRequest.new
request
@@ -22,6 +22,7 @@
require "warden/protocol/limit_memory"
require "warden/protocol/limit_disk"
+require "warden/protocol/limit_bandwidth"
require "warden/protocol/ping"
require "warden/protocol/list"
@@ -39,6 +39,7 @@ module Type
LimitMemory = 51
LimitDisk = 52
+ LimitBandwidth = 53
Ping = 91
List = 92
@@ -55,6 +55,13 @@ class DiskStat < BaseMessage
optional :inodes_used, :uint64, 2
end
+ class BandwidthStat < BaseMessage
+ optional :in_rate, :uint64, 1
+ optional :in_burst, :uint64, 2
+ optional :out_rate, :uint64, 3
+ optional :out_burst, :uint64, 4
+ end
+
optional :state, :string, 10
repeated :events, :string, 20
@@ -66,6 +73,7 @@ class DiskStat < BaseMessage
optional :memory_stat, MemoryStat, 40
optional :cpu_stat, CpuStat, 41
optional :disk_stat, DiskStat, 42
+ optional :bandwidth_stat, BandwidthStat, 43
end
end
end
@@ -0,0 +1,18 @@
+# coding: UTF-8
+
+require "warden/protocol/base"
+
+module Warden
+ module Protocol
+ class LimitBandwidthRequest < BaseRequest
+ required :handle, :string, 1
+ required :rate, :uint64, 2 # Bandwidth rate in byte(s)/sec
+ required :burst, :uint64, 3 # Allow burst size in byte(s)
+ end
+
+ class LimitBandwidthResponse < BaseResponse
+ required :rate, :uint64, 1 # Bandwidth rate in byte(s)/sec
+ required :burst, :uint64, 2 # Allow burst size in byte(s)
+ end
+ end
+end
@@ -565,6 +565,14 @@ def do_limit_disk(request, response)
raise WardenError.new("not implemented")
end
+ def before_limit_bandwidth
+ check_state_in(State::Active, State::Stopped)
+ end
+
+ def do_limit_bandwidth(request, response)
+ raise WardenError.new("not implemented")
+ end
+
def before_info
check_state_in(State::Active, State::Stopped)
end
@@ -13,10 +13,60 @@ module Net
include Spawn
+ INREG = /qdisc tbf \d+: root refcnt \d+ rate (\d+)([KMG]?)bit burst (\d+)([KMG]?)b lat 25.0ms/
+ OUTREG = /\s*police 0x[0-9a-f]+ rate (\d+)([KMG]?)bit burst (\d+)([KMG]?)b mtu \d+[KM]?b action drop overhead \d+b/
+
def self.included(base)
base.extend(ClassMethods)
end
+ def to_num(val, suffix)
+ kmg_map = {
+ "G" => 10 ** 9,
+ "M" => 10 ** 6,
+ "K" => 10 ** 3,
+ }
+ factor = kmg_map[suffix] || 1
+ val * factor
+ end
+
+ def do_info(request, response)
+ super(request, response)
+
+ id = request.handle
+
+ ret = {}
+
+ {:in => {:bash_key => "get_egress_info", :reg => INREG, :rate_key => :in_rate, :burst_key => :in_burst},
+ :out => {:bash_key => "get_ingress_info", :reg => OUTREG, :rate_key => :out_rate, :burst_key => :out_burst}}.each do |k, v|
+
+ # Set default rate value to 0xffffffff default burst value to 0xffffffff
+ ret[v[:rate_key]], ret[v[:burst_key]] = [0xffffffff, 0xffffffff]
+ info = sh File.join(container_path, "net.sh"), v[:bash_key], :env => {
+ "ID" => id
+ }
+ info.split("\n").each do |line|
+ if band_info = v[:reg].match(line)
+ ret[v[:rate_key]] = to_num(band_info[1].to_i, band_info[2]) / 8 # Bits to bytes
+ ret[v[:burst_key]] = to_num(band_info[3].to_i, band_info[4])
+ break
+ end
+ end
+ end
+
+ response.bandwidth_stat = Protocol::InfoResponse::BandwidthStat.new(ret)
+ nil
+ end
+
+ def do_limit_bandwidth(request, response)
+ sh File.join(container_path, "net_rate.sh"), :env => {
+ "BURST" => request.burst,
+ "RATE" => request.rate * 8, # Bytes to bits
+ }
+ response.rate = request.rate
+ response.burst = request.burst
+ end
+
def do_net_in(request, response)
host_port = self.class.port_pool.acquire
@@ -25,6 +25,7 @@ class Repl
list - list containers
info <handle> - show metadata for container <handle>
limit <handle> mem [<value>] - set or get the memory limit for the container (in bytes)
+limit <handle> bandwidth <rate> <bandwidth> - set the bandwidth limit for the container <rate> is the maxium transfer rate for both outbound and inbound(in bytes/sec) <burst> is the burst size(in bytes)
net <handle> #in - forward port #in on external interface to container <handle>
net <handle> #out <address[/mask][:port]> - allow traffic from the container <handle> to address <address>
copy <handle> <in|out> <src path> <dst path> [ownership opts] - Copy files/directories in and out of the container
@@ -124,7 +124,22 @@ case "${1}" in
iptables -I ${filter_instance_chain} 1 ${opts} --jump RETURN
;;
+ "get_ingress_info")
+ if [ -z "${ID:-}" ]; then
+ echo "Please specify container ID..." 1>&2
+ exit 1
+ fi
+ tc filter show dev w-${ID}-0 parent ffff:
+ ;;
+ "get_egress_info")
+ if [ -z "${ID:-}" ]; then
+ echo "Please specify container ID..." 1>&2
+ exit 1
+ fi
+ tc qdisc show dev w-${ID}-0
+
+ ;;
*)
echo "Unknown command: ${1}" 1>&2
exit 1
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+[ -n "$DEBUG" ] && set -o xtrace
+set -o nounset
+set -o errexit
+shopt -s nullglob
+
+cd $(dirname "${0}")
+
+source ./config
+
+if [ -z "${RATE:-}" ]; then
+ echo "Please specify RATE..." 1>&2
+ exit 1
+fi
+
+if [ -z "${BURST:-}" ]; then
+ echo "Please specify BURST..." 1>&2
+ exit 1
+fi
+
+# clear rule if exist
+# delete root egress tc qdisc
+tc qdisc del dev ${network_host_iface} root 2> /dev/null || true
+
+# delete root ingress tc qdisc
+tc qdisc del dev ${network_host_iface} ingress 2> /dev/null || true
+
+# set inbound(outside -> eth0 -> w-<cid>-0 -> w-<cid>-1) rule with tc's tbf(token bucket filter) qdisc
+# rate is the bandwidth
+# burst is the burst size
+# latency is the maxium time the packet wait to enqueue while no token left
+tc qdisc add dev ${network_host_iface} root tbf rate ${RATE}bit burst ${BURST} latency 25ms
+
+# set outbound(w-<cid>-1 -> w-<cid>-0 -> eth0 -> outside) rule
+tc qdisc add dev ${network_host_iface} ingress handle ffff:
+
+# use u32 filter with target(0.0.0.0) mask (0) to filter all the ingress packets
+tc filter add dev ${network_host_iface} parent ffff: protocol ip prio 1 u32 match ip src 0.0.0.0/0 police rate ${RATE}bit burst ${BURST} drop flowid :1
@@ -526,5 +526,15 @@ def initialize_container
container.limit_memory(Warden::Protocol::LimitDiskRequest.new)
}
end
+
+ describe "limit_bandwidth" do
+ before(:each) do
+ @container.stub(:do_limit_bandwidth)
+ end
+
+ include_examples "succeeds when active or stopped", Proc.new {
+ container.limit_bandwidth(Warden::Protocol::LimitBandwidthRequest.new)
+ }
+ end
end
end
@@ -221,6 +221,42 @@ def run(script)
end
end
+ describe "limit_bandwidth" do
+ attr_reader :handle
+
+ def limit_bandwidth(options = {})
+ response = client.limit_bandwidth(options.merge(:handle => handle))
+ response.should be_ok
+ response
+ end
+
+ before do
+ @handle = client.create.handle
+ end
+
+ it "should set the bandwidth" do
+ response = limit_bandwidth(:rate => 100 * 1000, :burst => 1000)
+ ret = client.info(:handle => handle)
+ [ret.bandwidth_stat.in_rate, ret.bandwidth_stat.out_rate].each do |v|
+ v.should == 100 * 1000
+ end
+ [ret.bandwidth_stat.in_burst, ret.bandwidth_stat.out_burst].each do |v|
+ v.should == 1000
+ end
+ end
+
+ it "should allow bandwidth to be changed" do
+ response = limit_bandwidth(:rate => 200 * 1000, :burst => 2000)
+ ret = client.info(:handle => handle)
+ [ret.bandwidth_stat.in_rate, ret.bandwidth_stat.out_rate].each do |v|
+ v.should == 200 * 1000
+ end
+ [ret.bandwidth_stat.in_burst, ret.bandwidth_stat.out_burst].each do |v|
+ v.should == 2000
+ end
+ end
+ end
+
describe "net_out", :netfilter => true do
attr_reader :handle
@@ -350,6 +386,16 @@ def check_mapping(response)
response = client.info(:handle => handle)
response.disk_stat.bytes_used.should be_within(32000).of(bytes_used + 1_000_000)
end
+
+ it "should include bandwidth stat" do
+ response = client.info(:handle => handle)
+ [response.bandwidth_stat.in_rate, response.bandwidth_stat.out_rate].each do |x|
+ x.should >= 0
+ end
+ [response.bandwidth_stat.in_burst, response.bandwidth_stat.out_burst].each do |x|
+ x.should >= 0
+ end
+ end
end
describe "bind mounts" do

0 comments on commit 3cce205

Please sign in to comment.