merge nested warden into master

warden/config/linux.yml

@@ -37,6 +37,8 @@ server:
   quota:
     disk_quota_enabled: true
 
+  allow_nested_warden: false
+
 health_check_server:
   port: 2345
 

warden/lib/warden/config.rb

@@ -14,6 +14,7 @@ def self.server_defaults
         "quota" => {
           "disk_quota_enabled" => true,
         },
+        "allow_nested_warden" => false,
       }
     end
 
@@ -59,6 +60,8 @@ def self.server_schema
           "quota" => {
             optional("disk_quota_enabled") => bool,
           },
+
+          "allow_nested_warden" => bool,
         }
       end
     end
@@ -97,6 +100,7 @@ def self.network_schema
           # Present for Backwards compatibility
           optional("pool_start_address") => String,
           optional("pool_size")          => Integer,
+          optional("release_delay")          => Integer,
           optional("mtu")                => Integer,
 
           "deny_networks"      => [String],
@@ -106,7 +110,12 @@ def self.network_schema
     end
 
     def self.ip_local_port_range
-      File.read("/proc/sys/net/ipv4/ip_local_port_range").split.map(&:to_i)
+      # if no ip_local_port_range found, make some guess"
+      if File.exist?("/proc/sys/net/ipv4/ip_local_port_range")
+        File.read("/proc/sys/net/ipv4/ip_local_port_range").split.map(&:to_i)
+      else
+        return 32768, 61000
+      end
     end
 
     def self.port_defaults
@@ -205,6 +214,10 @@ def rlimits
       @server["container_rlimits"] || {}
     end
 
+    def allow_nested_warden?
+      !!@server["allow_nested_warden"]
+    end
+
     def to_hash
       {
         "server"  => server,

warden/lib/warden/container/linux.rb

@@ -82,6 +82,7 @@ def env
           "network_netmask" => self.class.network_pool.pooled_netmask.to_human,
           "user_uid" => uid,
           "rootfs_path" => container_rootfs_path,
+	  "allow_nested_warden" => Server.config.allow_nested_warden?.to_s,
           "container_iface_mtu" => container_iface_mtu,
         }
         env
@@ -194,38 +195,53 @@ def perform_rsync(src_path, dst_path)
         sh *args
       end
 
-      def write_bind_mount_commands(request)
-        return if request.bind_mounts.nil? || request.bind_mounts.empty?
+      def add_bind_mount(file, src_path, dst_path, mode)
+            dst_path = File.join(container_path, "mnt", dst_path[1..-1])
+
+            file.puts "mkdir -p #{dst_path}"
+            file.puts "mount -n --bind #{src_path} #{dst_path}"
+            file.puts "mount -n --bind -o remount,#{mode} #{src_path} #{dst_path}"
+      end
 
-        File.open(File.join(container_path, "lib", "hook-parent-before-clone.sh"), "a") do |file|
+      def write_bind_mount_commands(request)
+        File.open(File.join(container_path, "lib", "hook-child-before-pivot.sh"), "a") do |file|
           file.puts
           file.puts
 
-          request.bind_mounts.each do |bind_mount|
-            src_path = bind_mount.src_path
-            dst_path = bind_mount.dst_path
-
-            # Check that the source path exists
-            stat = File.stat(src_path) rescue nil
-            if stat.nil?
-              raise WardenError.new("Source path for bind mount does not exist: #{src_path}")
+          if request.bind_mounts.respond_to?(:each)
+            request.bind_mounts.each do |bind_mount|
+              src_path = bind_mount.src_path
+              dst_path = bind_mount.dst_path
+
+              # Check that the source path exists
+              stat = File.stat(src_path) rescue nil
+              raise WardenError.new("Source path for bind mount does not exist: #{src_path}") if stat.nil?
+
+              mode = case bind_mount.mode
+                     when Protocol::CreateRequest::BindMount::Mode::RO
+                       "ro"
+                     when Protocol::CreateRequest::BindMount::Mode::RW
+                       "rw"
+                     else
+                       raise "Unknown mode"
+                     end
+
+              add_bind_mount(file, src_path, dst_path, mode)
             end
+          end
 
-            # Fix up destination path to be an absolute path inside the union
-            dst_path = File.join(container_path, "mnt", dst_path[1..-1])
+          # for nested warden, we share the host's cgroup fs to contrainers using bind mount
+          if Server.config.allow_nested_warden?
+            tmp_warden_cgroup = File.join(container_path, "tmp", "warden", "cgroup")
+            FileUtils.mkdir_p(tmp_warden_cgroup)
 
-            mode = case bind_mount.mode
-                   when Protocol::CreateRequest::BindMount::Mode::RO
-                     "ro"
-                   when Protocol::CreateRequest::BindMount::Mode::RW
-                     "rw"
-                   else
-                     raise "Unknown mode"
-                   end
+            # Bind-mount cgroups
+            add_bind_mount(file, tmp_warden_cgroup, "/tmp/warden/cgroup", "rw")
 
-            file.puts "mkdir -p #{dst_path}"
-            file.puts "mount -n --bind #{src_path} #{dst_path}"
-            file.puts "mount -n --bind -o remount,#{mode} #{src_path} #{dst_path}"
+            # for each subsystems, only pass the group of the current container instead of all groups, so that nested warder server will setup groups directly under parent container's hierarchy
+            %w(cpu cpuacct devices memory).each do |subsystem|
+              add_bind_mount(file, cgroup_path(subsystem), "/tmp/warden/cgroup/#{subsystem}", "rw")
+            end
           end
         end
       end

warden/lib/warden/server.rb

@@ -189,7 +189,7 @@ def self.setup_logging
     end
 
     def self.setup_network
-      network_pool = Pool::Network.new(config.network["pool_network"])
+      network_pool = Pool::Network.new(config.network["pool_network"], :release_delay => config.network["release_delay"])
       container_klass.network_pool = network_pool
     end
 

warden/root/linux/net.sh

@@ -103,7 +103,12 @@ function setup_filter() {
     iptables -A ${filter_default_chain} --destination "$n" --jump DROP
   done
 
+  # Forward outbound traffic via ${filter_forward_chain}
   iptables -A FORWARD -i w-+ --jump ${filter_forward_chain}
+
+  # Forward inbound traffic immediately
+  default_interface=$(ip route show | grep default | cut -d' ' -f5 | head -1)
+  iptables -I ${filter_forward_chain} -i $default_interface --jump ACCEPT
 }
 
 function teardown_nat() {

warden/root/linux/setup.sh

@@ -20,30 +20,27 @@ fi
 
 cgroup_path=/tmp/warden/cgroup
 
-mkdir -p $cgroup_path
-
-if grep "${cgroup_path} " /proc/mounts | cut -d' ' -f3 | grep -q cgroup
+if [ ! -d $cgroup_path ]
 then
-  find $cgroup_path -mindepth 1 -type d | sort | tac | xargs rmdir
-  umount $cgroup_path
-fi
-
-# Mount tmpfs
-if ! grep "${cgroup_path} " /proc/mounts | cut -d' ' -f3 | grep -q tmpfs
-then
-  mount -t tmpfs none $cgroup_path
-fi
+  mkdir -p $cgroup_path
 
-# Mount cgroup subsystems individually
-for subsystem in cpu cpuacct devices memory
-do
-  mkdir -p $cgroup_path/$subsystem
-
-  if ! grep -q "${cgroup_path}/$subsystem " /proc/mounts
+  # Mount tmpfs
+  if ! grep "${cgroup_path} " /proc/mounts | cut -d' ' -f3 | grep -q tmpfs
   then
-    mount -t cgroup -o $subsystem none $cgroup_path/$subsystem
+    mount -t tmpfs none $cgroup_path
   fi
-done
+
+  # Mount cgroup subsystems individually
+  for subsystem in cpu cpuacct devices memory
+  do
+    mkdir -p $cgroup_path/$subsystem
+
+    if ! grep -q "${cgroup_path}/$subsystem " /proc/mounts
+    then
+      mount -t cgroup -o $subsystem none $cgroup_path/$subsystem
+    fi
+  done
+fi
 
 ./net.sh setup
 

warden/root/linux/skeleton/lib/common.sh

@@ -44,6 +44,7 @@ function setup_fs_other() {
   overlay_directory_in_rootfs /etc rw
   overlay_directory_in_rootfs /home rw
   overlay_directory_in_rootfs /sbin rw
+  overlay_directory_in_rootfs /var rw
 
   mkdir -p tmp/rootfs/tmp
   chmod 777 tmp/rootfs/tmp

warden/root/linux/skeleton/lib/hook-parent-after-clone.sh

@@ -24,22 +24,28 @@ do
 
   if [ $(basename $system_path) == "devices" ]
   then
-    # disallow everything, allow explicitly
-    echo a > $instance_path/devices.deny
-    # /dev/null
-    echo "c 1:3 rw" > $instance_path/devices.allow
-    # /dev/zero
-    echo "c 1:5 rw" > $instance_path/devices.allow
-    # /dev/random
-    echo "c 1:8 rw" > $instance_path/devices.allow
-    # /dev/urandom
-    echo "c 1:9 rw" > $instance_path/devices.allow
-    # /dev/tty
-    echo "c 5:0 rw" > $instance_path/devices.allow
-    # /dev/ptmx
-    echo "c 5:2 rw" > $instance_path/devices.allow
-    # /dev/pts/*
-    echo "c 136:* rw" > $instance_path/devices.allow
+    if [ $allow_nested_warden == "true" ]
+    then
+      # Allow everything
+      echo "a *:* rw" > $instance_path/devices.allow
+    else
+      # Deny everything, allow explicitly
+      echo a > $instance_path/devices.deny
+      # /dev/null
+      echo "c 1:3 rw" > $instance_path/devices.allow
+      # /dev/zero
+      echo "c 1:5 rw" > $instance_path/devices.allow
+      # /dev/random
+      echo "c 1:8 rw" > $instance_path/devices.allow
+      # /dev/urandom
+      echo "c 1:9 rw" > $instance_path/devices.allow
+      # /dev/tty
+      echo "c 5:0 rw" > $instance_path/devices.allow
+      # /dev/ptmx
+      echo "c 5:2 rw" > $instance_path/devices.allow
+      # /dev/pts/*
+      echo "c 136:* rw" > $instance_path/devices.allow
+    fi
   fi
 
   echo $PID > $instance_path/tasks

warden/root/linux/skeleton/net.sh

@@ -38,7 +38,7 @@ function setup_filter() {
     --goto ${filter_default_chain}
 
   # Bind instance chain to forward chain
-  iptables -I ${filter_forward_chain} \
+  iptables -I ${filter_forward_chain} 2 \
     --in-interface ${network_host_iface} \
     --goto ${filter_instance_chain}
 }

warden/root/linux/skeleton/setup.sh

@@ -18,6 +18,7 @@ network_container_ip=${network_container_ip:-10.0.0.2}
 network_container_iface="w-${id}-1"
 user_uid=${user_uid:-10000}
 rootfs_path=$(readlink -f $rootfs_path)
+allow_nested_warden=${allow_nested_warden:-false}
 
 # Write configuration
 cat > etc/config <<-EOS
@@ -29,6 +30,7 @@ network_container_ip=$network_container_ip
 network_container_iface=$network_container_iface
 user_uid=$user_uid
 rootfs_path=$rootfs_path
+allow_nested_warden=$allow_nested_warden
 EOS
 
 setup_fs

warden/spec/assets/config/child-linux.yml

@@ -0,0 +1,62 @@
+---
+server:
+  container_klass: Warden::Container::Linux
+
+  # Wait this long before destroying a container, after the last client
+  # referencing it disconnected. The timer is cancelled when during this
+  # period, another client references the container.
+  #
+  # Clients can be forced to specify this setting by setting the
+  # server-wide variable to an invalid value:
+  #   container_grace_time: invalid
+  #
+  # The grace time can be disabled by setting it to nil:
+  #   container_grace_time: ~
+  #
+  container_grace_time: 30
+
+  unix_domain_permissions: 0777
+
+  # Specifies the path to the base chroot used as the read-only root
+  # filesystem
+  container_rootfs_path: /tmp/warden/rootfs
+
+  # Specifies the path to the parent directory under which all containers
+  # will live.
+  container_depot_path: /tmp/warden/containers
+
+  # See getrlimit(2) for details. Integer values are passed verbatim.
+  container_rlimits:
+    as: 4294967296
+    nofile: 8192
+    nproc: 512
+
+  # Specifies the output limit of a job (stdout/stderr combined).
+  job_output_limit: 10485760
+
+  quota:
+    disk_quota_enabled: false
+
+  allow_nested_warden: false
+
+health_check_server:
+  port: 2345
+
+logging:
+  level: debug2
+  file: /tmp/warden.log
+
+network:
+  # Use this /30 network as offset for the network pool.
+  pool_start_address: 10.254.0.0
+
+  # Pool this many /30 networks.
+  pool_size: 256
+
+  # Interface MTU size
+  # (for OpenStack use 1454 to avoid problems with rubygems with GRE tunneling)
+  mtu: 1500
+
+user:
+  pool_start_uid: 20000
+  pool_size: 25