Configurable grid subnet, supernet

agent/lib/docker/container.rb

@@ -176,11 +176,24 @@ def overlay_ip
     # Container CIDR address within the overlay network.
     # Will be missing/nil if container is not attached to the overlay network.
     #
+    # For kontena 0.16 containers, this will be 10.81.X.Y/19, and gets translated to 10.81.X.Y/16.
+    #
     # @return [String, NilClass]
     def overlay_cidr
       self.labels['io.kontena.container.overlay_cidr']
     end
 
+    # Container overlay network IPAM pool
+    # Will be missing/nil if container is not attached to the overlay network.
+    #
+    # For kontena 0.16 containers, this will be nil.
+    # For kontena 1.0 containers, this will always be 'kontena'.
+    #
+    # @return [String, NilClass]
+    def overlay_network
+      self.labels['io.kontena.container.overlay_network']
+    end
+
     # Container CIDR suffix within the overlay network.
     # Will be missing/nil if container is not attached to the overlay network.
     #

agent/lib/ipaddr_helpers.rb

@@ -0,0 +1,48 @@
+class IPAddr
+  # @return [Number] CIDR prefix length suffix
+  def prefixlen
+    @mask_addr.to_s(2).count('1')
+  end
+
+  # @return [String] The address + netmask in W.X.Y.Z/P format
+  #
+  # For a host address, this will include the /32 suffix
+  def to_cidr
+    "#{to_s}/#{prefixlen}"
+  end
+
+  def hostmask
+    case @family
+    when Socket::AF_INET
+      (IN4MASK ^ @mask_addr)
+    when Socket::AF_INET6
+      (IN6MASK ^ @mask_addr)
+    else
+      raise AddressFamilyError, "unsupported address family"
+    end
+  end
+
+  # @return [IPAddr) last address in subnet]
+  def last
+    self | hostmask
+  end
+
+  # @return [IPAddr] ith address in subnet
+  def [](i)
+    raise ArgumentError, "IP #{i} outside of subnet #{inspect}" if i > hostmask
+
+    self | i
+  end
+
+  # Split subnet into lower and upper subnets
+  # @return [IPAddr, IPAddr] two sub-subnets
+  def split
+    high_bit = hostmask + 1 >> 1
+    splitlen = prefixlen + 1
+
+    return [
+      mask(splitlen),
+      mask(splitlen) | high_bit,
+    ]
+  end
+end

agent/lib/kontena-agent.rb

@@ -9,6 +9,8 @@
 require 'active_support/core_ext/time'
 require 'active_support/core_ext/module/delegation'
 
+require_relative 'ipaddr_helpers'
+
 require_relative 'docker/version'
 require_relative 'docker/container'
 require_relative 'etcd/health'

agent/lib/kontena/launchers/etcd.rb

@@ -88,7 +88,7 @@ def create_container(image, info)
       cluster_size = info['grid']['initial_size']
       node_number = info['node_number']
       cluster_state = 'new'
-      weave_ip = weave_ip(info['node_number'])
+      weave_ip = info['overlay_ip']
 
       container = Docker::Container.get('kontena-etcd') rescue nil
       if container && container.info['Config']['Image'] != image
@@ -112,11 +112,12 @@ def create_container(image, info)
       name = "node-#{info['node_number']}"
       grid_name = info['grid']['name']
       docker_ip = docker_gateway
+      initial_cluster = initial_cluster(info['grid'])
 
       cmd = [
         '--name', name, '--data-dir', '/var/lib/etcd',
         '--listen-client-urls', "http://127.0.0.1:2379,http://#{weave_ip}:2379,http://#{docker_ip}:2379",
-        '--initial-cluster', initial_cluster(cluster_size).join(',')
+        '--initial-cluster', initial_cluster.join(',')
       ]
       if node_number <= cluster_size
         cmd = cmd + [
@@ -132,7 +133,7 @@ def create_container(image, info)
         cmd = cmd + ['--proxy', 'on']
         info "starting etcd service as a proxy"
       end
-      info "cluster members: #{initial_cluster(cluster_size).join(',')}"
+      info "cluster members: #{initial_cluster.join(',')}"
 
       container = Docker::Container.create(
         'name' => 'kontena-etcd',
@@ -161,7 +162,7 @@ def update_membership(info)
       etcd_connection = find_etcd_node(info)
       return 'new' unless etcd_connection # No etcd hosts available, bootstrapping first node --> new cluster
 
-      weave_ip = weave_ip(info['node_number'])
+      weave_ip = info['overlay_ip']
       peer_url = "http://#{weave_ip}:2380"
       client_url = "http://#{weave_ip}:2379"
 
@@ -193,9 +194,10 @@ def update_membership(info)
     # @param [Hash] node info
     # @return [Hash] The cluster members as given by etcd API
     def find_etcd_node(info)
+      grid_subnet = IPAddr.new(info['grid']['subnet'])
       tries = info['grid']['initial_size']
       begin
-        etcd_host = "http://10.81.0.#{tries}:2379/v2/members"
+        etcd_host = "http://#{grid_subnet[tries]}:2379/v2/members"
 
         info "connecting to existing etcd at #{etcd_host}"
         connection = Excon.new(etcd_host)
@@ -242,13 +244,11 @@ def add_dns(container_id, weave_ip)
 
     # @param [Integer] cluster_size
     # @return [Array<String>]
-    def initial_cluster(cluster_size)
-      initial_cluster = []
-      cluster_size.times do |i|
-        node = i + 1
-        initial_cluster << "node-#{node}=http://10.81.0.#{node}:2380"
-      end
-      initial_cluster
+    def initial_cluster(grid_info)
+      grid_subnet = IPAddr.new(grid_info['subnet'])
+      (1..grid_info['initial_size']).map { |i|
+        "node-#{i}=http://#{grid_subnet[i]}:2380"
+      }
     end
 
     ##
@@ -257,12 +257,6 @@ def docker_gateway
       interface_ip('docker0')
     end
 
-    ##
-    # @param [Hash] node info
-    # @return [String] weave network ip of the node
-    def weave_ip(node_number)
-      "10.81.0.#{node_number}"
-    end
     # @param [Exception] exc
     def log_error(exc)
       error "#{exc.class.name}: #{exc.message}"

agent/lib/kontena/launchers/ipam_plugin.rb

@@ -38,7 +38,7 @@ def on_network_start(topic, info)
     def start(info)
       create_container(@image_name, info)
       wait(message: "IPAM running") { running? }
-      Celluloid::Notifications.publish('ipam:start', nil)
+      Celluloid::Notifications.publish('ipam:start', info)
     end
 
     def image_exists?
@@ -85,6 +85,7 @@ def create_container(image, info)
           "NODE_ID=#{info['node_number']}",
           "LOG_LEVEL=#{ENV['LOG_LEVEL'] || 1}",
           "ETCD_ENDPOINT=http://127.0.0.1:2379",
+          "KONTENA_IPAM_SUPERNET=#{info['grid']['supernet']}",
         ],
         'HostConfig' => {
           'NetworkMode' => 'host',

agent/lib/kontena/network_adapters/weave.rb

@@ -177,8 +177,8 @@ def on_node_info(topic, info)
       start(info)
     end
 
-    def on_ipam_start(topic, data)
-      ensure_default_pool
+    def on_ipam_start(topic, info)
+      ensure_default_pool(info['grid'])
       Celluloid::Notifications.publish('network:ready', nil)
     end
 
@@ -206,9 +206,13 @@ def ensure_exposed(cidr)
       end
     end
 
-    def ensure_default_pool()
-      info 'network and ipam ready, ensuring default network existence'
-      @default_pool = @ipam_client.reserve_pool(DEFAULT_NETWORK, '10.81.0.0/16', '10.81.128.0/17')
+    def ensure_default_pool(grid_info)
+      grid_subnet = IPAddr.new(grid_info['subnet'])
+
+      lower, upper = grid_subnet.split
+
+      info "network and ipam ready, ensuring default network with subnet=#{grid_subnet.to_cidr} iprange=#{upper.to_cidr}"
+      @default_pool = @ipam_client.reserve_pool(DEFAULT_NETWORK, grid_subnet.to_cidr, upper.to_cidr)
     end
 
     # @param [Hash] info
@@ -275,8 +279,13 @@ def connect_peers(peer_ips)
 
     # @param [Hash] info
     def post_start(info)
-      if info['node_number']
-        ensure_exposed("10.81.0.#{info['node_number']}/16")
+      grid_subnet = IPAddr.new(info['grid']['subnet'])
+      overlay_ip = info['overlay_ip']
+
+      if grid_subnet && overlay_ip
+        weave_cidr = "#{overlay_ip}/#{grid_subnet.prefixlen}"
+
+        ensure_exposed(weave_cidr)
       end
     end
 

agent/lib/kontena/workers/weave_worker.rb

@@ -109,10 +109,9 @@ def on_container_destroy(event)
     # @param [String] container_id
     # @param [String] overlay_cidr
     def attach_overlay(container)
-      if container.overlay_suffix == OVERLAY_SUFFIX
-        network_adapter.attach_container(container.id, container.overlay_cidr)
-      else
-        # override label for existing containers that may need to be migrated
+      if container.overlay_network.nil?
+        # overlay network migration for 0.16 compat
+        # override overlay network /19 -> /16 suffix for existing containers that may need to be migrated
         overlay_cidr = "#{container.overlay_ip}/#{OVERLAY_SUFFIX}"
 
         # check for un-migrated containers cached at start
@@ -128,6 +127,8 @@ def attach_overlay(container)
 
           network_adapter.attach_container(container.id, overlay_cidr)
         end
+      else
+        network_adapter.attach_container(container.id, container.overlay_cidr)
       end
     end
 

agent/spec/lib/ipaddr_spec.rb

@@ -0,0 +1,52 @@
+describe IPAddr do
+  describe '#prefixlen' do
+    it 'has a prefix length' do
+      expect(IPAddr.new('0.0.0.0/0').prefixlen).to eq 0
+      expect(IPAddr.new('10.80.0.0/12').prefixlen).to eq 12
+      expect(IPAddr.new('10.80.0.0/24').prefixlen).to eq 24
+      expect(IPAddr.new('10.80.0.1/24').prefixlen).to eq 24
+      expect(IPAddr.new('10.80.1.1/32').prefixlen).to eq 32
+    end
+  end
+
+  describe 'last' do
+    it 'returns the correct IPAddr' do
+      expect(IPAddr.new('10.81.0.0/16').last.to_s).to eq '10.81.255.255'
+    end
+  end
+
+  describe '#[]' do
+    context "for 10.81.0.0/16" do
+      subject do
+        IPAddr.new('10.81.0.0/16')
+      end
+
+      it 'returns the correct IP' do
+        expect(subject[0].to_s).to eq '10.81.0.0'
+        expect(subject[1].to_s).to eq '10.81.0.1'
+        expect(subject[2].to_s).to eq '10.81.0.2'
+        expect(subject[255].to_s).to eq '10.81.0.255'
+        expect(subject[256].to_s).to eq '10.81.1.0'
+        expect(subject[257].to_s).to eq '10.81.1.1'
+        expect(subject[256 * 256 - 1].to_s).to eq '10.81.255.255'
+      end
+
+      it 'raises on invalid offset' do
+        expect{subject[256 * 256]}.to raise_error(ArgumentError)
+        expect{subject[256 * 256 + 1]}.to raise_error(ArgumentError)
+      end
+    end
+  end
+
+  describe '#split' do
+    context "for 10.81.0.0/16" do
+      subject do
+        IPAddr.new('10.81.0.0/16')
+      end
+
+      it "splits in twain" do
+        expect(subject.split.map{|i| i.to_cidr}).to eq ['10.81.0.0/17', '10.81.128.0/17']
+      end
+    end
+  end
+end