diff --git a/README.md b/README.md index 0757e43..3ddc9cb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Last Update: 11/17/2014 ## TL;DR #### Versions -CoreOS version used: [v490.0.0 - Alpha](https://coreos.com/releases/#490.0.0) +CoreOS version used: [v494.0.0 - Alpha](https://coreos.com/releases/#494.0.0) Kubernetes version used: [v0.4.3](https://github.com/GoogleCloudPlatform/kubernetes/releases/tag/v0.4.3) @@ -92,23 +92,22 @@ Since vxlan's function by encapsulating the MAC-based layer 2 ethernet frames wi * We get around the MAC filtering that the Cloud Network imposes, as vxlan traffic will still originate from the Cloud Network MAC address, and not the linux bridge used by Docker when a container creates the traffic * Communication paths for the entire group of all Kubernetes host machines & containers becomes automatically established because multicast allows all machines (hosts & containers) to not only send packets, but also, receive all packets sent on the overlay network; therefore, both Kubernetes host machines and containers can communicate with one another on their own subnet. -Below is the proposed network architecture that is configured on the Kubernetes machines via cloud-config & systemd units: +Below is the proposed network architecture that is configured on the Kubernetes machines using [CoreOS' Flannel](https://github.com/coreos/flannel) via cloud-config & systemd units: -* Each Kubernetes machine will have an interface, named "eth2", on the 192.168.3.0/24 Cloud Network -* We will then create a new bridge, named "cbr0" to differ from the default "docker0", with a network CIDR 10.244.0.0/15 where: - * Master nodes will have an address of 10.244.{master\_machine\_index}.1/24 - * Minion nodes will have an address of 10.245.{minion\_machine\_index}.1/24 -* A vxlan network device is then created named "vxlan0" on multicast mode which operates on the eth2 device, hence, creating an overlay on top of eth2 -* We then add the new vxlan device, vxlan0, to the bridge, cbr0 -* Finally, we swap out the default Docker bridge, docker0, with cbr0 in the Docker daemon so that it all container networking is based off of this new bridge +* Each Kubernetes machine will have an interface, named "eth2", on the isolated L2 192.168.3.0/24 Cloud Network +* Flannel then creates a TUN/TAP device named "flannel.1" that overlays onto the eth2 device +* Flannel also chooses a random subnet ID from the CIDR 10.244.0.0/15 that we've designated for the Flannel configuration and an MTU and assigns it to flannel.1 + * i.e 10.244.94.0/15 +* Flannel then requests a subnet lease on 10.244.0.0/15 for the Docker bridge +* We then update the Docker bridge interface's host CIDR by assigning it the new subnet chosen by Flannel (relevant to the overlay CIDR), and drop both the Docker bridge CIDR and flannel.1's MTU into /run/flannel/subnet.env so that we can make the Docker daemon aware of the new configuration + * i.e. $ cat /run/flannel/subnet.env + * FLANNEL_SUBNET=10.244.94.1/24 + * FLANNEL_MTU=1450 +* Docker is then restarted to take into account the new Docker bridge host CIDR & the flannel.1 MTU +* The Docker bridge is now set to instantiate containers on the new CIDR and depends on the host routing table to route all overlay traffic on the 10.244.0.0/15 CIDR via flannel.1

-**Note**: Some aspects of the networking such as the subnet assignment being based -off of the Heat machine index in the resource group are known hacks and are *not* meant to be used in production. Better subnet management, and potentially -creating the appropriate network for the Docker containers that Kubernetes -manages may be better suited with recently-released projects such as [Rudder](https://github.com/coreos/rudder) & [Weave](https://github.com/zettio/weave/). However, it is not in the near-term scope of Corekube to adopt one technology over the other. - **Note**: If you have RackConnect enabled you will require rules like the ones below. If you don't know what RackConnect is, you may safely ignore this. diff --git a/corekube-heat.yaml b/corekube-heat.yaml index 8820089..947abdd 100644 --- a/corekube-heat.yaml +++ b/corekube-heat.yaml @@ -18,7 +18,7 @@ parameters: default: 3 constraints: - range: - min: 2 + min: 1 max: 12 description: Must be between 2 and 12 servers. key-name: @@ -26,21 +26,18 @@ parameters: description: Name of key-pair to be used for compute instance flavor: type: string - default: 4 GB Performance + default: 4 GB General Purpose v1 constraints: - allowed_values: - - 2 GB Performance - - 4 GB Performance - - 8 GB Performance - - 15 GB Performance - - 30 GB Performance + - 4 GB General Purpose v1 + - 8 GB General Purpose v1 description: | Must be a valid Rackspace Cloud Server flavor for the region you have selected to deploy into. coreos-image: type: string - description: Rackspace Cloud Servers CoreOS Alpha (490.0.0) UUID - default: "3c7e97fa-a9f5-4b09-97aa-c94e66dbbfeb" + description: Rackspace Cloud Servers CoreOS Alpha (494.0.0) UUID + default: "1c423602-ea76-4263-b56b-0a2fa3e8c663" git-repo: type: string description: Public git repo url @@ -138,7 +135,7 @@ resources: [Install] WantedBy=sockets.target - name: fleet-local.service - command: stop + command: start content: | [Unit] Description=fleet-local @@ -214,61 +211,47 @@ resources: command: start - name: fleet.service command: start - - name: cbr0.netdev - command: start - content: | - [NetDev] - Kind=bridge - Name=cbr0 - - name: cbr0.network + - name: flannel-install.service command: start content: | - [Match] - Name=cbr0 - - [Network] - Address=10.244.%index%.1/24 - - [Route] - Destination=10.244.0.0/15 - Gateway=0.0.0.0 - - name: vxlan0.netdev - command: start - content: | - [NetDev] - Kind=vxlan - Name=vxlan0 - - [VXLAN] - Id=1 - Group=224.0.0.1 - - name: 49-eth2.network + [Unit] + After=network-online.target + Requires=network-online.target + + [Service] + ExecStart=/usr/bin/wget -N -P /opt/bin http://d294c9b5fc599b0924ac-1f970f440462fc899216e927cd570e77.r51.cf1.rackcdn.com/flanneld + ExecStart=/usr/bin/chmod +x /opt/bin/flanneld + RemainAfterExit=yes + Type=oneshot + - name: flannel.service command: start content: | - [Match] - Name=eth2 + [Unit] + After=network-online.target etcd.service flannel-install.service + Requires=network-online.target etcd.service flannel-install.service - [Network] - VXLAN=vxlan0 - - name: vxlan0.network + [Service] + ExecStartPre=/usr/bin/etcdctl mk /coreos.com/network/config '{"Network":"10.244.0.0/15", "Backend": {"Type": "vxlan"}}' + ExecStart=/opt/bin/flanneld -iface=eth2 + - name: flannel-env.path command: start content: | - [Match] - Name=vxlan0 - - [Network] - Bridge=cbr0 + [Path] + PathExists=/run/flannel/subnet.env + Unit=docker.service - name: docker.service command: start content: | [Unit] - After=network.target + After=flannel-env.path network-online.target flannel.service + Requires=flannel-env.path network-online.target flannel.service Description=Docker Application Container Engine - Documentation=http://docs.docker.io [Service] + EnvironmentFile=/run/flannel/subnet.env ExecStartPre=/bin/mount --make-rprivate / - ExecStart=/usr/bin/docker -d -s=btrfs -H fd:// -b cbr0 + ExecStartPre=/usr/bin/systemctl kill docker.service + ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} -s=btrfs [Install] WantedBy=multi-user.target @@ -318,61 +301,46 @@ resources: command: start - name: fleet.service command: start - - name: cbr0.netdev - command: start - content: | - [NetDev] - Kind=bridge - Name=cbr0 - - name: cbr0.network - command: start - content: | - [Match] - Name=cbr0 - - [Network] - Address=10.245.%index%.1/24 - - [Route] - Destination=10.244.0.0/15 - Gateway=0.0.0.0 - - name: vxlan0.netdev + - name: flannel-install.service command: start content: | - [NetDev] - Kind=vxlan - Name=vxlan0 - - [VXLAN] - Id=1 - Group=224.0.0.1 - - name: 49-eth2.network + [Unit] + After=network-online.target + Requires=network-online.target + + [Service] + ExecStart=/usr/bin/wget -N -P /opt/bin http://d294c9b5fc599b0924ac-1f970f440462fc899216e927cd570e77.r51.cf1.rackcdn.com/flanneld + ExecStart=/usr/bin/chmod +x /opt/bin/flanneld + RemainAfterExit=yes + Type=oneshot + - name: flannel.service command: start content: | - [Match] - Name=eth2 + [Unit] + After=etcd.service flannel-install.service + Requires=etcd.service flannel-install.service - [Network] - VXLAN=vxlan0 - - name: vxlan0.network + [Service] + ExecStart=/opt/bin/flanneld -iface=eth2 + - name: flannel-env.path command: start content: | - [Match] - Name=vxlan0 - - [Network] - Bridge=cbr0 + [Path] + PathExists=/run/flannel/subnet.env + Unit=docker.service - name: docker.service command: start content: | [Unit] - After=network.target + After=flannel-env.path network-online.target flannel.service + Requires=flannel-env.path network-online.target flannel.service Description=Docker Application Container Engine - Documentation=http://docs.docker.io [Service] + EnvironmentFile=/run/flannel/subnet.env ExecStartPre=/bin/mount --make-rprivate / - ExecStart=/usr/bin/docker -d -s=btrfs -H fd:// -b cbr0 + ExecStartPre=/usr/bin/systemctl kill docker.service + ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} -s=btrfs [Install] WantedBy=multi-user.target diff --git a/images/networking.png b/images/networking.png index 813882c..2d9a1d2 100644 Binary files a/images/networking.png and b/images/networking.png differ