ovn-k8s-overlay

#! /usr/bin/python
# Copyright (C) 2016 Nicira, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import distutils.spawn
import json
import netaddr
import os
import shlex
import sys
import traceback

import ovn_k8s.common.util as util
from ovn_k8s.common.util import ovs_vsctl
from ovn_k8s.common.util import ovn_nbctl
from ovn_k8s.common import variables

CNI_CONF_PATH = "/etc/cni/net.d"
CNI_LINK_PATH = "/opt/cni/bin/"
CNI_PLUGIN = "ovn-k8s-cni-overlay"
PRIVATE_KEY = "/etc/openvswitch/ovncontroller-privkey.pem"
CERTIFICATE = "/etc/openvswitch/ovncontroller-cert.pem"
CA_CERT = "/etc/openvswitch/ovnnb-ca.cert"


def fetch_ovn_nb():
    OVN_NB = ovs_vsctl("--if-exists", "get", "Open_vSwitch", ".",
                       "external_ids:ovn-nb").strip('"')
    if not OVN_NB:
        raise Exception("OVN central database's ip address not set")

    variables.OVN_NB = OVN_NB

    if OVN_NB.startswith("ssl"):
        variables.NBCTL_PRIVATE_KEY = PRIVATE_KEY
        variables.NBCTL_CERTIFICATE = CERTIFICATE
        variables.NBCTL_CA_CERT = CA_CERT

        if not os.path.isfile(PRIVATE_KEY):
            raise Exception("No private key %s found" % (PRIVATE_KEY))

        if not os.path.isfile(CERTIFICATE):
            raise Exception("No certificate %s found" % (CERTIFICATE))

        # We allow bootstrapping to get the CA_CERT.
        if not os.path.isfile(CA_CERT):
            # Bootstrap it with a simple transaction.
            try:
                print("No CA certificate found. Attempting bootstrapping...")
                ovn_nbctl("list", "logical_switch")
            except Exception:
                # exception is expected. But did we bootstrap?
                if not os.path.isfile(CA_CERT):
                    raise Exception("Bootstapping OVN NB's certificate failed")
                print("Bootstrapping successful. Continuing...")


def get_k8s_cluster_router():
    k8s_cluster_router = ovn_nbctl("--data=bare", "--no-heading",
                                   "--columns=_uuid", "find", "logical_router",
                                   "external_ids:k8s-cluster-router=yes")
    if not k8s_cluster_router:
        raise Exception("K8S_CLUSTER_ROUTER not found")

    return k8s_cluster_router


def get_local_system_id():
    system_id = ovs_vsctl("--if-exists", "get", "Open_vSwitch", ".",
                          "external_ids:system-id").strip('"')
    if not system_id:
        raise Exception("no system-id configured in the local host")

    return system_id


def configure_management_port_debian(node_name, cluster_subnet, router_ip,
                                     interface_name, interface_ip):
    bridge_exists = False
    interface_exists = False

    bridge_context = {"interface_name": interface_name}
    bridge_template = """
allow-ovs br-int
iface br-int inet manual
    ovs_type OVSBridge
    ovs_ports {interface_name}
    ovs_extra set bridge br-int fail_mode=secure
"""

    ip = netaddr.IPNetwork(interface_ip)
    cluster_ip = netaddr.IPNetwork(cluster_subnet)

    interface_context = {"interface_name": interface_name,
                         "address": str(ip.ip),
                         "netmask": str(ip.netmask),
                         "iface_id": "k8s-" + node_name,
                         "cluster_ip": str(cluster_ip.ip),
                         "cluster_mask": str(cluster_ip.netmask),
                         "gw_ip": router_ip}

    interface_template = """
allow-br-int {interface_name}
iface {interface_name} inet static
    address {address}
    netmask {netmask}
    ovs_type OVSIntPort
    ovs_bridge br-int
    ovs_extra set interface $IFACE external-ids:iface-id={iface_id}
    up route add -net {cluster_ip} netmask {cluster_mask} gw {gw_ip}
    down route del -net {cluster_ip} netmask {cluster_mask} gw {gw_ip}
"""

    with open("/etc/network/interfaces") as f:
        for line in f:
            # Look for a line of the form "allow-ovs br-int"
            if "allow-ovs" in line and "br-int" in line:
                # Config for bridge "br-int" exists.
                bridge_exists = True
                break

        for line in f:
            # Look for a line of the form "allow-br-int $interface_name"
            if "allow-br-int" in line and interface_name in line:
                # Config for interface $interface_name exists.
                interface_exists = True
                break

    if not bridge_exists:
        with open("/etc/network/interfaces", "a") as f:
            f.write(bridge_template.format(**bridge_context))

    if not interface_exists:
        with open("/etc/network/interfaces", "a") as f:
            f.write(interface_template.format(**interface_context))


def configure_management_port_redhat(node_name, cluster_subnet, router_ip,
                                     interface_name, interface_ip):
    bridge_template = """
DEVICE=br-int
ONBOOT=yes
DEVICETYPE=ovs
TYPE=OVSBridge
OVS_EXTRA="set bridge br-int fail_mode=secure"
"""

    with open("/etc/sysconfig/network-scripts/ifcfg-br-int", "w") as f:
        f.write(bridge_template)

    ip = netaddr.IPNetwork(interface_ip)
    interface_context = {"interface_name": interface_name,
                         "address": str(ip.ip),
                         "netmask": str(ip.netmask),
                         "iface_id": "k8s-" + node_name}

    interface_template = """
DEVICE={interface_name}
DEVICETYPE=ovs
TYPE=OVSIntPort
OVS_BRIDGE=br-int
IP_ADDR={address}
NETMASK={netmask}
OVS_EXTRA="set interface $DEVICE external-ids:iface-id={iface_id}
"""

    filename = "/etc/sysconfig/network-scripts/ifcfg-" + interface_name
    with open(filename, "w") as f:
        f.write(interface_template.format(**interface_context))

    cluster_ip = netaddr.IPNetwork(cluster_subnet)
    route_context = {"cluster_ip": str(cluster_ip.ip),
                     "cluster_mask": str(cluster_ip.netmask),
                     "gw_ip": router_ip}

    route_template = """
ADDRESS0={cluster_ip}
NETMASK0={cluster_mask}
GATEWAY0={gw_ip}
"""

    filename = "/etc/sysconfig/network-scripts/route-" + interface_name
    with open(filename, "w") as f:
        f.write(route_template.format(**route_context))


def configure_management_port(node_name, cluster_subnet, router_ip,
                              interface_name, interface_ip):
    # First, try to configure management ports via platform specific tools.

    # Identify whether the platform is Debian based.
    if os.path.isfile("/etc/network/interfaces"):
        configure_management_port_debian(node_name, cluster_subnet, router_ip,
                                         interface_name, interface_ip)

    if os.path.isfile("/etc/sysconfig/network-scripts/ifup-ovs"):
        configure_management_port_redhat(node_name, cluster_subnet, router_ip,
                                         interface_name, interface_ip)

    # Up the interface.
    command = "ip link set %s up" % (interface_name)
    util.call_popen(shlex.split(command))

    # The interface may already exist, in which case delete the routes and IP.
    command = "ip addr flush dev %s" % (interface_name)
    util.call_popen(shlex.split(command))

    # Assign IP address to the internal interface.
    try:
        command = "ip addr add %s dev %s" % (interface_ip, interface_name)
        util.call_popen(shlex.split(command))
    except Exception as e:
        sys.stderr.write("warning: failed to run \"%s\": %s"
                         % (command, str(e)))

    # Flush the route for the entire subnet (in case it was added before)
    command = "ip route flush %s" % (cluster_subnet)
    util.call_popen(shlex.split(command))

    # Create a route for the entire subnet.
    try:
        command = "ip route add %s via %s" % (cluster_subnet, router_ip)
        util.call_popen(shlex.split(command))
    except Exception as e:
        sys.stderr.write("warning: failed to run \"%s\": %s"
                         % (command, str(e)))


def create_management_port(node_name, local_subnet, cluster_subnet):
    # Create a logical switch for the node and connect it to
    # the distributed router.  This switch will start with
    # one logical port (A OVS internal interface).
    # 1.  This logical port is via which a node can access all other nodes and
    # the containers running inside them using the private IP addresses.
    # 2.  When this port is created on the master node, the K8s daemons
    # become reachable from the containers without any NAT.
    # 3. The nodes can health-check the pod IP addresses.

    # Create a router port and provide it the first address in the
    # 'local_subnet'.
    ip = netaddr.IPNetwork(local_subnet)
    ip.value = ip.value + 1
    router_ip_mask = str(ip)
    router_ip = str(ip.ip)

    router_mac = ovn_nbctl("--if-exist", "get", "logical_router_port",
                           "rtos-" + node_name, "mac").strip('"')
    if not router_mac:
        router_mac = util.generate_mac()
        cluster_router = get_k8s_cluster_router()
        ovn_nbctl("--may-exist", "lrp-add", cluster_router,
                  "rtos-" + node_name, router_mac, router_ip_mask)

    # Create a logical switch and set its subnet.
    ovn_nbctl("--", "--may-exist", "ls-add", node_name,
              "--", "set", "logical_switch", node_name,
              "other-config:subnet=" + local_subnet,
              "external-ids:gateway_ip=" + router_ip_mask)

    # Connect the switch to the router.
    ovn_nbctl("--", "--may-exist", "lsp-add", node_name,
              "stor-" + node_name, "--", "set", "logical_switch_port",
              "stor-" + node_name, "type=router",
              "options:router-port=rtos-" + node_name,
              "addresses=" + "\"" + router_mac + "\"")

    interface_name = "k8s-%s" % (node_name[:11])
    # Create a OVS internal interface
    ovs_vsctl("--", "--may-exist", "add-port", "br-int",
              interface_name, "--", "set", "interface",
              interface_name, "type=internal",
              "mtu_request=1400",
              "external-ids:iface-id=k8s-" + node_name)

    mac_address = ovs_vsctl("--if-exists", "get", "interface",
                            interface_name, "mac_in_use").strip('"')
    if not mac_address:
        raise Exception("failed to get mac address of %s" % (interface_name))

    # Create the OVN logical port.
    ip.value = ip.value + 1
    port_ip = str(ip.ip)
    ovn_nbctl("--", "--may-exist", "lsp-add", node_name,
              "k8s-" + node_name, "--", "lsp-set-addresses",
              "k8s-" + node_name, mac_address + " " + port_ip)

    configure_management_port(node_name, cluster_subnet, router_ip,
                              interface_name, str(ip))

    # Add the load_balancer to the switch.
    k8s_cluster_lb_tcp = ovn_nbctl("--data=bare", "--no-heading",
                                   "--columns=_uuid", "find", "load_balancer",
                                   "external_ids:k8s-cluster-lb-tcp=yes")
    if k8s_cluster_lb_tcp:
        ovn_nbctl("set", "logical_switch", node_name,
                  "load_balancer=" + k8s_cluster_lb_tcp)

    k8s_cluster_lb_udp = ovn_nbctl("--data=bare", "--no-heading",
                                   "--columns=_uuid", "find", "load_balancer",
                                   "external_ids:k8s-cluster-lb-udp=yes")
    if k8s_cluster_lb_udp:
        ovn_nbctl("add", "logical_switch", node_name,
                  "load_balancer", k8s_cluster_lb_udp)


def master_init(args):
    fetch_ovn_nb()

    if not args.node_name or not args.master_switch_subnet \
       or not args.cluster_ip_subnet:
        raise Exception("Arguments should be non-null")

    node_name = args.node_name

    # Create a single common distributed router for the cluster.
    ovn_nbctl("--", "--may-exist", "lr-add", node_name, "--", "set",
              "logical_router", node_name,
              "external_ids:k8s-cluster-router=yes")

    # Create 2 load-balancers for east-west traffic.  One handles UDP
    # and another handles TCP.
    k8s_cluster_lb_tcp = ovn_nbctl("--data=bare", "--no-heading",
                                   "--columns=_uuid", "find", "load_balancer",
                                   "external_ids:k8s-cluster-lb-tcp=yes")
    if not k8s_cluster_lb_tcp:
        ovn_nbctl("--", "create", "load_balancer",
                  "external_ids:k8s-cluster-lb-tcp=yes")

    k8s_cluster_lb_udp = ovn_nbctl("--data=bare", "--no-heading",
                                   "--columns=_uuid", "find", "load_balancer",
                                   "external_ids:k8s-cluster-lb-udp=yes")
    if not k8s_cluster_lb_udp:
        ovn_nbctl("--", "create", "load_balancer",
                  "external_ids:k8s-cluster-lb-udp=yes", "protocol=udp")

    # Create a logical switch called "join" that will be used to connect
    # gateway routers to the distributed router. The "join" will be
    # allocated IP addresses in the range 100.64.1.0/24
    ovn_nbctl("--may-exist", "ls-add", "join")

    # Connect the distributed router to "join"
    router_mac = ovn_nbctl("--if-exist", "get", "logical_router_port",
                           "rtoj-" + node_name, "mac").strip('"')
    if not router_mac:
        router_mac = util.generate_mac()
        ovn_nbctl("--", "--may-exist", "lrp-add", node_name,
                  "rtoj-" + node_name, router_mac, "100.64.1.1/24",
                  "--", "set", "logical_router_port", "rtoj-" + node_name,
                  "external_ids:connect_to_join=yes")

    # Connect the switch "join" to the router.
    ovn_nbctl("--", "--may-exist", "lsp-add", "join",
              "jtor-" + node_name, "--", "set", "logical_switch_port",
              "jtor-" + node_name, "type=router",
              "options:router-port=rtoj-" + node_name,
              "addresses=" + "\"" + router_mac + "\"")

    create_management_port(node_name, args.master_switch_subnet,
                           args.cluster_ip_subnet)


def minion_init(args):
    fetch_ovn_nb()

    if not args.node_name or not args.minion_switch_subnet \
       or not args.cluster_ip_subnet:
        raise Exception("Arguments should be non-null")

    node_name = args.node_name

    cni_plugin_path = distutils.spawn.find_executable(CNI_PLUGIN)
    if not cni_plugin_path:
        raise Exception("No CNI plugin %s found" % CNI_PLUGIN)

    if not os.path.exists(CNI_LINK_PATH):
        os.makedirs(CNI_LINK_PATH)

    cni_file = "%s/ovn_cni" % CNI_LINK_PATH
    if not os.path.isfile(cni_file):
        command = "ln -s %s %s" % (cni_plugin_path, cni_file)
        util.call_popen(shlex.split(command))

    # Create the CNI config
    if not os.path.exists(CNI_CONF_PATH):
        os.makedirs(CNI_CONF_PATH)

    CNI_FILE = "%s/10-net.conf" % CNI_CONF_PATH

    if not os.path.isfile(CNI_FILE):
        data = {
                "name": "net",
                "type": "ovn_cni",
                "bridge": "br-int",
                "isGateway": "true",
                "ipMasq": "false",
                "ipam": {
                         "type": "host-local",
                         "subnet": args.minion_switch_subnet
                        }
                }
        with open(CNI_FILE, 'w') as outfile:
            json.dump(data, outfile)

    create_management_port(node_name, args.minion_switch_subnet,
                           args.cluster_ip_subnet)


def generate_gateway_ip():
    # All the routers connected to "join" switch are in 100.64.1.0/24
    # network and they have their external_ids:connect_to_join set.
    ips = ovn_nbctl("--data=bare", "--no-heading", "--columns=network",
                    "find", "logical_router_port",
                    "external_ids:connect_to_join=yes").split()

    ip_start = netaddr.IPNetwork("100.64.1.0/24")
    ip_max = netaddr.IPNetwork("100.64.1.255/24")

    while ip_start.value != ip_max.value:
        ip_start.value = ip_start.value + 1
        if str(ip_start) not in ips:
            return str(ip_start)

    raise Exception("Ran out of IPs for gateway routers")


def gateway_init(args):
    if not args.node_name or not args.cluster_ip_subnet \
       or not args.physical_ip:
        raise Exception("Arguments should be non-null")

    # We want either of args.physical_interface or args.bridge_interface
    # provided. But not both. (XOR)
    if bool(args.physical_interface) == bool(args.bridge_interface):
        raise Exception("One of physical-interface or bridge-interface "
                        "has to be specified")

    physical_ip = netaddr.IPNetwork(args.physical_ip)
    if args.default_gw:
        default_gw = netaddr.IPNetwork(args.default_gw)

    fetch_ovn_nb()

    node_name = args.node_name

    k8s_cluster_router = get_k8s_cluster_router()

    system_id = get_local_system_id()

    # Find if gateway routers have been created before.
    first_gateway = "no"
    physical_gateways = ovn_nbctl(
                                "--data=bare", "--no-heading",
                                "--columns=_uuid", "find",
                                "logical_router",
                                "options:chassis!=null").split()
    if not physical_gateways:
        first_gateway = "yes"

    # Create a gateway router.
    gateway_router = "GR_%s" % (node_name)
    ovn_nbctl("--", "--may-exist", "lr-add", gateway_router, "--", "set",
              "logical_router", gateway_router, "options:chassis=" + system_id,
              "external_ids:physical_ip=" + str(physical_ip.ip),
              "external_ids:first_gateway=" + first_gateway)

    # Connect gateway router to switch "join".
    router_ip = None
    router_mac = ovn_nbctl("--if-exist", "get", "logical_router_port",
                           "rtoj-" + gateway_router, "mac").strip('"')
    if not router_mac:
        router_mac = util.generate_mac()
        router_ip = generate_gateway_ip()
        ovn_nbctl("--", "--may-exist", "lrp-add", gateway_router,
                  "rtoj-" + gateway_router, router_mac, router_ip,
                  "--", "set", "logical_router_port", "rtoj-" + gateway_router,
                  "external_ids:connect_to_join=yes")

    # Connect the switch "join" to the router.
    ovn_nbctl("--", "--may-exist", "lsp-add", "join",
              "jtor-" + gateway_router, "--", "set", "logical_switch_port",
              "jtor-" + gateway_router, "type=router",
              "options:router-port=rtoj-" + gateway_router,
              "addresses=" + "\"" + router_mac + "\"")

    # Add a static route in GR with distributed router as the nexthop.
    ovn_nbctl("--may-exist", "lr-route-add", gateway_router,
              args.cluster_ip_subnet, "100.64.1.1")
    # Add a static route in GR with physical gateway as the default next hop.
    if args.default_gw:
        ovn_nbctl("--may-exist", "lr-route-add", gateway_router,
                  "0.0.0.0/0", str(default_gw.ip))

    # Add a default route in distributed router with first GR as the nexthop.
    ovn_nbctl("--may-exist", "lr-route-add", k8s_cluster_router,
              "0.0.0.0/0", "100.64.1.2")

    # Create 2 load-balancers for north-south traffic for each gateway
    # router.  One handles UDP and another handles TCP.
    k8s_ns_lb_tcp = ovn_nbctl("--data=bare", "--no-heading",
                              "--columns=_uuid", "find", "load_balancer",
                              "external_ids:TCP_lb_gateway_router=" +
                              gateway_router)
    if not k8s_ns_lb_tcp:
        k8s_ns_lb_tcp = ovn_nbctl("--", "create", "load_balancer",
                                  "external_ids:TCP_lb_gateway_router=" +
                                  gateway_router)

    k8s_ns_lb_udp = ovn_nbctl("--data=bare", "--no-heading",
                              "--columns=_uuid", "find", "load_balancer",
                              "external_ids:UDP_lb_gateway_router=" +
                              gateway_router)
    if not k8s_ns_lb_udp:
        k8s_ns_lb_udp = ovn_nbctl("--", "create", "load_balancer",
                                  "external_ids:UDP_lb_gateway_router=" +
                                  gateway_router, "protocol=udp")

    # Add north-south load-balancers to the gateway router.
    ovn_nbctl("set", "logical_router", gateway_router,
              "load_balancer=" + k8s_ns_lb_tcp)
    ovn_nbctl("add", "logical_router", gateway_router,
              "load_balancer", k8s_ns_lb_udp)

    # Create the external switch for the physical interface to connect to.
    external_switch = "ext_%s" % (node_name)
    ovn_nbctl("--may-exist", "ls-add", external_switch)

    if args.physical_interface:
        # Connect physical interface to br-int. Get its mac address
        iface_id = "%s_%s" % (args.physical_interface, node_name)
        ovs_vsctl("--", "--may-exist", "add-port", "br-int",
                  args.physical_interface, "--", "set", "interface",
                  args.physical_interface, "external-ids:iface-id=" + iface_id)

        mac_address = ovs_vsctl("--if-exists", "get", "interface",
                                args.physical_interface,
                                "mac_in_use").strip('"')

        # Flush the IP address of the physical interface.
        command = "ip addr flush dev %s" % (args.physical_interface)
        util.call_popen(shlex.split(command))
    else:
        # A OVS bridge's mac address can change when ports are added to it.
        # We cannot let that happen, so make the bridge mac address permanent.
        mac_address = ovs_vsctl("--if-exists", "get", "interface",
                                args.bridge_interface,
                                "mac_in_use").strip('"')
        if not mac_address:
            raise Exception("No mac_address found for the bridge-interface")

        ovs_vsctl("set", "bridge", args.bridge_interface,
                  "other-config:hwaddr=" + mac_address)

        iface_id = "%s_%s" % (args.bridge_interface, node_name)

        # Connect bridge interface to br-int via patch ports.
        patch1 = "k8s-patch-br-int-" + args.bridge_interface
        patch2 = "k8s-patch-" + args.bridge_interface + "-br-int"
        ovs_vsctl("--may-exist", "add-port", args.bridge_interface, patch2,
                  "--", "set", "interface", patch2, "type=patch",
                  "options:peer=" + patch1)
        ovs_vsctl("--may-exist", "add-port", "br-int", patch1,
                  "--", "set", "interface", patch1, "type=patch",
                  "options:peer=" + patch2,
                  "external-ids:iface-id=" + iface_id)

    # Add external interface as a logical port to external_switch. This is
    # a learning switch port with "unknown" address.  The external world
    # is accessed via this port.
    ovn_nbctl("--", "--may-exist", "lsp-add", external_switch,
              iface_id, "--", "lsp-set-addresses",
              iface_id, "unknown")

    # Connect GR to external_switch with mac address of external interface
    # and that IP address.
    ovn_nbctl("--", "--may-exist", "lrp-add", gateway_router,
              "rtoe-" + gateway_router, mac_address, str(physical_ip),
              "--", "set", "logical_router_port", "rtoe-" + gateway_router,
              "external-ids:gateway-physical-ip=yes")

    # Connect the external_switch to the router.
    ovn_nbctl("--", "--may-exist", "lsp-add", external_switch,
              "etor-" + gateway_router, "--", "set", "logical_switch_port",
              "etor-" + gateway_router, "type=router",
              "options:router-port=rtoe-" + gateway_router,
              "addresses=" + "\"" + mac_address + "\"")

    # Default SNAT rules.
    ovn_nbctl("--", "--id=@nat", "create", "nat", "type=snat",
              "logical_ip=" + args.cluster_ip_subnet,
              "external_ip=" + str(physical_ip.ip),
              "--", "add", "logical_router", gateway_router, "nat", "@nat")

    # When there are multiple gateway routers (which would be the likely
    # default for any sane deployment), we need to SNAT traffic heading
    # to the logical space with the Gateway router's IP so that return
    # traffic comes back to the same gateway router.
    if router_ip:
        ip = netaddr.IPNetwork(router_ip)
        ovn_nbctl("set", "logical_router", gateway_router,
                  "options:lb_force_snat_ip=" + str(ip.ip))

        if args.rampout_ip_subnets:
            try:
                rampout_ip_subnets = args.rampout_ip_subnets.split(',')
                for rampout_ip_subnet in rampout_ip_subnets:
                    try:
                        netaddr.IPNetwork(rampout_ip_subnet)
                    except Exception as e:
                        sys.stderr.write("warning: %s is not valid subnet (%s)"
                                         % (rampout_ip_subnet, str(e)))
                        continue

                    # Add source IP address based routes in distributed router
                    # for this gateway router.
                    ovn_nbctl("--may-exist", "--policy=src-ip", "lr-route-add",
                              k8s_cluster_router, rampout_ip_subnet,
                              str(ip.ip))
            except Exception as e:
                raise Exception("Failed to split args.rampout_ip_subnets %s"
                                % str(e))


def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(title='Subcommands',
                                       dest='command_name')

    # Parser for sub-command 'master-init'.
    parser_master_init = subparsers.add_parser(
                                           'master-init',
                                           help="Initialize k8s master node")
    parser_master_init.add_argument('--cluster-ip-subnet', required=True,
                                    help="The cluster wide larger subnet of "
                                    "private ip addresses.")
    parser_master_init.add_argument('--master-switch-subnet', required=True,
                                    help="The smaller subnet just for master.")
    parser_master_init.add_argument('--node-name',
                                    required=True,
                                    help="A unique node name.")
    parser_master_init.set_defaults(func=master_init)

    # Parser for sub-command 'minion-init'.
    parser_minion_init = subparsers.add_parser(
                                           'minion-init',
                                           help="Initialize k8s minion node")
    parser_minion_init.add_argument('--cluster-ip-subnet', required=True,
                                    help="The cluster wide larger subnet of "
                                    "private ip addresses.")
    parser_minion_init.add_argument('--minion-switch-subnet', required=True,
                                    help="The smaller subnet just for this "
                                    "master.")
    parser_minion_init.add_argument('--node-name',
                                    required=True,
                                    help="A unique node name.")
    parser_minion_init.set_defaults(func=minion_init)

    # Parser for sub-command 'gateway-init'.
    parser_gateway_init = subparsers.add_parser(
                                           'gateway-init',
                                           help="Initialize k8s gateway node")
    parser_gateway_init.add_argument('--cluster-ip-subnet', required=True,
                                     help="The cluster wide larger subnet of "
                                     "private ip addresses.")
    parser_gateway_init.add_argument('--physical-interface',
                                     help="The physical interface via which "
                                     "external connectivity is provided.")
    parser_gateway_init.add_argument('--bridge-interface',
                                     help="The OVS bridge interface via which "
                                     "external connectivity is provided.")
    parser_gateway_init.add_argument('--physical-ip', required=True,
                                     help="The ip address of the physical "
                                     "interface or bridge interface via which "
                                     "external connectivity is provided. This "
                                     "should be of the form IP/MASK.")
    parser_gateway_init.add_argument('--default-gw',
                                     help="The next hop IP address for your "
                                     "physical interface.")
    parser_gateway_init.add_argument('--node-name',
                                     required=True,
                                     help="A unique node name.")
    parser_gateway_init.add_argument('--rampout-ip-subnets',
                                     help="Uses this gateway to rampout "
                                     "traffic originating from the specified "
                                     "comma separated ip subnets.  Used to "
                                     "distribute outgoing traffic via "
                                     "multiple gateways.")
    parser_gateway_init.set_defaults(func=gateway_init)

    args = parser.parse_args()
    args.func(args)


if __name__ == '__main__':
    try:
        main()
        print("Initialization complete.")
    except Exception as e:
        sys.stderr.write("Failed operation.\n(%s)\n at %s"
                         % (str(e), traceback.format_exc()))
        sys.exit(1)