default

{% set daemon_args = "$DAEMON_ARGS" -%}
{% if grains['os_family'] == 'RedHat' -%}
  {% set daemon_args = "" -%}
{% endif -%}

{% if grains.api_servers is defined -%}
  {% set api_servers = "--api-servers=https://" + grains.api_servers -%}
{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form
  {% set api_servers = "--api-servers=https://" + grains.apiservers -%}
{% elif grains['roles'][0] == 'kubernetes-master' -%}
  {% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%}
  {% set api_servers = "--api-servers=https://" + master_ipv4 -%}
{% else -%}
  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
  {% set api_servers = "--api-servers=https://" + ips[0][0] -%}
{% endif -%}

# TODO: remove nginx for other cloud providers.
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy']  %}
  {% set api_servers_with_port = api_servers -%}
{% else -%}
  {% set api_servers_with_port = api_servers + ":6443" -%}
{% endif -%}

{% set master_kubelet_args = "" %}

{% set debugging_handlers = "--enable-debugging-handlers=true" -%}

{% set reconcile_cidr_args = "" -%}
{% if grains['roles'][0] == 'kubernetes-master' -%}
  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack', 'azure-legacy'] -%}

    # Unless given a specific directive, disable registration for the kubelet
    # running on the master.
    {% if grains.kubelet_api_servers is defined -%}
      {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%}
      {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%}
      {% set reconcile_cidr_args =  "--reconcile-cidr=false" -%}
    {% else -%}
      {% set api_servers_with_port = "" -%}
    {% endif -%}

    # Disable the debugging handlers (/run and /exec) to prevent arbitrary
    # code execution on the master.
    # TODO(roberthbailey): Relax this constraint once the master is self-hosted.
    {% set debugging_handlers = "--enable-debugging-handlers=false" -%}
  {% endif -%}
{% endif -%}

{% set cloud_provider = "" -%}
{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
  {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
{% endif -%}

{% set cloud_config = "" -%}
{% if grains.cloud in [ 'openstack', 'vsphere' ] and grains.cloud_config is defined -%}
  {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
{% endif -%}

{% set config = "--config=/etc/kubernetes/manifests" -%}

{% set manifest_url = "" -%}
{% set manifest_url_header = "" -%}
{% if pillar.get('enable_manifest_url', '').lower() == 'true' %}
  {% set manifest_url = "--manifest-url=" + pillar['manifest_url'] + " --manifest-url-header=" + pillar['manifest_url_header'] -%}
{% endif -%}

{% set hostname_override = "" -%}
{% if grains.hostname_override is defined -%}
  {% set hostname_override = " --hostname-override=" + grains.hostname_override -%}
{% endif -%}

{% set cluster_dns = "" %}
{% set cluster_domain = "" %}
{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %}
  {% set cluster_dns = "--cluster-dns=" + pillar['dns_server'] %}
  {% set cluster_domain = "--cluster-domain=" + pillar['dns_domain'] %}
{% endif %}

{% set docker_root = "" -%}
{% if grains.docker_root is defined -%}
  {% set docker_root = " --docker-root=" + grains.docker_root -%}
{% endif -%}

{% set kubelet_root = "" -%}
{% if grains.kubelet_root is defined -%}
  {% set kubelet_root = " --root-dir=" + grains.kubelet_root -%}
{% endif -%}

{% set configure_cbr0 = "" -%}
{% if pillar['allocate_node_cidrs'] is defined -%}
  {% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%}
{% endif -%}

{% set non_masquerade_cidr = "" -%}
{% if pillar.get('non_masquerade_cidr','') -%}
  {% set non_masquerade_cidr = "--non-masquerade-cidr=" + pillar.non_masquerade_cidr -%}
{% endif -%}

# The master kubelet cannot wait for the flannel daemon because it is responsible
# for starting up the flannel server in a static pod. So even though the flannel
# daemon runs on the master, it doesn't hold up cluster bootstrap. All the pods
# on the master run with host networking, so the master flannel doesn't care
# even if the network changes. We only need it for the master proxy.
{% set experimental_flannel_overlay = "" -%}
{% if pillar.get('network_provider', '').lower() == 'flannel' and grains['roles'][0] != 'kubernetes-master' %}
  {% set experimental_flannel_overlay = "--experimental-flannel-overlay=true" %}
{% endif -%}

# Setup cgroups hierarchies.
{% set cgroup_root = "" -%}
{% set system_container = "" -%}
{% set kubelet_container = "" -%}
{% set runtime_container = "" -%}
{% if grains['os_family'] == 'Debian' -%}
  {% if pillar.get('is_systemd') %}
    {% set cgroup_root = "--cgroup-root=docker" -%}
  {% else %}
    {% set cgroup_root = "--cgroup-root=/" -%}
    {% set system_container = "--system-cgroups=/system" -%}
    {% set runtime_container = "--runtime-cgroups=/docker-daemon" -%}
    {% set kubelet_container= "--kubelet-cgroups=/kubelet" -%}
  {% endif %}
{% endif -%}
{% if grains['oscodename'] in ['vivid','wily'] -%}
  {% set cgroup_root = "--cgroup-root=docker" -%}
{% endif -%}

{% set pod_cidr = "" %}
{% if grains['roles'][0] == 'kubernetes-master' %}
  {% if grains.get('cbr-cidr') %}
    {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %}
  {% elif api_servers_with_port == '' and pillar.get('network_provider', '').lower() == 'kubenet' %}
    # Kubelet standalone mode needs a PodCIDR since there is no controller-manager
    {% set pod_cidr = "--pod-cidr=10.76.0.0/16" %}
  {% endif -%}
{% endif %}

{% set cpu_cfs_quota = "" %}
{% if pillar['enable_cpu_cfs_quota'] is defined -%}
 {% set cpu_cfs_quota = "--cpu-cfs-quota=" + pillar['enable_cpu_cfs_quota'] -%}
{% endif -%}

{% set feature_gates = "" -%}
{% if grains['feature_gates'] is defined -%}
  {% set feature_gates = "--feature-gates=" + grains['feature_gates'] -%}
{% endif %}

{% set test_args = "" -%}
{% if pillar['kubelet_test_args'] is defined -%}
  {% set test_args=pillar['kubelet_test_args'] %}
{% endif -%}

{% set network_plugin = "" -%}
{% if pillar.get('network_provider', '').lower() == 'opencontrail' %}
  {% set network_plugin = "--network-plugin=opencontrail" %}
{% elif pillar.get('network_provider', '').lower() == 'cni' %}
  {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %}
{%elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' -%}
  {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %}
{% elif pillar.get('network_provider', '').lower() == 'kubenet' %}
  {% set network_plugin = "--network-plugin=kubenet" -%}
  {% if reconcile_cidr_args == '' -%}
    {% set reconcile_cidr_args =  "--reconcile-cidr=true" -%}
  {% endif -%}
{% endif -%}

# Don't pipe the --hairpin-mode flag by default. This allows the kubelet to pick
# an appropriate value.
{% set hairpin_mode = "" -%}
# The master cannot see Services because it doesn't run kube-proxy, so we don't
# need to make its container bridge promiscuous. We also don't want to set
# the hairpin-veth flag on the master because it increases the chances of
# running into the kernel bug described in #20096.
{% if grains['roles'][0] == 'kubernetes-master' -%}
  {% set hairpin_mode = "--hairpin-mode=none" -%}
{% elif pillar['hairpin_mode'] is defined and pillar['hairpin_mode'] in ['promiscuous-bridge', 'hairpin-veth', 'none'] -%}
  {% set hairpin_mode = "--hairpin-mode=" + pillar['hairpin_mode'] -%}
{% endif -%}

{% set babysit_daemons = "" -%}
{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce' ]  %}
  {% set babysit_daemons = "--babysit-daemons=true" -%}
{% endif -%}

{% set kubelet_port = "" -%}
{% if pillar['kubelet_port'] is defined -%}
  {% set kubelet_port="--port=" + pillar['kubelet_port'] %}
{% endif -%}

{% set log_level = pillar['log_level'] -%}
{% if pillar['kubelet_test_log_level'] is defined -%}
  {% set log_level = pillar['kubelet_test_log_level'] -%}
{% endif -%}

{% set enable_custom_metrics = "" -%}
{% if pillar['enable_custom_metrics'] is defined -%}
  {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}
{% endif -%}

{% set node_labels = "" %}
{% if pillar['node_labels'] is defined -%}
  {% set node_labels="--node-labels=" + pillar['node_labels'] %}
{% endif -%}

{% set eviction_hard = "" %}
{% if pillar['eviction_hard'] is defined -%}
  {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %}
{% endif -%}

# test_args has to be kept at the end, so they'll overwrite any prior configuration
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{feature_gates}} {{test_args}}"