Calico with Mesos and Kubernetes integrated

group_vars/all/kubernetes_vars.yml

@@ -23,12 +23,20 @@ kube_users:
     role: admin
 
 # Select which network manager to use. After choosing, edit kubernetes.yml file and include aproriate roles.
-# network: calico
-network: flannel
+network: calico
+#network: flannel
 
 # Kubernetes cluster name, also will be used as DNS domain
 cluster_name: cluster.local
 
+# Enable to leverage some unique features of a specific cloud provider.
+enable_cloud_provider: false
+
+# Only OpenStack provider is currently supported.
+cloud_provider: openstack
+
+cloud_provider_config: /etc/kubernetes/cloud-config
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.254.0.0/16
 

library/kube.py

@@ -250,6 +250,7 @@ def main():
             resource=dict(),
             label=dict(),
             server=dict(),
+            api_version=dict(default='v1', choices=['v1', 'v1beta3']),
             force=dict(default=False, type='bool'),
             all=dict(default=False, type='bool'),
             log_level=dict(default=0, type='int'),

roles/calico/defaults/main.yml

@@ -1,9 +1,11 @@
 ---
+calico_version: 0.15.0
+
 calico_image: calico/node
-calico_image_tag: v0.9.0
+calico_image_tag: v{{ calico_version }}
 
 bintray_baseurl: "https://bintray.com/artifact/download/asteris"
-calico_binary: "{{ bintray_baseurl }}/calico/calicoctl/calicoctl"
+calico_binary: "{{ bintray_baseurl }}/mantl-rpm/calico-{{ calico_version }}-1.x86_64.rpm"
 
 etcd_service_name: etcd.service.{{ consul_dns_domain }}
 etcd_client_port: 2379

roles/calico/files/modules.json

@@ -0,0 +1,26 @@
+{
+  "libraries": [
+    {
+      "file": "/usr/local/lib/mesos/libmesos_network_isolator.so",
+      "modules": [
+        {
+          "name": "com_mesosphere_mesos_NetworkIsolator",
+          "parameters": [
+            {
+              "key": "isolator_command",
+              "value": "/usr/local/bin/calico_mesos"
+            },
+            {
+              "key": "ipam_command",
+              "value": "/usr/local/bin/calico_mesos"
+            }
+          ]
+        },
+        {
+          "name": "com_mesosphere_mesos_NetworkHook"
+        }
+      ]
+    }
+  ]
+}
+

roles/calico/tasks/kubernetes.yml

@@ -1,19 +1,15 @@
----
-- name: Calico directory for plugins
+- name: Calico directory for cni config
   sudo: yes
   file:
-    path: /usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/
+    path: /etc/cni/net.d
     state: directory
   tags:
     - calico
 
-- name: Get calico-kubernetes plugin
+- name: Calico cni config for kubernetes
   sudo: yes
-  get_url:
-    url: https://github.com/projectcalico/calico-docker/releases/download/v0.5.4/calico_kubernetes
-    dest: /usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico
-    mode: 0755
+  template:
+    src: calico-plugin.conf
+    dest: /etc/cni/net.d/10-calico.conf
   tags:
     - calico
-
-

roles/calico/tasks/main.yml

@@ -1,55 +1,13 @@
 ---
-# Ansible modprobe module doesn't provide such functionality
-- name: add kernel modules to load at boot time
+- name: install calico 
   sudo: yes
-  copy:
-    src: calico-modules.conf
-    dest: /etc/modules-load.d/calico-modules.conf
-    owner: root
-    group: root
-    mode: 0644
-  tags:
-    - calico
-
-- name: load kernel module xt_set
-  sudo: yes
-  modprobe:
-    name: xt_set
+  yum:
+    name: "{{ calico_binary }}"
     state: present
   tags:
     - calico
-
-- name: enable kernel packet forwarding
-  sudo: yes
-  sysctl:
-    name: net.ipv4.ip_forward
-    value: 1
-    state: present
-    reload: yes
-  tags:
-    - calico
-
-- name: download calicoctl binary
-  sudo: yes
-  get_url:
-    url: "{{ calico_binary }}"
-    dest: /usr/bin/calicoctl
-    mode: 0755
-  tags:
-    - calico
     - bootstrap
 
-#- name: download calicoctl bash completion script
-#  sudo: yes
-#  get_url:
-#    url: "{{ bintray_baseurl }}/calico/calicoctl.sh"
-#    dest: /etc/bash_completion.d/calicoctl.sh
-#    mode: 0644
-#  tags:
-#    - calico
-
-# We can't use the Ansible standard docker module as it always starts
-# containers after image was pulled
 - name: ensure calico docker image is present
   sudo: yes
   command: /usr/bin/docker pull {{ calico_image }}:{{ calico_image_tag }}
@@ -107,11 +65,29 @@
   tags:
     - calico
 
+- name: calico etc directory
+  sudo: yes
+  file: 
+    path: /etc/calico
+    state: directory
+  tags:
+    - calico
+
+- name: put calico-mesos integration file
+  sudo: yes
+  copy:
+    src: modules.json
+    dest: /etc/calico/modules.json
+  tags:
+    - calico
+
 - include: openstack.yml
   when: provider == 'openstack'
 
 - include: kubernetes.yml
-  when: kube_build is defined
+  when: kube_version is defined
+
+- include: calico-configure.yml
 
 - name: reload calico systemd file
   sudo: yes
@@ -144,17 +120,5 @@
   failed_when: false
   changed_when: false
 
-- name: configure mesos-slave for networking integration
-  sudo: yes
-  template:
-    src: executor_environment_variables
-    dest: /etc/mesos-slave/executor_environment_variables
-    mode: 0644
-  notify:
-    - restart mesos agent
-  tags:
-    - calico
-
 - meta: flush_handlers
 
-- include: calico-configure.yml

roles/calico/templates/calico-plugin.conf

@@ -0,0 +1,9 @@
+{
+    "name": "ng_calico-k8s-network",
+    "type": "calico",
+    "etcd_authority": "{{calico_etcdaddr}}",
+    "log_level": "info",
+    "ipam": {
+        "type": "calico-ipam"
+    }
+}

roles/calico/templates/calico.env.j2

@@ -1,15 +1,2 @@
-# This node's IPv4 address
-DEFAULT_IPV4={{ private_ipv4 }}
-
 # IP and port of etcd instance used by Calico
 ETCD_AUTHORITY={{ calico_etcdaddr }}
-
-{% if hostvars[inventory_hostname]['role'] == 'kubeworker' %}
-# The Kubernetes master IP
-KUBERNETES_MASTER={{ groups[master_group_name][0] }}
-
-KUBE_API_ROOT=https://{{ groups[master_group_name][0] }}:{{ kube_master_port }}/api/v1/
-{% endif  %}
-
-# Enable Calcio IPAM
-CALICO_IPAM=true

roles/calico/templates/calico.service.j2

@@ -13,7 +13,7 @@ TimeoutStartSec=20m
 ExecStartPre=-/usr/bin/docker rm -f calico-node
 EnvironmentFile=/etc/default/calico
 ExecStart=/usr/bin/calicoctl node --ip={{ private_ipv4 }} \
-                                  --node-image={{ calico_image }}:{{ calico_image_tag }} {% if hostvars[inventory_hostname]['role'] == 'kubeworker' %}--kubernetes{% endif %}
+                                  --node-image={{ calico_image }}:{{ calico_image_tag }} 
 
 ExecStop=/usr/bin/calicoctl node stop
 ExecStopPost=-/usr/bin/docker rm -f calico-node

roles/kubernetes-addons/templates/skydns-rc.yaml.j2

@@ -29,7 +29,6 @@ spec:
         args:
         - -domain={{ dns_domain }}
         - -etcd-server=http://{{ private_ipv4 }}:{{ etcd_client_port }}
-        - -kubecfg_file=/etc/kubernetes/node.kubeconfig
         volumeMounts:
           - mountPath: /etc/kubernetes/
             name: kube-etc
@@ -50,4 +49,3 @@ spec:
       - name: kube-etc
         hostPath:
           path: /etc/kubernetes
-

roles/kubernetes-master/templates/kube-apiserver.yml

@@ -24,7 +24,8 @@ spec:
     - --client-ca-file={{ kube_cert_dir }}/ca.crt
     - --service-account-key-file={{ kube_cert_dir }}/server.key
     - --basic-auth-file={{ kube_users_dir }}/known_users.csv
-    - --v=3
+    - --runtime-config=extensions/v1beta1/daemonsets=true
+    - --v={{kube_log_level}}
     ports:
     - containerPort: {{ kube_master_port }}
       hostPort: {{ kube_master_port }}

roles/kubernetes-master/templates/kube-controller-manager.yml

@@ -14,6 +14,7 @@ spec:
     - --master=http://127.0.0.1:{{ kube_insecure_port }}
     - --service-account-private-key-file={{ kube_cert_dir }}/server.key
     - --root-ca-file={{ kube_cert_dir }}/ca.crt
+    - --v={{kube_log_level}}
     livenessProbe:
       httpGet:
         host: 127.0.0.1

roles/kubernetes-master/templates/kube-proxy.yml

@@ -14,6 +14,7 @@ spec:
     - --master=https://kubernetes.service.consul:{{ kube_master_port }}
     - --kubeconfig=/etc/kubernetes/master.kubeconfig
     - --proxy-mode=iptables
+    - --v={{kube_log_level}}
     securityContext:
       privileged: true
     volumeMounts:

roles/kubernetes-master/templates/kube-scheduler.yml

@@ -12,6 +12,7 @@ spec:
     - /hyperkube
     - scheduler
     - --master=http://127.0.0.1:{{ kube_insecure_port }}
+    - --v={{kube_log_level}}
     livenessProbe:
       httpGet:
         host: 127.0.0.1

roles/kubernetes-node/templates/kube-proxy.yml

@@ -14,6 +14,7 @@ spec:
     - --master=https://kubernetes.service.consul:{{ kube_master_port }}
     - --kubeconfig=/etc/kubernetes/node.kubeconfig
     - --proxy-mode=iptables
+    - --v={{kube_log_level}}
     securityContext:
       privileged: true
     volumeMounts:

roles/kubernetes/defaults/main.yml

@@ -33,3 +33,11 @@ dns_domain: "{{ cluster_name }}"
 # magic will actually pick the 10th ip address in the kube_service_addresses
 # range and use that.
 dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+
+# OpenStack cloud provider authentication variables.
+os_auth_url: "{{ lookup('env','OS_AUTH_URL') }}"
+os_username: "{{ lookup('env','OS_USERNAME') }}"
+os_password: "{{ lookup('env','OS_PASSWORD') }}"
+os_region_name: "{{ lookup('env','OS_REGION_NAME') }}"
+os_tenant_id: "{{ lookup('env','OS_TENANT_ID') }}"
+

roles/kubernetes/tasks/main.yml

@@ -1,13 +1,4 @@
 ---
-- name: evaluate first_master
-  add_host:
-    name: "{{ groups[master_group_name][0] }}"
-    groups: first_master
-    when: "master in groups and groups[master_group_name] | length > 1"
-  tags:
-    - common
-    - kubernetes
-
 - name: create kubernetes config directories
   sudo: yes
   file:

roles/kubernetes/tasks/secrets.yml

@@ -51,3 +51,16 @@
   tags:
     - secrets
     - kubernetes
+
+- name: create cloud provider config
+  sudo: yes
+  template:
+    src: cloud-config.j2
+    dest: "{{ cloud_provider_config }}"
+    owner: root
+    group: root
+    mode: 0644
+  when: enable_cloud_provider and cloud_provider == "openstack"
+  tags:
+    - secrets
+    - kubernetes

roles/marathon/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-marathon_version: 0.13.0
+marathon_version: 1.0.0_RC1
 marathon_package: "marathon-{{ marathon_version }}"
 
 marathon_zk_auth: "{% if zk_marathon_user_secret is defined %}{{ zk_marathon_user }}:{{ zk_marathon_user_secret }}@{% endif %}"

roles/mesos/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-mesos_version: 0.25.0
+mesos_version: 0.27.0
 
 mesos_package: "mesos-{{ mesos_version }}"
 mesos_master_package: "mesos-master-{{ mesos_version }}"

roles/mesos/templates/mesos-agent.sysconfig.j2

@@ -1,8 +1,7 @@
 # common configuration
 MESOS_PORT={{ mesos_follower_port }}
-MESOS_ADVERTISE_IP={{ private_ipv4 }}
-MESOS_IP={{ private_ipv4 }}
-MESOS_HOSTNAME={{ mesos_hostname | default(inventory_hostname) }}
+MESOS_HOSTNAME={{ mesos_hostname | default(inventory_hostname + ".node." + consul_dns_domain) }}
+MESOS_LOG_DIR={{ mesos_log_dir }}
 MESOS_EXTERNAL_LOG_FILE=/var/log/mesos/mesos-agent.log
 MESOS_LOGGING_LEVEL={{ mesos_logging_level }}
 
@@ -21,4 +20,9 @@ MESOS_WORK_DIR={{ mesos_work_dir }}
 # authentication
 {% if not (do_mesos_follower_auth|bool) %}# {% endif %}MESOS_CREDENTIAL=file:///etc/sysconfig/mesos-agent-credential
 
-EXTRA_OPTS=""
+EXTRA_OPTS="{% if calico_etcdaddr is defined %}--modules=file:///etc/calico/modules.json --isolation=com_mesosphere_mesos_NetworkIsolator --hooks=com_mesosphere_mesos_NetworkHook {% endif %}"
+
+# calico integration
+{% if calico_etcdaddr is defined %}
+ETCD_AUTHORITY={{ calico_etcdaddr }}
+{% endif %}