Mergebot Train - May 4th

gen/build_deploy/bash.py

@@ -361,6 +361,47 @@ def validate_custom_check_bins_dir(custom_check_bins_dir):
     fi
 }
 
+function d_type_enabled_if_xfs()
+{
+    # Return 1 if $1 is a directory on XFS volume with ftype ! = 1
+    # otherwise return 0
+    DIRNAME="$1"
+
+    RC=0
+    # "df", the command being used to get the filesystem device and type,
+    # fails if the directory does not exist, hence we need to iterate up the
+    # directory chain to find a directory that exists before executing the command
+    while [[ ! -d "$DIRNAME" ]]; do
+        DIRNAME="$(dirname "$DIRNAME")"
+    done
+    read -r filesystem_device filesystem_type <<<"$(df --portability --print-type "$DIRNAME" | awk 'END{print $1,$2}')"
+    if [[ "$filesystem_type" == "xfs" ]]; then
+        echo -n -e "Checking if $DIRNAME is mounted with \"fytpe=1\": "
+        ftype_value="$(xfs_info $filesystem_device | grep -oE ftype=[0-9])"
+        if [[ "$ftype_value" != "ftype=1" ]]; then
+            RC=1
+        fi
+        print_status $RC "${NORMAL}(${ftype_value})"
+    fi
+    return $RC
+}
+
+# check node storage has d_type (ftype=1) support enabled if using XFS
+function check_xfs_ftype() {
+    RC=0
+
+    mesos_agent_dir="{{ mesos_agent_work_dir }}"
+    # Check if ftype=1 on the volume, for $mesos_agent_dir, if its on XFS filesystem
+    ( d_type_enabled_if_xfs "$mesos_agent_dir" ) || RC=1
+
+    # Check if ftype=1 on the volume, for docker root dir, if its on XFS filesystem
+    docker_root_dir="$(docker info | grep 'Docker Root Dir' | cut -d ':' -f 2  | tr -d '[[:space:]]')"
+    ( d_type_enabled_if_xfs "$docker_root_dir" ) || RC=1
+
+    (( OVERALL_RC += $RC ))
+    return $RC
+}
+
 function check_all() {
     # Disable errexit because we want the preflight checks to run all the way
     # through and not bail in the middle, which will happen as it relies on
@@ -436,6 +477,7 @@ def validate_custom_check_bins_dir(custom_check_bins_dir):
     check unzip
     check ipset
     check systemd-notify
+    check ifconfig
 
     # $ systemctl --version ->
     # systemd nnn
@@ -492,6 +534,7 @@ def validate_custom_check_bins_dir(custom_check_bins_dir):
         do
             check_service $service
         done
+        check_xfs_ftype
     fi
 
     # Check we're not in docker on devicemapper loopback as storage driver.
@@ -654,7 +697,8 @@ def make_bash(gen_out) -> None:
         'dcos_image_commit': util.dcos_image_commit,
         'generation_date': util.template_generation_date,
         'setup_flags': setup_flags,
-        'setup_services': setup_services})
+        'setup_services': setup_services,
+        'mesos_agent_work_dir': gen_out.arguments['mesos_agent_work_dir']})
 
     # Output the dcos install script
     install_script_filename = 'dcos_install.sh'

gen/calc.py

@@ -317,6 +317,9 @@ def validate_dcos_overlay_network(dcos_overlay_network):
         assert 'vtep_mac_oui' in overlay_network.keys(), (
             'Missing "vtep_mac_oui" in overlay configuration {}'.format(overlay_network))
 
+        vtep_mtu = overlay_network.get('vtep_mtu', 1500)
+        validate_int_in_range(vtep_mtu, 552, None)
+
         if 'subnet' in overlay:
             # Check the VTEP IP is present in the overlay configuration
             assert 'vtep_subnet' in overlay_network, (
@@ -764,6 +767,11 @@ def calculate_check_config(check_time):
                     'cmd': ['/opt/mesosphere/bin/dcos-checks', 'executable', 'unzip'],
                     'timeout': '1s'
                 },
+                'ifconfig': {
+                    'description': 'The ifconfig utility is available',
+                    'cmd': ['/opt/mesosphere/bin/dcos-checks', 'executable', 'ifconfig'],
+                    'timeout': '1s'
+                },
                 'ip_detect_script': {
                     'description': 'The IP detect script produces valid output',
                     'cmd': ['/opt/mesosphere/bin/dcos-checks', 'ip'],
@@ -795,6 +803,7 @@ def calculate_check_config(check_time):
                 'tar',
                 'curl',
                 'unzip',
+                'ifconfig',
                 'ip_detect_script',
                 'mesos_master_replog_synchronized',
                 'mesos_agent_registered_with_masters',

gen/dcos-config.yaml

@@ -103,6 +103,13 @@ package:
         "IPSources": {{ mesos_dns_ip_sources }},
         "SetTruncateBit": {{ mesos_dns_set_truncate_bit }}
       }
+  - path: /etc/dcos.network
+    content: |
+      [Match]
+      Name=docker* m-* d-* vtep* spartan minuteman
+
+      [Link]
+      Unmanaged=yes
   - path: /etc/dcos_net
     content: |
       DCOS_NET_WATCHDOG={{ dcos_net_watchdog }}
@@ -489,6 +496,7 @@ package:
       MESOS_LAUNCHER_DIR=/opt/mesosphere/active/mesos/libexec/mesos
       MESOS_EXECUTOR_ENVIRONMENT_VARIABLES=file:///opt/mesosphere/etc/mesos-executor-environment.json
       MESOS_EXECUTOR_REGISTRATION_TIMEOUT=10mins
+      MESOS_EXECUTOR_REREGISTRATION_TIMEOUT=10secs
       MESOS_RECONFIGURATION_POLICY=additive
       MESOS_RECOVERY_TIMEOUT={{ mesos_recovery_timeout }}
       MESOS_CGROUPS_ENABLE_CFS=true

owners.json

@@ -3,15 +3,16 @@
   "asridharan ": {"slack": "avinash", "jira": "avinash"},
   "branden": {"slack": "branden", "jira": "branden"},
   "gpaul": {"slack": "gpaul", "jira": "gpaul"},
+  "greggomann": {"slack": "greg", "jira": "greg"},
   "jgehrcke": {"slack": "jp", "jira": "jp"},
+  "karya0": {"slack": "kapil", "jira": "kapil"},
   "klueska": {"slack": "klueska", "jira": "klueska"},
   "MatApple": {"slack": "mat", "jira": "mat"},
   "mnaboka": {"slack": "mnaboka", "jira": "mnaboka"},
-  "mellenburg": {"slack": "mellenburg", "jira": "michael.ellenburg"},
   "meichstedt": {"slack": "matthias.eichstedt", "jira": "matthias.eichstedt"},
   "orlandohohmeier": {"slack": "orlando", "jira": "orlando"},
   "orsenthil" : {"slack": "skumaran", "jira": "skumaran"},
-  "spahl": {"slack": "seb", "jira": "sebastien"},
+  "rukletsov": {"slack": "alexr", "jira": "alexr"},
   "urbanserj": {"slack": "urbanserj", "jira": "sergeyurbanovich"},
   "vespian": {"slack": "prozlach", "jira": "prozlach"}
 }

packages/dcos-checks/buildinfo.json

@@ -2,7 +2,7 @@
   "single_source": {
     "kind": "git",
     "git": "https://github.com/dcos/dcos-checks.git",
-    "ref": "c5a667cd20713af21c618df48b0ab2b3fc5ed0a6",
+    "ref": "c34c5a76bd49bf5df224e0c8abca51a71dece475",
     "ref_origin": "master"
   },
   "state_directory": true,

packages/dcos-net/buildinfo.json

@@ -4,7 +4,7 @@
     "dcos-net": {
       "kind": "git",
       "git": "https://github.com/dcos/dcos-net.git",
-      "ref": "56db2d021378db6f4aa9a6d2b7d1b06bcc809753",
+      "ref": "73fe6d28a38dfcc58347615493958be37ce97bf9",
       "ref_origin": "master"
     }
   },

packages/dcos-net/extra/dcos-net-setup.py

@@ -11,9 +11,12 @@
 Also the script prevents from duplicating iptables rules [3]
 
 [3] ExecStartPre=/path/dcos-net-setup.py iptables --wait -A FORWARD -j ACCEPT
+
+The script allows to add configuration for networkd
 """
 
 import os
+import shutil
 import subprocess
 import sys
 
@@ -37,10 +40,38 @@ def main():
         else:
             del sys.argv[1]
             result = subprocess.run(sys.argv)
+    elif sys.argv[1:3] == ['networkd', 'add'] and len(sys.argv) == 4:
+        result = add_networkd_config(sys.argv[3])
     else:
         result = subprocess.run(sys.argv[1:])
     sys.exit(result.returncode)
 
 
+def add_networkd_config(src):
+    networkd = b'systemd-networkd.service'
+
+    # check if there is networkd
+    result = subprocess.run(['systemctl', 'list-unit-files', networkd],
+                            stdout=subprocess.PIPE)
+    if result.returncode != 0:
+        return result
+    if networkd not in result.stdout:
+        return result
+
+    # copy the configuration
+    bname = os.path.basename(src)
+    dst = os.path.join('/etc/systemd/network', bname)
+    shutil.copyfile(src, dst)
+
+    # Restart networkd only if it's active
+    result = subprocess.run(['systemctl', 'is-active', networkd],
+                            stdout=subprocess.PIPE)
+    if result.returncode != 0:
+        result.returncode = 0
+        return result
+
+    return subprocess.run(['systemctl', 'restart', networkd])
+
+
 if __name__ == "__main__":
     main()

packages/dcos-net/extra/dcos-net.service

@@ -21,6 +21,7 @@ ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py ipta
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py mkdir -p /var/lib/dcos/navstar/mnesia
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py mkdir -p /var/lib/dcos/navstar/lashup
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py modprobe dummy
+ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py networkd add /opt/mesosphere/etc/dcos.network
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py ip link add minuteman type dummy
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py ip link set minuteman up
 ExecStartPre=/opt/mesosphere/active/dcos-net/dcos-net/bin/dcos-net-setup.py ip link add spartan type dummy

packages/marathon/build

@@ -37,6 +37,7 @@ EnvironmentFile=-/opt/mesosphere/etc/marathon-extras
 EnvironmentFile=-/var/lib/dcos/marathon/environment.ip.marathon
 Environment=JAVA_HOME=${JAVA_HOME}
 ExecStartPre=/bin/ping -c1 leader.mesos
+ExecStartPre=/bin/ping -c1 zk-1.zk
 ExecStartPre=/opt/mesosphere/bin/bootstrap dcos-marathon
 ExecStartPre=/bin/bash -c 'echo "HOST_IP=\$(\$MESOS_IP_DISCOVERY_COMMAND)" > /var/lib/dcos/marathon/environment.ip.marathon'
 ExecStartPre=/bin/bash -c 'echo "MARATHON_HOSTNAME=\$(\$MESOS_IP_DISCOVERY_COMMAND)" >> /var/lib/dcos/marathon/environment.ip.marathon'

packages/mesos/extra/start_mesos.sh

@@ -8,29 +8,4 @@ if [ -x $FAULT_DOMAIN_SCRIPT ]; then
   export MESOS_DOMAIN="$($FAULT_DOMAIN_SCRIPT)"
 fi
 
-# Helper to configure `networkd` on CoreOS to ignore all DC/OS overlay interfaces.
-function coreos_networkd_config() {
- network_config="/etc/systemd/network/dcos.network"
- sudo tee $network_config > /dev/null<<'EOF'
- [Match]
-  Type=bridge
-  Name=docker* m-* d-* vtep*
-
- [Link]
-  Unmanaged=yes
-EOF
-}
-
-distro="$(source /etc/os-release && echo "${ID}")"
-if [[ "${distro}" == 'coreos' ]]; then
-     if systemctl list-unit-files | grep systemd-networkd.service > /dev/null; then
-       coreos_networkd_config
-       echo "Configuring systemd-networkd to ignore docker bridge and DC/OS overlay interfaces..."
-
-       if systemctl is-enabled systemd-networkd > /dev/null; then
-          sudo systemctl restart systemd-networkd
-       fi
-    fi
-fi
-
 exec "$@"