Workaround for Docker 1.11.2 startup issue

On a percentage of DC/OS agents (~5%) with Docker 1.11.2, the Daemon will fail to start up with the following error: > Error starting daemon: Error initializing network controller: Error > creating default "bridge" network: failed to allocate gateway > (172.17.0.1): Address already in use This seems to be related to a Docker bug around the network controller initialization, where the controller has allocated an ip pool and persisted some state but not all of it. See: * moby/moby#22834 * moby/moby#23078 This fix simply removes the docker0 interface if it exists before starting the Docker daemon. This fix will need to be re-evaluated if we want to enable the 1.12+ containerd live-restore like Docker options as discussed in: * https://docs.docker.com/engine/admin/live-restore/ * moby/moby#2658
lingmann · Sep 1, 2016 · 5b6e106 · 5b6e106
1 parent cb91274
commit 5b6e106
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 37 deletions.
diff --git a/cloud_images/CHANGELOG.md b/cloud_images/CHANGELOG.md
@@ -1,5 +1,9 @@
 ## Changelog
 
+### 2016-08-30
+
+* Fix intermittent issue with Docker 1.11.2 startup where a small percentage of agents (~5%) fail to start.
+
 ### 2016-08-09
 
 Move AMI builds to dcos.io CI: https://teamcity.mesosphere.io/project.html?projectId=DcosIo_Dcos_BuildCloudBaseImages&tab=projectOverview

diff --git a/cloud_images/centos7/install_prereqs.sh b/cloud_images/centos7/install_prereqs.sh
@@ -55,6 +55,10 @@ docker_service_d=/etc/systemd/system/docker.service.d
 mkdir -p "$docker_service_d"
 cat << 'EOF' > "${docker_service_d}/execstart.conf"
 [Service]
+Restart=always
+StartLimitInterval=0
+RestartSec=15
+ExecStartPre=-/sbin/ip link del docker0
 ExecStart=
 ExecStart=/usr/bin/docker daemon -H fd:// --graph=/var/lib/docker --storage-driver=overlay
 EOF

diff --git a/gen/aws/templates/advanced/advanced-master.json b/gen/aws/templates/advanced/advanced-master.json
@@ -72,43 +72,43 @@
     "RegionToAmi": {
       "ap-northeast-1": {
         "coreos": "ami-965899f7",
-        "el7": "ami-abf535ca"
+        "el7": "ami-264f8747"
       },
       "ap-southeast-1": {
         "coreos": "ami-3120fe52",
-        "el7": "ami-c47ba5a7"
+        "el7": "ami-0765bd64"
       },
       "ap-southeast-2": {
         "coreos": "ami-b1291dd2",
-        "el7": "ami-bc2f1bdf"
+        "el7": "ami-3f1a2c5c"
       },
       "eu-central-1": {
         "coreos": "ami-3ae31555",
-        "el7": "ami-d934c2b6"
+        "el7": "ami-846e9eeb"
       },
       "eu-west-1": {
         "coreos": "ami-b7cba3c4",
-        "el7": "ami-dabdd6a9"
+        "el7": "ami-250c7f56"
       },
       "sa-east-1": {
         "coreos": "ami-61e3750d",
-        "el7": "ami-ac980ec0"
+        "el7": "ami-0e019062"
       },
       "us-east-1": {
         "coreos": "ami-6d138f7a",
-        "el7": "ami-44f66853"
+        "el7": "ami-47096750"
       },
       "us-gov-west-1": {
         "coreos": "ami-b712acd6",
         "el7": ""
       },
       "us-west-1": {
         "coreos": "ami-ee57148e",
-        "el7": "ami-a883c0c8"
+        "el7": "ami-e4afe284"
       },
       "us-west-2": {
         "coreos": "ami-dc6ba3bc",
-        "el7": "ami-a4dd16c4"
+        "el7": "ami-ab07d1cb"
       }
     },
     "Parameters": {

diff --git a/gen/aws/templates/advanced/advanced-priv-agent.json b/gen/aws/templates/advanced/advanced-priv-agent.json
@@ -255,43 +255,43 @@
     "RegionToAmi": {
       "ap-northeast-1": {
         "coreos": "ami-965899f7",
-        "el7": "ami-abf535ca"
+        "el7": "ami-264f8747"
       },
       "ap-southeast-1": {
         "coreos": "ami-3120fe52",
-        "el7": "ami-c47ba5a7"
+        "el7": "ami-0765bd64"
       },
       "ap-southeast-2": {
         "coreos": "ami-b1291dd2",
-        "el7": "ami-bc2f1bdf"
+        "el7": "ami-3f1a2c5c"
       },
       "eu-central-1": {
         "coreos": "ami-3ae31555",
-        "el7": "ami-d934c2b6"
+        "el7": "ami-846e9eeb"
       },
       "eu-west-1": {
         "coreos": "ami-b7cba3c4",
-        "el7": "ami-dabdd6a9"
+        "el7": "ami-250c7f56"
       },
       "sa-east-1": {
         "coreos": "ami-61e3750d",
-        "el7": "ami-ac980ec0"
+        "el7": "ami-0e019062"
       },
       "us-east-1": {
         "coreos": "ami-6d138f7a",
-        "el7": "ami-44f66853"
+        "el7": "ami-47096750"
       },
       "us-gov-west-1": {
         "coreos": "ami-b712acd6",
         "el7": ""
       },
       "us-west-1": {
         "coreos": "ami-ee57148e",
-        "el7": "ami-a883c0c8"
+        "el7": "ami-e4afe284"
       },
       "us-west-2": {
         "coreos": "ami-dc6ba3bc",
-        "el7": "ami-a4dd16c4"
+        "el7": "ami-ab07d1cb"
       }
     }
   },

diff --git a/gen/aws/templates/advanced/advanced-pub-agent.json b/gen/aws/templates/advanced/advanced-pub-agent.json
@@ -247,43 +247,43 @@
     "RegionToAmi": {
       "ap-northeast-1": {
         "coreos": "ami-965899f7",
-        "el7": "ami-abf535ca"
+        "el7": "ami-264f8747"
       },
       "ap-southeast-1": {
         "coreos": "ami-3120fe52",
-        "el7": "ami-c47ba5a7"
+        "el7": "ami-0765bd64"
       },
       "ap-southeast-2": {
         "coreos": "ami-b1291dd2",
-        "el7": "ami-bc2f1bdf"
+        "el7": "ami-3f1a2c5c"
       },
       "eu-central-1": {
         "coreos": "ami-3ae31555",
-        "el7": "ami-d934c2b6"
+        "el7": "ami-846e9eeb"
       },
       "eu-west-1": {
         "coreos": "ami-b7cba3c4",
-        "el7": "ami-dabdd6a9"
+        "el7": "ami-250c7f56"
       },
       "sa-east-1": {
         "coreos": "ami-61e3750d",
-        "el7": "ami-ac980ec0"
+        "el7": "ami-0e019062"
       },
       "us-east-1": {
         "coreos": "ami-6d138f7a",
-        "el7": "ami-44f66853"
+        "el7": "ami-47096750"
       },
       "us-gov-west-1": {
         "coreos": "ami-b712acd6",
         "el7": ""
       },
       "us-west-1": {
         "coreos": "ami-ee57148e",
-        "el7": "ami-a883c0c8"
+        "el7": "ami-e4afe284"
       },
       "us-west-2": {
         "coreos": "ami-dc6ba3bc",
-        "el7": "ami-a4dd16c4"
+        "el7": "ami-ab07d1cb"
       }
     }
   },

diff --git a/gen/aws/templates/cloudformation.json b/gen/aws/templates/cloudformation.json
@@ -47,34 +47,34 @@
   "Mappings": {
     "RegionToAmi": {
       "ap-northeast-1": {
-        "stable": "ami-965899f7"
+        "stable": "ami-264f8747"
       },
       "ap-southeast-1": {
-        "stable": "ami-3120fe52"
+        "stable": "ami-0765bd64"
       },
       "ap-southeast-2": {
-        "stable": "ami-b1291dd2"
+        "stable": "ami-3f1a2c5c"
       },
       "eu-central-1": {
-        "stable": "ami-3ae31555"
+        "stable": "ami-846e9eeb"
       },
       "eu-west-1": {
-        "stable": "ami-b7cba3c4"
+        "stable": "ami-250c7f56"
       },
       "sa-east-1": {
-        "stable": "ami-61e3750d"
+        "stable": "ami-0e019062"
       },
       "us-east-1": {
-        "stable": "ami-6d138f7a"
+        "stable": "ami-47096750"
       },
       "us-gov-west-1": {
-        "stable": "ami-b712acd6"
+        "stable": ""
       },
       "us-west-1": {
-        "stable": "ami-ee57148e"
+        "stable": "ami-e4afe284"
       },
       "us-west-2": {
-        "stable": "ami-dc6ba3bc"
+        "stable": "ami-ab07d1cb"
       }
     },
     "NATAmi" : {

diff --git a/gen/azure/cloud-config.yaml b/gen/azure/cloud-config.yaml
@@ -19,6 +19,7 @@ root:
       Restart=always
       StartLimitInterval=0
       RestartSec=15
+      ExecStartPre=-/sbin/ip link del docker0
       ExecStart=
       ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay
   - path: /etc/systemd/system/docker.socket