docker deploy not deploying across swarm hosts #31121

nshCore · 2017-02-17T16:45:18Z

Im trying to deploy a docker-compose stack across a 3 node docker engine swarm mode cluster

The cluster looks like so

ID                           HOSTNAME   STATUS  AVAILABILITY  MANAGER STATUS
i4q2svsryhao18ziaolic8mp1    qg-node-3  Ready   Active        
tuszowwcqc0g3cix4fdhntl9s *  qg-node-1  Ready   Active        Leader
u1ufvlrtjjjluh7rrmn63eueg    qg-node-4  Ready   Active

All nodes are started with the --experimental daemon flag

All the nodes are logged into my private registry.

Here is a my docker-compose-swarm.yml


version: '3'

networks: 
  api:
    driver: overlay    
  echo:
    driver: overlay    
  cache:
    driver: overlay    
  search:
    driver: overlay
  webapp:
    driver: overlay        
  backend:
    driver: overlay    
  frontend:
    driver: overlay    
  messages:
    driver: overlay    
  database:
    driver: overlay    
  broadcaster:
    driver: overlay    
  redis-admin:
    driver: overlay    
  beanstalkd-admin:
    driver: overlay    

services:

  qg-rp:
    image: {some_registry}/v2/qg/rp
    volumes:
      - /srv/qg-rp/www:/srv/www
    depends_on:
      - "qg-webapp"          
      - "qg-api"
    networks:
      - frontend   
      - webapp
      - api
    container_name: "qg-rp"
    hostname: "qg-rp"
    expose:
      - 1025
    ports:
      - 443:1025
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true        
    deploy:
      placement:
        constraints: [node.role == manager] 
      labels: [APP=QG-RP]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s            

  qg-webapp:
    image: {some_registry}/v2/qg/webapp      
    volumes:
      - /srv/qg-webapp/www:/srv/www
    networks:
      - webapp
    container_name: "qg-webapp"
    hostname: "qg-webapp"
    expose:
      - 1025    
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true    
    deploy:     
      mode: global
      labels: [APP=API-WEBAPP]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s    

  qg-api:
    image: {some_registry}/v2/qg/api
    volumes:
      - /srv/qg-api/www:/srv/www
    networks:
      - api
      - backend 
    container_name: "qg-api"
    hostname:  "qg-api"
    expose:
      - 1025    
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true       
    deploy:
      mode: replicated
      replicas: 3
      labels: [APP=API-NGINX]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s      

  qg-api-fpm:
    image: {some_registry}v2/qg/api-fpm
    volumes:
      - /srv/qg-api-fpm/www:/srv/www
    networks:
      - backend
      - search
      - cache
      - messages
      - database
    container_name: "qg-api-fpm"
    hostname:  "qg-api-fpm"
    expose:
      - 9000      
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true        
    deploy:
      placement:
        constraints: [node.role == worker]       
      mode: replicated
      replicas: 2
      labels: [APP=API-FPM]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s             

  qg-queue-fpm:
    image:{some_registry}/v2/qg/queue-fpm      
    volumes:
      - /srv/qg-queue/www:/srv/www
    networks:
      - cache
      - messages
      - broadcaster
    expose:
      - 9000       
    container_name: "qg-queue-fpm"
    hostname:  "qg-queue-fpm"  
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true        
    deploy:
      placement:
        constraints: [node.role == worker] 
      labels: [APP=JOB_APP]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 2
        window: 120s      

  qg-echo:
    image: {some_registry}v2/qg/echo      
    volumes:
      - /srv/qg-echo/www:/srv/www
    networks:
      - echo
      - broadcaster   
    container_name: "qg-echo"
    hostname: "qg-echo"      
    expose:
      - 22622
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true      
    deploy:
      placement:
        constraints: [node.role == worker]      
      mode: replicated
      replicas: 2
      labels: [APP=ECHO]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s     

  qg-echo-rp: 
    image: {some_registry}/v2/qg/echo-rp      
    volumes:
      - /srv/qg-echo-rp/www:/srv/www
    networks:
      - frontend   
      - echo
    container_name: "qg-echo-rp"
    hostname: "qg-echo-rp"
    expose:
      - 1025
    ports:
      - 9090:1025
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true      
    deploy:
      placement:
        constraints: [node.role == manager] 
      labels: [APP=QG-ECHO-RP]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s                                          
  
  qg-beanstalkd:  
    image: {some_registry}/v2/qg/beanstalkd         
    volumes:
      - /srv/qg-beanstalkd:/srv
    container_name: "qg-beanstalkd"
    hostname: "qg-beanstalkd"
    networks:
      - beanstalkd-admin
      - messages 
      - broadcaster  
    expose:
      - 11300
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true         
    deploy:
      placement:
        constraints: [node.role == manager] 
      labels: [APP=QG-BEANSTALKD]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s        

  qg-elasticsarch-master:    
    image: {some_registry}/v2/qg/elasticsarch-master      
    volumes:
      - /srv/qg-elasticsearch:/srv
    networks:
      - search
    container_name: "qg-elasticsarch-master"
    hostname: "qg-elasticsarch-master"  
    expose:
      - 9200
      - 9300
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true
    privileged: true     
    deploy:
      placement:
        constraints: [node.role == manager] 
      labels: [APP=QG-ELASICSEARCH]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s      

  qg-redis:     
    image:{some_registry}:8080/v2/qg/redis      
    volumes:
      - /srv/qg-redis:/srv
    networks:
      - echo
      - broadcaster
      - redis-admin
    container_name: "qg-redis"
    hostname: "qg-redis"          
    expose:
      - 6379
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true      
    deploy:
      mode: replicated
      replicas: 2
      placement:
        constraints: [node.role == worker]       
      labels: [APP=QG-REDIS]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s    

  qg-memcached:
    image: {some_registry}:8080/v2/qg/memcached
    networks:
      - cache
    container_name: "qg-memcached"
    hostname: "qg-memcached"
    expose:
      - 11211
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true    
    deploy:
      mode: replicated
      replicas: 2
      placement:
        constraints: [node.role == worker]       
      labels: [APP=QG-MEMCACHED]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s          

  qg-sql:
    image: registry.ops.quickgroom.co.uk:8080/v2/qg/sql      
    environment:
      MYSQL_ROOT_PASSWORD: {summet_secure}
      MYSQL_PASSWORD: {summet_secure}
      MYSQL_DATABASE: {some_name}
      MYSQL_USER: {some_name}
    volumes:
      - /srv/qg-sql:/srv/data
    networks:
      - database
    container_name: "qg-sql"
    hostname: "qg-sql"      
    expose:
      - 3306
    environment:
      MYSQL_USER: qg-dba
      MYSQL_DATABASE: qg-db
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000
    read_only: false
    shm_size: 64M
    stdin_open: true                  
    deploy:
      placement:
        constraints: [node.role == manager] 
      labels: [APP=QG-SQL]
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3
        window: 120s

im running the deploy command from the swarm manager 'qg-node-1'

docker stack deploy -c docker-compose-swarm.yml up --with-registry-auth

also tried

docker deploy -c docker-compose-swarm.yml qg-prod

and only the services constrained to the master node are started. Looking through the example app isnt the expected behaviour to start all services regardless of what node they are constrained on?

Am i missing something here?

relevant docker info

docker -v
Docker version 1.13.1, build 092cba3

docker-compose -v
docker-compose version 1.11.1, build 7c5d5e4

qg-node-1

docker info
Containers: 14
Running: 10
Paused: 0
Stopped: 4
Images: 84
Server Version: 1.13.1
Storage Driver: aufs
Root Dir: /var/lib/docker/aufs
Backing Filesystem: extfs
Dirs: 198
Dirperm1 Supported: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Swarm: active
NodeID: tuszowwcqc0g3cix4fdhntl9s
Is Manager: true
ClusterID: bljkirz2n473gwshlne2jmraq
Managers: 1
Nodes: 3
Orchestration:
Task History Retention Limit: 5
Raft:
Snapshot Interval: 10000
Number of Old Snapshots to Retain: 0
Heartbeat Tick: 1
Election Tick: 3
Dispatcher:
Heartbeat Period: 5 seconds
CA Configuration:
Expiry Duration: 3 months
Node Address: {someip}
Manager Addresses:
{someip}:2377
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: aa8187dbd3b7ad67d8e5e3a15115d3eef43a7ed1
runc version: 9df8b306d01f59d3a8029be411de015b7304dd8f
init version: 949e6fa
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 4.4.0-62-generic
Operating System: Ubuntu 16.04.1 LTS
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 992.4 MiB
Name: qg-node-1
ID: KPGO:KDTE:MY4Z:SOUY:SVPA:7BPS:THGP:YZR2:EL35:PYMY:K36L:VOSY
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
WARNING: No swap limit support
Labels:
provider=digitalocean
Experimental: true
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false

qg-node-2

Containers: 0
Running: 0
Paused: 0
Stopped: 0
Images: 3
Server Version: 1.13.1
Storage Driver: aufs
Root Dir: /var/lib/docker/aufs
Backing Filesystem: extfs
Dirs: 10
Dirperm1 Supported: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Swarm: active
NodeID: i4q2svsryhao18ziaolic8mp1
Is Manager: false
Node Address: {someip}
Manager Addresses:
{someip}:2377
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: aa8187dbd3b7ad67d8e5e3a15115d3eef43a7ed1
runc version: 9df8b306d01f59d3a8029be411de015b7304dd8f
init version: 949e6fa
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 4.4.0-62-generic
Operating System: Ubuntu 16.04.1 LTS
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 488.4 MiB
Name: qg-node-3
ID: TD23:4VN6:RHKK:Y3QY:ZNVP:VVTS:4X65:6KTT:TBC4:KBD7:3YVE:ZRQW
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
WARNING: No swap limit support
Labels:
provider=digitalocean
Experimental: true
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false

qg-node-4

Containers: 0
Running: 0
Paused: 0
Stopped: 0
Images: 0
Server Version: 1.13.1
Storage Driver: aufs
Root Dir: /var/lib/docker/aufs
Backing Filesystem: extfs
Dirs: 0
Dirperm1 Supported: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Swarm: active
NodeID: u1ufvlrtjjjluh7rrmn63eueg
Is Manager: false
Node Address: {someip}
Manager Addresses:
{someip}:2377
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: aa8187dbd3b7ad67d8e5e3a15115d3eef43a7ed1
runc version: 9df8b306d01f59d3a8029be411de015b7304dd8f
init version: 949e6fa
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 4.4.0-62-generic
Operating System: Ubuntu 16.04.1 LTS
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 488.4 MiB
Name: qg-node-4
ID: 7JRT:2M43:AVPF:25VI:IIHK:AFQN:NCII:FGN7:YP7E:7H25:PPLB:F733
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
WARNING: No swap limit support
Labels:
provider=digitalocean
Experimental: true
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false

The text was updated successfully, but these errors were encountered:

willseward · 2017-02-17T17:22:45Z

Swarm won't schedule services with local volumes across hosts.

nshCore · 2017-02-17T17:28:58Z

Any documentation on this, as I was just planning on NFS sharing a folder across nodes

willseward · 2017-02-17T17:39:27Z

I'm not sure about the docs. I'll take a look.

Did you mean NFS? If so, you'll have to use a driver for the mount, I believe. If you actually meant NTFS, I have no clue. I don't think Linux likes NTFS very much, haha...

willseward · 2017-02-17T17:41:11Z

Here's this FWIW: #24089

nshCore · 2017-02-17T20:16:57Z

Yep typo meant NFS, I installed the NFS plugin and re-jigged the volumes to take advantage of it, and its getting somewhere now, but it has only booted the memcached containers on the two worker nodes

av0p844cq3vd qg-prod_qg-memcached.2 someregistry/v2/qg/memcached:latest qg-node-3 Running Running 10 minutes ago

everything else is just pending

av0p844cq3vd qg-prod_qg-memcached.2 someregistryv2/qg/memcached:latest qg-node-3 Running Running 10 minutes ago

I can see the NFS mounted on the workers so they have access to it

willseward · 2017-02-17T21:03:19Z

Looks like a typo to me:

image: {some_registry}v2/qg/api-fpm

should be:

image: {some_registry}/v2/qg/api-fpm

willseward · 2017-02-17T21:04:43Z

I am going to jump out of this thread. I believe your questions would be better suited for the Docker Forums since there is no identifiable bug in Docker itself.

cpuguy83 · 2017-02-17T21:09:59Z

Can you provide any details on the actual state of the services? You should be able to do docker stack ps <stack> to see all the tasks, along with errors

nshCore · 2017-02-18T06:57:57Z

@willseward that's just a typo.

It looks like none of the plugins mentioned on the docker site actually support docker 1.13 and nfs shares so now im a bit dumbfounded on how im actually supposed to deploy this in a swarm cluster, so i guess you can close this

nshCore · 2017-02-18T06:59:48Z

@cpuguy83 this is the output i put above the error column is blank

pbaderia01 · 2019-10-08T16:20:39Z

Hi, @cpuguy83 @thaJeztah @tiborvass this issue is safe to close. Can you please close this?

thaJeztah · 2019-10-08T22:25:24Z

yup, let's close this one as it's really old and not enough information to investigate

thaJeztah closed this as completed Oct 8, 2019

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

docker deploy not deploying across swarm hosts #31121

docker deploy not deploying across swarm hosts #31121

nshCore commented Feb 17, 2017

willseward commented Feb 17, 2017

nshCore commented Feb 17, 2017 •

edited

willseward commented Feb 17, 2017

willseward commented Feb 17, 2017

nshCore commented Feb 17, 2017 •

edited

willseward commented Feb 17, 2017

willseward commented Feb 17, 2017

cpuguy83 commented Feb 17, 2017

nshCore commented Feb 18, 2017

nshCore commented Feb 18, 2017

pbaderia01 commented Oct 8, 2019

thaJeztah commented Oct 8, 2019

docker deploy not deploying across swarm hosts #31121

docker deploy not deploying across swarm hosts #31121

Comments

nshCore commented Feb 17, 2017

willseward commented Feb 17, 2017

nshCore commented Feb 17, 2017 • edited

willseward commented Feb 17, 2017

willseward commented Feb 17, 2017

nshCore commented Feb 17, 2017 • edited

willseward commented Feb 17, 2017

willseward commented Feb 17, 2017

cpuguy83 commented Feb 17, 2017

nshCore commented Feb 18, 2017

nshCore commented Feb 18, 2017

pbaderia01 commented Oct 8, 2019

thaJeztah commented Oct 8, 2019

nshCore commented Feb 17, 2017 •

edited

nshCore commented Feb 17, 2017 •

edited