all training run on the first worker

[zuowang]

@jiezhang could you take a look at this issue?

root@openmpi-master-0:/examples# ssh openmpi-worker-1.openmpi.kubeflow.svc.cluster.local
Warning: Permanently added '[openmpi-worker-1.openmpi.kubeflow.svc.cluster.local]:2022,[192.168.173.254]:2022' (ECDSA) to the list of known hosts.
Last login: Wed Apr 25 11:47:57 2018 from 192.168.201.150
root@openmpi-worker-1:~# ps -ef
UID        PID  PPID  C STIME TTY          TIME CMD
root         1     0  0 08:09 ?        00:00:00 sh /kubeflow/openmpi/assets/init.sh
root        14     1  0 08:09 ?        00:00:00 /usr/sbin/sshd -D -e -f /kubeflow/openmpi/assets/sshd_config
root        27    14  0 11:46 ?        00:00:00 sshd: root@notty
root        28    27  0 11:47 ?        00:00:00 bash -c     PATH=/usr/local/bin:$PATH ; export PATH ; LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH ; export LD_L
root        29    28  0 11:47 ?        00:00:00 /usr/local/bin/orted -mca ess env -mca ess_base_jobid 924975104 -mca ess_base_vpid 2 -mca ess_base_num_procs 5 -mca
root        44    14  0 11:57 ?        00:00:00 sshd: root@pts/0
root        45    44  0 11:57 pts/0    00:00:00 -bash
root        51    45  0 11:57 pts/0    00:00:00 ps -ef
root@openmpi-worker-1:~# exit
logout
Connection to openmpi-worker-1.openmpi.kubeflow.svc.cluster.local closed.
root@openmpi-master-0:/examples# ssh openmpi-worker-0.openmpi.kubeflow.svc.cluster.local
Warning: Permanently added '[openmpi-worker-0.openmpi.kubeflow.svc.cluster.local]:2022,[192.168.99.142]:2022' (ECDSA) to the list of known hosts.
Last login: Wed Apr 25 11:47:43 2018 from 192.168.201.150
root@openmpi-worker-0:~# ps -ef
UID        PID  PPID  C STIME TTY          TIME CMD
root         1     0  0 08:09 ?        00:00:00 sh /kubeflow/openmpi/assets/init.sh
root        14     1  0 08:09 ?        00:00:00 /usr/sbin/sshd -D -e -f /kubeflow/openmpi/assets/sshd_config
root       529    14  0 11:46 ?        00:00:00 sshd: root@notty
root       530   529  0 11:46 ?        00:00:00 bash -c     PATH=/usr/local/bin:$PATH ; export PATH ; LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH ; export LD_L
root       531   530  0 11:46 ?        00:00:00 /usr/local/bin/orted -mca ess env -mca ess_base_jobid 924975104 -mca ess_base_vpid 1 -mca ess_base_num_procs 5 -mca
root       535   531  0 11:46 ?        00:00:00 sh -c LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/keras_mn
root       536   531  0 11:46 ?        00:00:00 sh -c LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/keras_mn
root       537   535 99 11:46 ?        00:27:23 python /examples/keras_mnist_advanced.py
root       538   531  0 11:46 ?        00:00:00 sh -c LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/keras_mn
root       539   536 99 11:46 ?        00:27:17 python /examples/keras_mnist_advanced.py
root       540   531  0 11:46 ?        00:00:00 sh -c LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/keras_mn
root       541   538 99 11:46 ?        00:27:23 python /examples/keras_mnist_advanced.py
root       542   540 99 11:46 ?        00:27:18 python /examples/keras_mnist_advanced.py
root      1119    14  0 11:57 ?        00:00:00 sshd: root@pts/4
root      1120  1119  0 11:57 pts/4    00:00:00 -bash
root      1126  1120  0 11:57 pts/4    00:00:00 ps -ef
root@openmpi-worker-0:~#

[zuowang]

not a issue