When accessing a controller from an environment that does not have network access to all of the controller's endpoints the client will not handle OSError: Network unreachable gracefully.
Expected behavior:
Try each endpoint until success, only throw an exception if all endpoints have been attempted and failed.
Example
controllers:
serverstack:
uuid: 7a854a0d-0a1b-419b-8f09-217bdd61fa2c
api-endpoints: ['10.5.0.9:17070', '[fd00::f816:3eff:fe75:4d86]:17070', '252.0.9.1:17070']
When I access this controller via shuttle which only has access to the 10.5.0.0/16 network the client will fail on both the fan network address and the IPv6 address with the following.
NOTE: please ignore the Zaza framework portion of the traceback which just calls python libjuju.
Traceback (most recent call last):
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/bin/functest-run-suite", line 8, in
sys.exit(main())
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 199, in main
func_test_runner(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 140, in func_test_runner
run_env_deployment(env_deployment, keep_model=preserve_model,
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 53, in run_env_deployment
prepare.prepare(deployment.model_name)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/utilities/run_report.py", line 144, in wrapper
result = f(*args, **kwds)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/prepare.py", line 36, in prepare
zaza.controller.add_model(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 48, in _wrapper
return run(_run_it())
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 36, in run
return task.result()
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 47, in _run_it
return await f(*args, **kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/controller.py", line 34, in async_add_model
await controller.connect()
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/controller.py", line 114, in connect
await self._connector.connect_controller(controller_name, **kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connector.py", line 92, in connect_controller
await self.connect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connector.py", line 69, in connect
self._connection = await Connection.connect(**kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 303, in connect
await self._connect_with_redirect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 679, in _connect_with_redirect
login_result = await self._connect_with_login(endpoints)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 644, in _connect_with_login
await self._connect(endpoints)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 604, in _connect
result = await task
File "/usr/lib/python3.8/asyncio/tasks.py", line 608, in _wait_for_one
return f.result() # May raise f.exception().
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 593, in _try_endpoint
return await self._open(endpoint, cacert)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 339, in _open
return (await websockets.connect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/websockets/py35/client.py", line 12, in await_impl
transport, protocol = await self._creating_connection
File "/usr/lib/python3.8/asyncio/base_events.py", line 1025, in create_connection
raise exceptions[0]
File "/usr/lib/python3.8/asyncio/base_events.py", line 1010, in create_connection
sock = await self._connect_sock(
File "/usr/lib/python3.8/asyncio/base_events.py", line 924, in _connect_sock
await self.sock_connect(sock, address)
File "/usr/lib/python3.8/asyncio/selector_events.py", line 494, in sock_connect
return await fut
File "/usr/lib/python3.8/asyncio/selector_events.py", line 499, in _sock_connect
sock.connect(address)
OSError: [Errno 101] Network is unreachable
When accessing a controller from an environment that does not have network access to all of the controller's endpoints the client will not handle OSError: Network unreachable gracefully.
Expected behavior:
Try each endpoint until success, only throw an exception if all endpoints have been attempted and failed.
Example
controllers:
serverstack:
uuid: 7a854a0d-0a1b-419b-8f09-217bdd61fa2c
api-endpoints: ['10.5.0.9:17070', '[fd00::f816:3eff:fe75:4d86]:17070', '252.0.9.1:17070']
When I access this controller via shuttle which only has access to the 10.5.0.0/16 network the client will fail on both the fan network address and the IPv6 address with the following.
NOTE: please ignore the Zaza framework portion of the traceback which just calls python libjuju.
Traceback (most recent call last):
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/bin/functest-run-suite", line 8, in
sys.exit(main())
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 199, in main
func_test_runner(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 140, in func_test_runner
run_env_deployment(env_deployment, keep_model=preserve_model,
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/func_test_runner.py", line 53, in run_env_deployment
prepare.prepare(deployment.model_name)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/utilities/run_report.py", line 144, in wrapper
result = f(*args, **kwds)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/charm_lifecycle/prepare.py", line 36, in prepare
zaza.controller.add_model(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 48, in _wrapper
return run(_run_it())
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 36, in run
return task.result()
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/init.py", line 47, in _run_it
return await f(*args, **kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/zaza/controller.py", line 34, in async_add_model
await controller.connect()
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/controller.py", line 114, in connect
await self._connector.connect_controller(controller_name, **kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connector.py", line 92, in connect_controller
await self.connect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connector.py", line 69, in connect
self._connection = await Connection.connect(**kwargs)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 303, in connect
await self._connect_with_redirect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 679, in _connect_with_redirect
login_result = await self._connect_with_login(endpoints)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 644, in _connect_with_login
await self._connect(endpoints)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 604, in _connect
result = await task
File "/usr/lib/python3.8/asyncio/tasks.py", line 608, in _wait_for_one
return f.result() # May raise f.exception().
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 593, in _try_endpoint
return await self._open(endpoint, cacert)
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/juju/client/connection.py", line 339, in _open
return (await websockets.connect(
File "/home/thedac/dev/mysql-innodb-cluster/build/builds/mysql-innodb-cluster/.tox/func-smoke/lib/python3.8/site-packages/websockets/py35/client.py", line 12, in await_impl
transport, protocol = await self._creating_connection
File "/usr/lib/python3.8/asyncio/base_events.py", line 1025, in create_connection
raise exceptions[0]
File "/usr/lib/python3.8/asyncio/base_events.py", line 1010, in create_connection
sock = await self._connect_sock(
File "/usr/lib/python3.8/asyncio/base_events.py", line 924, in _connect_sock
await self.sock_connect(sock, address)
File "/usr/lib/python3.8/asyncio/selector_events.py", line 494, in sock_connect
return await fut
File "/usr/lib/python3.8/asyncio/selector_events.py", line 499, in _sock_connect
sock.connect(address)
OSError: [Errno 101] Network is unreachable