Nest Integration does not retry on some exceptions on startup

[oneofthemany]

The problem

After every power outage I have to manually reload nest Integration

What version of Home Assistant Core has the issue?

2023.3.5 Frontend 20230309.1 - latest

What was the last working version of Home Assistant Core?

No response

What type of installation are you running?

Home Assistant Container

Integration causing the issue

Google Nest

Link to integration documentation on our website

https://www.home-assistant.io/integrations/nest/

Diagnostics information

{
	"home_assistant":{
		"installation_type":"Unknown",
		"version":"2023.3.5",
		"dev":false,
		"hassio":false,
		"virtualenv":false,
		"python_version":"3.10.10",
		"docker":false,
		"arch":"aarch64",
		"timezone":"Europe/London",
		"os_name":"Linux",
		"os_version":"5.15.0-1025-raspi",
		"run_as_root":true
	},
	"custom_components":{
		"mass":{
			"version":"2023.2.1",
			"requirements":[
				"music-assistant==1.8.7.4"
			]
		},
		"gree":{
			"version":"2.3.1",
			"requirements":[
				"pycryptodome"
			]
		},
		"frigate":{
			"version":"3.0.1",
			"requirements":[]
		},
		"hacs":{
			"version":"1.31.0",
			"requirements":[
				"aiogithubapi>=22.10.1"
			]
		},
		"realtime_trains_api":{
			"version":"1.3.1",
			"requirements":[]
		}
	},
	"integration_manifest":{
		"domain":"nest",
		"name":"GoogleNest",
		"after_dependencies":[
			"media_source"
		],
		"codeowners":[
			"@allenporter"
		],
		"config_flow":true,
		"dependencies":[
			"ffmpeg",
			"http",
			"application_credentials"
		],
		"dhcp":[
			{
				"macaddress":"18B430*"
			},
			{
				"macaddress":"641666*"
			},
			{
				"macaddress":"D8EB46*"
			}
		],
		"documentation":"https://www.home-assistant.io/integrations/nest",
		"iot_class":"cloud_push",
		"loggers":[
			"google_nest_sdm",
			"nest"
		],
		"quality_scale":"platinum",
		"requirements":[
			"python-nest==4.2.0",
			"google-nest-sdm==2.2.4"
		],
		"is_built_in":true
	},
	"data":{
		"subscriber":{
			"start":2,
			"message_received_count":2,
			"message_received_sum":285178,
			"message_acked_count":2,
			"message_acked_sum":2
		},
		"devices":[
			{
				"data":{
					"name":"**REDACTED**",
					"type":"sdm.devices.types.THERMOSTAT",
					"assignee":"**REDACTED**",
					"traits":{
						"sdm.devices.traits.Info":{
							"customName":"**REDACTED**"
						},
						"sdm.devices.traits.Humidity":{
							"ambientHumidityPercent":45
						},
						"sdm.devices.traits.Connectivity":{
							"status":"ONLINE"
						},
						"sdm.devices.traits.Fan":{},
						"sdm.devices.traits.ThermostatMode":{
							"mode":"HEAT",
							"availableModes":[
								"HEAT",
								"OFF"
							]
						},
						"sdm.devices.traits.ThermostatEco":{
							"availableModes":[
								"OFF",
								"MANUAL_ECO"
							],
							"mode":"OFF",
							"heatCelsius":19.567902,
							"coolCelsius":24.444443
						},
						"sdm.devices.traits.ThermostatHvac":{
							"status":"HEATING"
						},
						"sdm.devices.traits.Settings":{
							"temperatureScale":"CELSIUS"
						},
						"sdm.devices.traits.ThermostatTemperatureSetpoint":{
							"heatCelsius":32
						},
						"sdm.devices.traits.Temperature":{
							"ambientTemperatureCelsius":22.75
						}
					},
					"parentRelations":[
						{
							"parent":"**REDACTED**",
							"displayName":"**REDACTED**"
						}
					]
				}
			},
			{
				"data":{
					"name":"**REDACTED**",
					"type":"sdm.devices.types.THERMOSTAT",
					"assignee":"**REDACTED**",
					"traits":{
						"sdm.devices.traits.Info":{
							"customName":"**REDACTED**"
						},
						"sdm.devices.traits.Humidity":{
							"ambientHumidityPercent":59
						},
						"sdm.devices.traits.Connectivity":{
							"status":"ONLINE"
						},
						"sdm.devices.traits.Fan":{},
						"sdm.devices.traits.ThermostatMode":{
							"mode":"OFF",
							"availableModes":[
								"HEAT",
								"OFF"
							]
						},
						"sdm.devices.traits.ThermostatEco":{
							"availableModes":[
								"OFF",
								"MANUAL_ECO"
							],
							"mode":"OFF",
							"heatCelsius":19.478592,
							"coolCelsius":24.444443
						},
						"sdm.devices.traits.ThermostatHvac":{
							"status":"OFF"
						},
						"sdm.devices.traits.Settings":{
							"temperatureScale":"CELSIUS"
						},
						"sdm.devices.traits.ThermostatTemperatureSetpoint":{},
						"sdm.devices.traits.Temperature":{
							"ambientTemperatureCelsius":19.229996
						}
					},
					"parentRelations":[
						{
							"parent":"**REDACTED**",
							"displayName":"**REDACTED**"
						}
					]
				},
				"command":{
					"sdm.devices.commands.ThermostatMode.SetMode_count":1,
					"sdm.devices.commands.ThermostatMode.SetMode_sum":1096
				},
				"event_media":{
					"event":1
				}
			}
		]
	}
}

Example YAML snippet

No response

Anything in the logs that might be useful for us?

No response

Additional information

No response

[home-assistant]

Hey there @allenporter, mind taking a look at this issue as it has been labeled with an integration (nest) you are listed as a code owner for? Thanks!

Code owner commands

Code owners of nest can trigger bot actions by commenting:

• @home-assistant close Closes the issue.
• @home-assistant rename Awesome new title Renames the issue.
• @home-assistant reopen Reopen the issue.
• @home-assistant unassign nest Removes the current integration label and assignees on the issue, add the integration domain after the command.

_{^{(message by CodeOwnersMention)}}

---

nest documentation
nest source
_{^{(message by IssueLinks)}}

[allenporter]

@oneofthemany can you elaborate on the issue that requires you to reload the integration? What are the symptoms?
Are there any interesting bits in the logs?

Thanks!

[oneofthemany]

@allenporter : Hi I have been narrowing it down and it would appear if there is a power cut, or I update my Ruckus AP which is my router, the Pi is either already up or booted before the Internet comes back on line and the Nest Integration retry value runs out and requires a manual reload:

urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0xffff68b80fa0>: Failed to establish a new connection: [Errno -3] Try again
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: /oauth2/v4/token (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0xffff68b80fa0>: Failed to establish a new connection: [Errno -3] Try again'))

Looking at it, it would appear to only retry once?

Is there any way we could introduce a heartbeat monitor that determines net up and looks at any cloud based integration and reloads them if down?

[oneofthemany]

@allenporter : Its cool I have created an API post automation to keep an eye on the devices associated with the integration and if they are unavailable it will reload the integration

shell_command:
  nest_reload: "/usr/bin/curl -X 'POST' -H 'authorization: Bearer <token>' http://localhost:8123/api/config/config_entries/entry/<my_reload_entry>/reload"

alias: Nest Reload
description: ""
trigger:
  - platform: state
    entity_id:
      - climate.nest_living_room
      - climate.nest_master_bedroom
    to: unavailable
    for:
      hours: 0
      minutes: 2
      seconds: 0
condition: []
action:
  - service: shell_command.nest_reload
    data: {}
mode: single

[allenporter]

The problem is that the exception is not caught, and therefore it does not get treated as a retryable error by home assistant. If we catch the right exception and raise properly, then this should retry properly on startup.

The report in the documentation component included the full exception which I am copying here:

Logger: homeassistant.config_entries
Source: components/nest/__init__.py:213
First occurred: 1:34:36 PM (1 occurrences)
Last logged: 1:34:36 PM

Error setting up entry Türksan for nest
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn
    conn = connection.create_connection(
  File "/usr/local/lib/python3.10/site-packages/urllib3/util/connection.py", line 72, in create_connection
    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
  File "/usr/local/lib/python3.10/socket.py", line 955, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -3] Try again

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 403, in _make_request
    self._validate_conn(conn)
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1053, in _validate_conn
    conn.connect()
  File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 363, in connect
    self.sock = conn = self._new_conn()
  File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 186, in _new_conn
    raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7f5c65a860>: Failed to establish a new connection: [Errno -3] Try again

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/requests/adapters.py", line 489, in send
    resp = conn.urlopen(
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen
    retries = retries.increment(
  File "/usr/local/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: /oauth2/v4/token (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f5c65a860>: Failed to establish a new connection: [Errno -3] Try again'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/google/auth/transport/requests.py", line 193, in __call__
    response = self.session.request(
  File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/requests/adapters.py", line 565, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: /oauth2/v4/token (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f5c65a860>: Failed to establish a new connection: [Errno -3] Try again'))

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/src/homeassistant/homeassistant/config_entries.py", line 387, in async_setup
    result = await component.async_setup_entry(hass, self)
  File "/usr/src/homeassistant/homeassistant/components/nest/__init__.py", line 213, in async_setup_entry
    await subscriber.start_async()
  File "/usr/local/lib/python3.10/site-packages/google_nest_sdm/google_nest_subscriber.py", line 415, in start_async
    await self._subscriber_factory.async_new_subscriber(
  File "/usr/local/lib/python3.10/site-packages/google_nest_sdm/google_nest_subscriber.py", line 260, in async_new_subscriber
    return await loop.run_in_executor(
  File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/usr/local/lib/python3.10/site-packages/google_nest_sdm/google_nest_subscriber.py", line 283, in _new_subscriber
    creds = self._refresh_creds(creds)
  File "/usr/local/lib/python3.10/site-packages/google_nest_sdm/google_nest_subscriber.py", line 271, in _refresh_creds
    creds.refresh(Request())
  File "/usr/local/lib/python3.10/site-packages/google/oauth2/credentials.py", line 335, in refresh
    ) = reauth.refresh_grant(
  File "/usr/local/lib/python3.10/site-packages/google/oauth2/reauth.py", line 324, in refresh_grant
    response_status_ok, response_data, retryable_error = _client._token_endpoint_request_no_throw(
  File "/usr/local/lib/python3.10/site-packages/google/oauth2/_client.py", line 213, in _token_endpoint_request_no_throw
    request_succeeded, response_data, retryable_error = _perform_request()
  File "/usr/local/lib/python3.10/site-packages/google/oauth2/_client.py", line 189, in _perform_request
    response = request(
  File "/usr/local/lib/python3.10/site-packages/google/auth/transport/requests.py", line 199, in __call__
    six.raise_from(new_exc, caught_exc)
  File "<string>", line 3, in raise_from
google.auth.exceptions.TransportError: HTTPSConnectionPool(host='www.googleapis.com', port=443): Max retries exceeded with url: /oauth2/v4/token (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f5c65a860>: Failed to establish a new connection: [Errno -3] Try again'))