Make the frontend available sooner (Part 1 of 2) (#36263)

* Part 1 of 2 (no breaking changes in part 1).

When integrations configured via the UI block startup or fail to start,
the webserver can remain offline which make it is impossible
to recover without manually changing files in
.storage since the UI is not available.

This change is the foundation that part 2 will build on
and enable a listener to start the webserver when the frontend
is finished loading.

Frontend Changes (home-assistant/frontend#6068)

* Address review comments

* bump timeout to 1800s, adjust comment

* bump timeout to 4h

* remove timeout failsafe

* and the test

homeassistant/bootstrap.py

@@ -44,6 +44,13 @@
     "mqtt_eventstream",
     # To provide account link implementations
     "cloud",
+    # Ensure supervisor is available
+    "hassio",
+    # Get the frontend up and running as soon
+    # as possible so problem integrations can
+    # be removed
+    "frontend",
+    "config",
 }
 
 
@@ -399,6 +406,8 @@ async def _async_log_pending_setups() -> None:
     )
 
     if stage_1_domains:
+        _LOGGER.info("Setting up %s", stage_1_domains)
+
         await async_setup_multi_components(stage_1_domains)
 
     # Load all integrations

homeassistant/components/http/__init__.py

@@ -4,7 +4,7 @@
 import os
 import ssl
 from traceback import extract_stack
-from typing import Optional, cast
+from typing import Dict, Optional, cast
 
 from aiohttp import web
 from aiohttp.web_exceptions import HTTPMovedPermanently
@@ -15,7 +15,7 @@
     EVENT_HOMEASSISTANT_STOP,
     SERVER_PORT,
 )
-from homeassistant.core import HomeAssistant
+from homeassistant.core import Event, HomeAssistant
 from homeassistant.helpers import storage
 import homeassistant.helpers.config_validation as cv
 from homeassistant.loader import bind_hass
@@ -216,29 +216,25 @@ async def async_setup(hass, config):
         ssl_profile=ssl_profile,
     )
 
-    async def stop_server(event):
+    startup_listeners = []
+
+    async def stop_server(event: Event) -> None:
         """Stop the server."""
         await server.stop()
 
-    async def start_server(event):
+    async def start_server(event: Event) -> None:
         """Start the server."""
-        hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_server)
-        await server.start()
 
-        # If we are set up successful, we store the HTTP settings for safe mode.
-        store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
+        for listener in startup_listeners:
+            listener()
 
-        if CONF_TRUSTED_PROXIES in conf:
-            conf_to_save = dict(conf)
-            conf_to_save[CONF_TRUSTED_PROXIES] = [
-                str(ip.network_address) for ip in conf_to_save[CONF_TRUSTED_PROXIES]
-            ]
-        else:
-            conf_to_save = conf
+        hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, stop_server)
 
-        await store.async_save(conf_to_save)
+        await start_http_server_and_save_config(hass, dict(conf), server)
 
-    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_START, start_server)
+    startup_listeners.append(
+        hass.bus.async_listen(EVENT_HOMEASSISTANT_START, start_server)
+    )
 
     hass.http = server
 
@@ -418,3 +414,20 @@ async def stop(self):
         """Stop the aiohttp server."""
         await self.site.stop()
         await self.runner.cleanup()
+
+
+async def start_http_server_and_save_config(
+    hass: HomeAssistant, conf: Dict, server: HomeAssistantHTTP
+) -> None:
+    """Startup the http server and save the config."""
+    await server.start()  # type: ignore
+
+    # If we are set up successful, we store the HTTP settings for safe mode.
+    store = storage.Store(hass, STORAGE_VERSION, STORAGE_KEY)
+
+    if CONF_TRUSTED_PROXIES in conf:
+        conf[CONF_TRUSTED_PROXIES] = [
+            str(ip.network_address) for ip in conf[CONF_TRUSTED_PROXIES]
+        ]
+
+    await store.async_save(conf)

homeassistant/components/http/view.py

@@ -14,7 +14,7 @@
 import voluptuous as vol
 
 from homeassistant import exceptions
-from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK
+from homeassistant.const import CONTENT_TYPE_JSON, HTTP_OK, HTTP_SERVICE_UNAVAILABLE
 from homeassistant.core import Context, is_callback
 from homeassistant.helpers.json import JSONEncoder
 
@@ -107,8 +107,8 @@ def request_handler_factory(view: HomeAssistantView, handler: Callable) -> Calla
 
     async def handle(request: web.Request) -> web.StreamResponse:
         """Handle incoming request."""
-        if not request.app[KEY_HASS].is_running:
-            return web.Response(status=503)
+        if request.app[KEY_HASS].is_stopping:
+            return web.Response(status=HTTP_SERVICE_UNAVAILABLE)
 
         authenticated = request.get(KEY_AUTHENTICATED, False)
 

homeassistant/components/websocket_api/decorators.py

@@ -1,4 +1,5 @@
 """Decorators for the Websocket API."""
+import asyncio
 from functools import wraps
 import logging
 from typing import Awaitable, Callable
@@ -31,7 +32,9 @@ def async_response(
     @wraps(func)
     def schedule_handler(hass, connection, msg):
         """Schedule the handler."""
-        hass.async_create_task(_handle_async_response(func, hass, connection, msg))
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responders
+        asyncio.create_task(_handle_async_response(func, hass, connection, msg))
 
     return schedule_handler
 

homeassistant/components/websocket_api/http.py

@@ -165,7 +165,9 @@ def handle_hass_stop(event):
             EVENT_HOMEASSISTANT_STOP, handle_hass_stop
         )
 
-        self._writer_task = self.hass.async_create_task(self._writer())
+        # As the webserver is now started before the start
+        # event we do not want to block for websocket responses
+        self._writer_task = asyncio.create_task(self._writer())
 
         auth = AuthPhase(self._logger, self.hass, self._send_message, request)
         connection = None

homeassistant/core.py

@@ -209,6 +209,11 @@ def is_running(self) -> bool:
         """Return if Home Assistant is running."""
         return self.state in (CoreState.starting, CoreState.running)
 
+    @property
+    def is_stopping(self) -> bool:
+        """Return if Home Assistant is stopping."""
+        return self.state in (CoreState.stopping, CoreState.final_write)
+
     def start(self) -> int:
         """Start Home Assistant.
 
@@ -260,6 +265,7 @@ async def async_start(self) -> None:
 
         setattr(self.loop, "_thread_ident", threading.get_ident())
         self.bus.async_fire(EVENT_HOMEASSISTANT_START)
+        self.bus.async_fire(EVENT_CORE_CONFIG_UPDATE)
 
         try:
             # Only block for EVENT_HOMEASSISTANT_START listener
@@ -1391,6 +1397,7 @@ def as_dict(self) -> Dict:
             "version": __version__,
             "config_source": self.config_source,
             "safe_mode": self.safe_mode,
+            "state": self.hass.state.value,
             "external_url": self.external_url,
             "internal_url": self.internal_url,
         }

tests/components/hassio/test_discovery.py

@@ -60,6 +60,9 @@ async def test_hassio_discovery_startup(hass, aioclient_mock, hassio_client):
 
 async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client):
     """Test startup and discovery with hass discovery."""
+    aioclient_mock.post(
+        "http://127.0.0.1/supervisor/options", json={"result": "ok", "data": {}},
+    )
     aioclient_mock.get(
         "http://127.0.0.1/discovery",
         json={
@@ -101,7 +104,7 @@ async def test_hassio_discovery_startup_done(hass, aioclient_mock, hassio_client
         await async_setup_component(hass, "hassio", {})
         await hass.async_block_till_done()
 
-        assert aioclient_mock.call_count == 2
+        assert aioclient_mock.call_count == 3
         assert mock_mqtt.called
         mock_mqtt.assert_called_with(
             {

tests/components/http/test_data_validator.py

@@ -11,7 +11,7 @@
 async def get_client(aiohttp_client, validator):
     """Generate a client that hits a view decorated with validator."""
     app = web.Application()
-    app["hass"] = Mock(is_running=True)
+    app["hass"] = Mock(is_stopping=False)
 
     class TestView(HomeAssistantView):
         url = "/"

tests/components/http/test_view.py

@@ -19,7 +19,13 @@
 @pytest.fixture
 def mock_request():
     """Mock a request."""
-    return Mock(app={"hass": Mock(is_running=True)}, match_info={})
+    return Mock(app={"hass": Mock(is_stopping=False)}, match_info={})
+
+
+@pytest.fixture
+def mock_request_with_stopping():
+    """Mock a request."""
+    return Mock(app={"hass": Mock(is_stopping=True)}, match_info={})
 
 
 async def test_invalid_json(caplog):
@@ -55,3 +61,11 @@ async def test_handling_service_not_found(mock_request):
             Mock(requires_auth=False),
             AsyncMock(side_effect=ServiceNotFound("test", "test")),
         )(mock_request)
+
+
+async def test_not_running(mock_request_with_stopping):
+    """Test we get a 503 when not running."""
+    response = await request_handler_factory(
+        Mock(requires_auth=False), AsyncMock(side_effect=Unauthorized)
+    )(mock_request_with_stopping)
+    assert response.status == 503

tests/components/logbook/test_init.py

@@ -42,7 +42,8 @@ def setUp(self):
         """Set up things to be run when tests are started."""
         self.hass = get_test_home_assistant()
         init_recorder_component(self.hass)  # Force an in memory DB
-        assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG)
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
+            assert setup_component(self.hass, logbook.DOMAIN, self.EMPTY_CONFIG)
 
     def tearDown(self):
         """Stop everything that was started."""

tests/components/panel_iframe/test_init.py

@@ -4,6 +4,7 @@
 from homeassistant import setup
 from homeassistant.components import frontend
 
+from tests.async_mock import patch
 from tests.common import get_test_home_assistant
 
 
@@ -26,38 +27,42 @@ def test_wrong_config(self):
         ]
 
         for conf in to_try:
-            assert not setup.setup_component(
-                self.hass, "panel_iframe", {"panel_iframe": conf}
-            )
+            with patch(
+                "homeassistant.components.http.start_http_server_and_save_config"
+            ):
+                assert not setup.setup_component(
+                    self.hass, "panel_iframe", {"panel_iframe": conf}
+                )
 
     def test_correct_config(self):
         """Test correct config."""
-        assert setup.setup_component(
-            self.hass,
-            "panel_iframe",
-            {
-                "panel_iframe": {
-                    "router": {
-                        "icon": "mdi:network-wireless",
-                        "title": "Router",
-                        "url": "http://192.168.1.1",
-                        "require_admin": True,
-                    },
-                    "weather": {
-                        "icon": "mdi:weather",
-                        "title": "Weather",
-                        "url": "https://www.wunderground.com/us/ca/san-diego",
-                        "require_admin": True,
-                    },
-                    "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"},
-                    "ftp": {
-                        "icon": "mdi:weather",
-                        "title": "FTP",
-                        "url": "ftp://some/ftp",
-                    },
-                }
-            },
-        )
+        with patch("homeassistant.components.http.start_http_server_and_save_config"):
+            assert setup.setup_component(
+                self.hass,
+                "panel_iframe",
+                {
+                    "panel_iframe": {
+                        "router": {
+                            "icon": "mdi:network-wireless",
+                            "title": "Router",
+                            "url": "http://192.168.1.1",
+                            "require_admin": True,
+                        },
+                        "weather": {
+                            "icon": "mdi:weather",
+                            "title": "Weather",
+                            "url": "https://www.wunderground.com/us/ca/san-diego",
+                            "require_admin": True,
+                        },
+                        "api": {"icon": "mdi:weather", "title": "Api", "url": "/api"},
+                        "ftp": {
+                            "icon": "mdi:weather",
+                            "title": "FTP",
+                            "url": "ftp://some/ftp",
+                        },
+                    }
+                },
+            )
 
         panels = self.hass.data[frontend.DATA_PANELS]
 

tests/test_bootstrap.py

@@ -261,7 +261,9 @@ async def test_setup_hass(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"browser": {}, "frontend": {}},
-    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000):
+    ), patch.object(bootstrap, "LOG_SLOW_STARTUP_INTERVAL", 5000), patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,
@@ -338,7 +340,7 @@ async def test_setup_hass_invalid_yaml(
     """Test it works."""
     with patch(
         "homeassistant.config.async_hass_config_yaml", side_effect=HomeAssistantError
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -391,7 +393,9 @@ async def test_setup_hass_safe_mode(
     hass.config_entries._async_schedule_save()
     await flush_store(hass.config_entries._store)
 
-    with patch("homeassistant.components.browser.setup") as browser_setup:
+    with patch("homeassistant.components.browser.setup") as browser_setup, patch(
+        "homeassistant.components.http.start_http_server_and_save_config"
+    ):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -421,7 +425,7 @@ async def test_setup_hass_invalid_core_config(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"homeassistant": {"non-existing": 1}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=False,
@@ -451,7 +455,7 @@ async def test_setup_safe_mode_if_no_frontend(
     with patch(
         "homeassistant.config.async_hass_config_yaml",
         return_value={"map": {}, "person": {"invalid": True}},
-    ):
+    ), patch("homeassistant.components.http.start_http_server_and_save_config"):
         hass = await bootstrap.async_setup_hass(
             config_dir=get_test_config_dir(),
             verbose=verbose,

tests/test_core.py

@@ -35,7 +35,7 @@
 import homeassistant.util.dt as dt_util
 from homeassistant.util.unit_system import METRIC_SYSTEM
 
-from tests.async_mock import MagicMock, Mock, patch
+from tests.async_mock import MagicMock, Mock, PropertyMock, patch
 from tests.common import async_mock_service, get_test_home_assistant
 
 PST = pytz.timezone("America/Los_Angeles")
@@ -901,6 +901,8 @@ def test_path_with_dir_and_file(self):
     def test_as_dict(self):
         """Test as dict."""
         self.config.config_dir = "/test/ha-config"
+        self.config.hass = MagicMock()
+        type(self.config.hass.state).value = PropertyMock(return_value="RUNNING")
         expected = {
             "latitude": 0,
             "longitude": 0,
@@ -914,6 +916,7 @@ def test_as_dict(self):
             "version": __version__,
             "config_source": "default",
             "safe_mode": False,
+            "state": "RUNNING",
             "external_url": None,
             "internal_url": None,
         }