Added docstring for file explaining reason we use it. Used fancy mult…

…i-context manager to avoid double nesting.

electrumsv/cached_headers.py

@@ -1,3 +1,27 @@
+"""
+When bitcoinx loads a headers file, it processes all headers on startup and builds a picture of
+the chain state for those headers. What forks there are and at what heights. Unfortunately at the
+time of writing this, this can take up to 40 seconds to do and is an unacceptable and perplexing
+user experience. Either bitcoinx has to write the chain state to avoid recalculating it or we do.
+It makes little difference so we do it for now.
+
+We need to know which headers in the headers file the chain data applies to. As the headers file is
+only ever appended to we can associate the chain data with a partial hash covering the headers that
+have been factored into the chain data. We also know the index of the last header processed and can
+process the headers after that point which should only be a few blocks at most and incur no
+noticeable startup delay.
+
+Additional notes:
+
+* bitcoinx processes all headers above a hard-coded checkpoint. We used to set a checkpoint and
+  fetch headers on demand. We stopped because this was a poor user experience and it is much simpler
+  to bundle the full headers and have them ready to use immediately. Now our checkpoint is always
+  the Genesis block. The checkpoint mechanic could be removed from our perspective.
+
+* The headers file has a leading reserved space with values like the header count. We exclude that
+  from the hashing to ensure that the hash is deterministic even with additional appended headers.
+"""
+
 from __future__ import annotations
 import array
 from hashlib import sha256

electrumsv/tests/test_cached_headers.py

@@ -227,47 +227,47 @@ def test_incremental_update(make_regtest_headers_copy: Callable[[str|None], Head
 
     original_read_cached_chains_data = cached_headers.read_cached_chains_data
     original_read_unprocessed_headers = cached_headers.read_unprocessed_headers
-    with unittest.mock.patch("electrumsv.cached_headers.read_cached_chains_data") as mock_read:
-        with unittest.mock.patch(
-                "electrumsv.cached_headers.read_unprocessed_headers") as mock_read_file:
-            def mocked_read_cached_chains_data(f: io.BufferedReader, headers: ElectrumSVHeaders) \
-                    -> None:
-                original_read_cached_chains_data(f, headers)
-                # Verify that the restored state at this point matches the chain data we saved.
-                assert len(headers._chains) == 1
-                assert headers._chains[0].tip == original_headers1_tip
-            mock_read.side_effect = mocked_read_cached_chains_data
-
-            def mocked_read_unprocessed_headers(local_headers2: Headers, last_index: int) -> None:
-                # Verify that the extended headers are present, but the chain data knows it's
-                # state is only up to the pre-extension header index. The incremental read that
-                # we are intercepting will process the outstanding headers.
-                assert local_headers2._storage.header_count == 1 + 115 + 10
-                assert last_index == 115
-                original_read_unprocessed_headers(local_headers2, last_index)
-                assert local_headers2._storage.header_count == 1 + 115 + 10
-            mock_read_file.side_effect = mocked_read_unprocessed_headers
-
-            # This should be a full processed copy of the double chain headers store.
-            headers2 = make_regtest_headers_copy(headers1._storage.filename)
-
-            mock_read.assert_called_once()
-            mock_read_file.assert_called_once()
-
-            # These should be all the internal data structures in a headers object.
-            assert headers1._short_hashes == headers2._short_hashes
-            assert headers1._heights == headers2._heights
-            assert headers1._chain_indices == headers2._chain_indices
-
-            # Verify that the unextended chain data was used as a base and the extra headers
-            # connected to give the same result as the original extended headers object.
-            assert len(headers2._chains) == 2
-            for chain_index, chain in enumerate(headers1._chains):
-                chain_copy = headers2._chains[chain_index]
-                if chain.parent is not None or chain_copy.parent is not None:
-                    assert chain.parent.index == chain_copy.parent.index
-                assert chain.tip == chain_copy.tip
-                assert chain.work == chain_copy.work
-                assert chain.first_height == chain_copy.first_height
-                assert chain._header_indices == chain_copy._header_indices
-                assert chain.index == chain_copy.index
+    with unittest.mock.patch("electrumsv.cached_headers.read_cached_chains_data") as mock_read, \
+            unittest.mock.patch("electrumsv.cached_headers.read_unprocessed_headers") \
+                as mock_read_file:
+        def mocked_read_cached_chains_data(f: io.BufferedReader, headers: ElectrumSVHeaders) \
+                -> None:
+            original_read_cached_chains_data(f, headers)
+            # Verify that the restored state at this point matches the chain data we saved.
+            assert len(headers._chains) == 1
+            assert headers._chains[0].tip == original_headers1_tip
+        mock_read.side_effect = mocked_read_cached_chains_data
+
+        def mocked_read_unprocessed_headers(local_headers2: Headers, last_index: int) -> None:
+            # Verify that the extended headers are present, but the chain data knows it's
+            # state is only up to the pre-extension header index. The incremental read that
+            # we are intercepting will process the outstanding headers.
+            assert local_headers2._storage.header_count == 1 + 115 + 10
+            assert last_index == 115
+            original_read_unprocessed_headers(local_headers2, last_index)
+            assert local_headers2._storage.header_count == 1 + 115 + 10
+        mock_read_file.side_effect = mocked_read_unprocessed_headers
+
+        # This should be a full processed copy of the double chain headers store.
+        headers2 = make_regtest_headers_copy(headers1._storage.filename)
+
+        mock_read.assert_called_once()
+        mock_read_file.assert_called_once()
+
+        # These should be all the internal data structures in a headers object.
+        assert headers1._short_hashes == headers2._short_hashes
+        assert headers1._heights == headers2._heights
+        assert headers1._chain_indices == headers2._chain_indices
+
+        # Verify that the unextended chain data was used as a base and the extra headers
+        # connected to give the same result as the original extended headers object.
+        assert len(headers2._chains) == 2
+        for chain_index, chain in enumerate(headers1._chains):
+            chain_copy = headers2._chains[chain_index]
+            if chain.parent is not None or chain_copy.parent is not None:
+                assert chain.parent.index == chain_copy.parent.index
+            assert chain.tip == chain_copy.tip
+            assert chain.work == chain_copy.work
+            assert chain.first_height == chain_copy.first_height
+            assert chain._header_indices == chain_copy._header_indices
+            assert chain.index == chain_copy.index