Add dpp-proxy to mainnet api_servers.json & updates to reflect that t…

…he mainnet reference server returns not only the longest chain tip but stale chain tips as well

electrumsv/data/api_servers.json

@@ -30,5 +30,12 @@
         "url": "https://mapi.gorillapool.io/mapi/",
         "api_key_required": false,
         "static_data_date": "2022-06-30T23:45:00+0000"
+    },
+    {
+        "id": 30001,
+        "type": "DPP_PROXY",
+        "url": "https://x.bitcoinsv.io/dpp-proxy/",
+        "api_key_required": false,
+        "static_data_date": "2021-07-6T04:45:05+0000"
     }
 ]

electrumsv/network.py

@@ -40,7 +40,8 @@
 from .network_support.api_server import APIServerDefinition
 from .network_support.types import TipResponse
 from .network_support.headers import get_batched_headers_by_height_async, get_chain_tips_async, \
-    HeaderServerState, ServerConnectivityMetadata, subscribe_to_headers_async
+    HeaderServerState, ServerConnectivityMetadata, subscribe_to_headers_async, \
+    filter_tips_for_longest_chain
 from .networks import Net
 from .types import NetworkStatusDict, ServerAccountKey
 from .util import TriggeredCallbacks
@@ -140,7 +141,8 @@ async def _synchronise_headers_for_server_tip(self, server_state: HeaderServerSt
         Raises `ServiceUnavailableError` via _request_and_connect_headers_at_heights_async
         """
         assert app_state.headers is not None
-        tip_header = await get_chain_tips_async(server_state, self.aiohttp_session)
+        tip_headers = await get_chain_tips_async(server_state, self.aiohttp_session)
+        tip_header = filter_tips_for_longest_chain(tip_headers)
         server_state.tip_header = tip_header
         while True:
             try:
@@ -439,14 +441,14 @@ async def _request_and_connect_headers_at_heights_async(self, server_state: Head
             header_array = await get_batched_headers_by_height_async(server_state,
                 self.aiohttp_session, min_height, count)
             stream = BytesIO(header_array)
-
             count_of_raw_headers = len(header_array) // 80
             for i in range(count_of_raw_headers):
                 raw_header = stream.read(80)
                 # This will acquire a lock for every call, but unless we see that in profiling
                 # we are okay with this.
                 app_state.connect_header(raw_header)
 
+            logger.debug("Connected %s headers", count_of_raw_headers)
             app_state.headers_update_event.set()
             app_state.headers_update_event.clear()
 

electrumsv/network_support/headers.py

@@ -2,6 +2,7 @@
 import concurrent.futures
 import dataclasses
 import http
+from io import BytesIO
 from typing import AsyncIterable, cast, Optional
 
 import aiohttp
@@ -67,7 +68,7 @@ async def get_batched_headers_by_height_async(server_state: HeaderServerState,
 
 
 async def get_chain_tips_async(server_state: HeaderServerState, session: aiohttp.ClientSession) \
-        -> Header:
+        -> list[Header]:
     url = f"{server_state.server_key.url}api/v1/headers/tips"
     headers = {
         "Accept": "application/octet-stream"
@@ -87,15 +88,30 @@ async def get_chain_tips_async(server_state: HeaderServerState, session: aiohttp
                 logger.error(error_message)
                 raise HeaderResponseError(error_message)
 
-            data: bytes = await response.content.read()
-            raw_header = data[0:80]
-            height = bitcoinx.le_bytes_to_int(data[80:84])
-            # TODO(technical-debt) Look into why this is not `Net.COIN`?
-            return Net._net.COIN.deserialized_header(raw_header, height)
+            headers_array: bytes = await response.content.read()
+            assert len(headers_array) % 84 == 0  # 80 byte header + 4 byte int32 height
+            count_headers = len(headers_array) // 84
+            stream = BytesIO(headers_array)
+
+            block_headers: list[Header] = []
+            for i in range(count_headers):
+                raw_header = stream.read(80)
+                height = bitcoinx.le_bytes_to_int(stream.read(4))
+                # TODO(technical-debt) Look into why this is not `Net.COIN`?
+                block_headers.append(Net._net.COIN.deserialized_header(raw_header, height))
+            return block_headers
     except aiohttp.ClientConnectionError:
         raise ServiceUnavailableError(f"Cannot connect to header API at {url}")
 
 
+def filter_tips_for_longest_chain(tips: list[Header]) -> Header:
+    longest_chain_tip = tips[0]
+    for tip in tips:
+        if tip.height > longest_chain_tip.height:
+            longest_chain_tip = tip
+    return longest_chain_tip
+
+
 async def subscribe_to_headers_async(server_state: HeaderServerState,
         session: aiohttp.ClientSession) -> AsyncIterable[TipResponse]:
     url = f"{server_state.server_key.url}api/v1/headers/tips/websocket"

electrumsv/tests/test_network_general_api.py

@@ -9,7 +9,7 @@
 from aiohttp import ClientSession
 from aiohttp.web_ws import WebSocketResponse
 import bitcoinx
-from bitcoinx import hash_to_hex_str
+from bitcoinx import hash_to_hex_str, double_sha256
 
 from electrumsv.app_state import AppStateProxy
 from electrumsv.constants import NetworkServerType, ServerCapability
@@ -18,7 +18,8 @@
     ServerWebsocketNotification, TokenPermissions
 from electrumsv.network_support.general_api import unpack_binary_restoration_entry
 from electrumsv.network_support.headers import get_batched_headers_by_height_async, \
-    get_chain_tips_async, get_single_header_async, HeaderServerState, subscribe_to_headers_async
+    get_chain_tips_async, get_single_header_async, HeaderServerState, subscribe_to_headers_async, \
+    filter_tips_for_longest_chain
 from electrumsv.network_support.peer_channel import create_peer_channel_async, \
     create_peer_channel_api_token_async, create_peer_channel_message_json_async, \
     delete_peer_channel_async, get_peer_channel_max_sequence_number_async, \
@@ -175,7 +176,7 @@ async def mock_get_chain_tips(request: web.Request):
     try:
         accept_type = request.headers.get('Accept')
         assert accept_type == 'application/octet-stream'
-        headers_array = GENESIS_HEADER + bitcoinx.int_to_le_bytes(0)
+        headers_array = GENESIS_HEADER + bitcoinx.pack_le_int32(0)
         return web.Response(body=headers_array, content_type='application/octet-stream')
     except AssertionError as e:
         raise web.HTTPBadRequest(reason=str(e))
@@ -389,12 +390,17 @@ async def test_get_batched_headers_by_height(mock_app_state: AppStateProxy, aioh
 @unittest.mock.patch('electrumsv.network_support.general_api.app_state')
 async def test_get_chain_tips(mock_app_state: AppStateProxy, aiohttp_client):
     mock_app_state.credentials.get_indefinite_credential = lambda *args: REGTEST_BEARER_TOKEN
-    expected_response = GENESIS_HEADER + bitcoinx.int_to_le_bytes(0)
+    bitcoinx.unpack_header(GENESIS_HEADER)
+
+    expected_response = bitcoinx.Header(*bitcoinx.unpack_header(GENESIS_HEADER),
+        hash=double_sha256(GENESIS_HEADER), raw=GENESIS_HEADER, height=0)
     test_session = await aiohttp_client(create_app())
     state = HeaderServerState(ServerAccountKey(BASE_URL, NetworkServerType.GENERAL, None), None)
-    header = await get_chain_tips_async(state, test_session)
-    assert header.height == 0
-    assert header.raw == GENESIS_HEADER
+    tip_headers = await get_chain_tips_async(state, test_session)
+    tip_header = filter_tips_for_longest_chain(tip_headers)
+    assert tip_header == expected_response
+    assert tip_header.height == 0
+    assert tip_header.raw == GENESIS_HEADER
 
 
 @unittest.mock.patch('electrumsv.network_support.general_api.app_state')