Updated IPv8 while keeping Community kwargs (#7645)

doc/requirements.txt

@@ -9,5 +9,5 @@ sphinxcontrib-openapi==0.7.0
 configobj==5.0.6
 mistune==0.8.4  # sphinxcontrib-openapi==0.7.0 cannot work with the latest mistune version (2.0.0)
 MarkupSafe==2.0.1  # used by jinja2; 2.1.0 version removes soft_unicode and breaks jinja2-2.11.3
-pyipv8==2.11.0
+pyipv8==2.12.0
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

requirements-core.txt

@@ -18,7 +18,7 @@ sentry-sdk==1.31.0
 yappi==1.4.0
 yarl==1.9.2 # keep this dependency higher than 1.6.3. See: https://github.com/aio-libs/yarl/issues/517
 bitarray==2.7.6
-pyipv8==2.11.0
+pyipv8==2.12.0
 libtorrent==1.2.19
 file-read-backwards==3.0.0
 Brotli==1.0.9 # to prevent AttributeError on macOs: module 'brotli' has no attribute 'error' (in urllib3.response)

src/tribler/core/components/bandwidth_accounting/tests/test_bandwidth_endpoint.py

@@ -1,7 +1,6 @@
 import pytest
 from ipv8.keyvault.crypto import default_eccrypto
 from ipv8.peer import Peer
-from ipv8.test.mocking.ipv8 import MockIPv8
 
 from tribler.core.components.bandwidth_accounting.community.bandwidth_accounting_community import (
     BandwidthAccountingCommunity,
@@ -10,6 +9,7 @@
 from tribler.core.components.bandwidth_accounting.db.transaction import BandwidthTransactionData, EMPTY_SIGNATURE
 from tribler.core.components.bandwidth_accounting.restapi.bandwidth_endpoint import BandwidthEndpoint
 from tribler.core.components.bandwidth_accounting.settings import BandwidthAccountingSettings
+from tribler.core.components.ipv8.adapters_tests import TriblerMockIPv8
 from tribler.core.components.restapi.rest.base_api_test import do_request
 from tribler.core.utilities.unicode import hexlify
 
@@ -27,9 +27,9 @@ def bandwidth_database(tmp_path, peer):
 
 @pytest.fixture
 async def bw_community(bandwidth_database, peer):
-    ipv8 = MockIPv8(peer, BandwidthAccountingCommunity,
-                    database=bandwidth_database,
-                    settings=BandwidthAccountingSettings())
+    ipv8 = TriblerMockIPv8(peer, BandwidthAccountingCommunity,
+                           database=bandwidth_database,
+                           settings=BandwidthAccountingSettings())
     community = ipv8.get_overlay(BandwidthAccountingCommunity)
     yield community
     await ipv8.stop()

src/tribler/core/components/bandwidth_accounting/tests/test_community.py

@@ -1,7 +1,5 @@
 from ipv8.keyvault.crypto import default_eccrypto
 from ipv8.peer import Peer
-from ipv8.test.base import TestBase
-from ipv8.test.mocking.ipv8 import MockIPv8
 
 from tribler.core.components.bandwidth_accounting.community.bandwidth_accounting_community import (
     BandwidthAccountingCommunity,
@@ -10,22 +8,23 @@
 from tribler.core.components.bandwidth_accounting.db.database import BandwidthDatabase
 from tribler.core.components.bandwidth_accounting.db.transaction import BandwidthTransactionData, EMPTY_SIGNATURE
 from tribler.core.components.bandwidth_accounting.settings import BandwidthAccountingSettings
+from tribler.core.components.ipv8.adapters_tests import TriblerMockIPv8, TriblerTestBase
 from tribler.core.utilities.utilities import MEMORY_DB
 
 ID1, ID2, ID3 = range(3)
 
 
-class TestBandwidthAccountingCommunity(TestBase):
+class TestBandwidthAccountingCommunity(TriblerTestBase):
 
     def setUp(self):
         super().setUp()
         self.initialize(BandwidthAccountingCommunity, 2)
 
-    def create_node(self):
+    def create_node(self, *args, **kwargs):
         peer = Peer(default_eccrypto.generate_key("curve25519"), address=("1.2.3.4", 5))
         db = BandwidthDatabase(db_path=MEMORY_DB, my_pub_key=peer.public_key.key_to_bin())
-        ipv8 = MockIPv8(peer, BandwidthAccountingCommunity, database=db,
-                        settings=BandwidthAccountingSettings())
+        ipv8 = TriblerMockIPv8(peer, BandwidthAccountingCommunity, database=db,
+                               settings=BandwidthAccountingSettings())
         return ipv8
 
     def database(self, i):

src/tribler/core/components/database/db/tests/test_tribler_database.py

@@ -1,13 +1,13 @@
 import pytest
-from ipv8.test.base import TestBase
 from pony.orm import db_session
 
 from tribler.core.components.database.db.tribler_database import TriblerDatabase
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 
 
 # pylint: disable=protected-access
 
-class TestTriblerDatabase(TestBase):
+class TestTriblerDatabase(TriblerTestBase):
     def setUp(self):
         super().setUp()
         self.db = TriblerDatabase()

src/tribler/core/components/gigachannel/community/tests/test_gigachannel_community.py

@@ -7,7 +7,6 @@
 import pytest
 from ipv8.keyvault.crypto import default_eccrypto
 from ipv8.peer import Peer
-from ipv8.test.base import TestBase
 from pony.orm import db_session
 
 from tribler.core.components.gigachannel.community.gigachannel_community import (
@@ -17,6 +16,7 @@
     happy_eyeballs_delay
 )
 from tribler.core.components.gigachannel.community.settings import ChantSettings
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 from tribler.core.components.metadata_store.db.store import MetadataStore
 from tribler.core.components.metadata_store.remote_query_community.remote_query_community import EvaSelectRequest, \
     SelectRequest, RemoteSelectPayload, RemoteSelectPayloadEva, SelectResponsePayload
@@ -52,7 +52,7 @@ def __len__(self):
         return len(fields(self))
 
 
-class TestGigaChannelUnits(TestBase):
+class TestGigaChannelUnits(TriblerTestBase):
     overlay: Callable[[int], GigaChannelCommunity]
 
     def setUp(self):

src/tribler/core/components/gigachannel/community/tests/test_sync_strategy.py

@@ -1,9 +1,9 @@
 from ipv8.keyvault.crypto import default_eccrypto
 from ipv8.peer import Peer
 from ipv8.peerdiscovery.network import Network
-from ipv8.test.base import TestBase
 
 from tribler.core.components.gigachannel.community.sync_strategy import RemovePeers
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 
 
 class MockCommunity:
@@ -23,7 +23,7 @@ def get_peers(self):
         return self.get_peers_return
 
 
-class TestRemovePeers(TestBase):
+class TestRemovePeers(TriblerTestBase):
     def setUp(self):
         self.community = MockCommunity()
         self.strategy = RemovePeers(self.community)

src/tribler/core/components/ipv8/adapters_tests.py

@@ -0,0 +1,35 @@
+import inspect
+
+from ipv8.test.base import TestBase
+from ipv8.test.mocking.ipv8 import MockIPv8
+
+
+def backport_to_settings_class(overlay_class, kwargs):
+    signature = inspect.signature(overlay_class.__init__)
+    defaults = {k: v.default for k, v in signature.parameters.items()
+                if v.default is not inspect.Parameter.empty}
+    defaults.update(kwargs)
+    return overlay_class.settings_class(**defaults)
+
+
+class TriblerMockIPv8(MockIPv8):
+
+    def __init__(self, crypto_curve_or_peer, overlay_class, create_dht = False, enable_statistics = False,
+                 **kwargs):
+        community_settings = backport_to_settings_class(overlay_class, kwargs)
+
+        class ProxyOverlay(overlay_class):
+
+            def __init__(self, settings):
+                super().__init__(**settings.__dict__)
+
+        super().__init__(crypto_curve_or_peer, ProxyOverlay, community_settings, create_dht, enable_statistics)
+
+
+class TriblerTestBase(TestBase):
+
+    def create_node(self, *args, **kwargs):
+        create_dht = args[1] if len(args) > 1 else False
+        enable_statistics = args[2] if len(args) > 2 else False
+        return TriblerMockIPv8("low", self.overlay_class, create_dht=create_dht,
+                               enable_statistics=enable_statistics, **kwargs)

src/tribler/core/components/ipv8/eva/tests/test_protocol.py

@@ -11,10 +11,9 @@
 from unittest.mock import Mock, patch
 
 import pytest
-from ipv8.community import Community
 from ipv8.messaging.lazy_payload import VariablePayload
-from ipv8.test.base import TestBase
 from ipv8.types import Peer
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 
 from tribler.core.components.ipv8.eva.exceptions import RequestRejected, SizeException, TimeoutException, \
     TransferCancelledException, TransferException, \
@@ -23,6 +22,7 @@
 from tribler.core.components.ipv8.eva.protocol import EVAProtocol
 from tribler.core.components.ipv8.eva.result import TransferResult
 from tribler.core.components.ipv8.eva.settings import EVASettings, Retransmission, Termination
+from tribler.core.components.ipv8.tribler_community import TriblerCommunity
 
 # pylint: disable=redefined-outer-name, protected-access, attribute-defined-outside-init
 
@@ -45,7 +45,7 @@ async def drain_loop(loop: AbstractEventLoop):
         await asyncio.sleep(0)
 
 
-class MockCommunity(Community):  # pylint: disable=too-many-ancestors
+class MockCommunity(TriblerCommunity):  # pylint: disable=too-many-ancestors
     community_id = os.urandom(20)
 
     def __init__(self, *args, **kwargs):
@@ -83,7 +83,7 @@ async def on_error(self, _, exception):
         self.error_has_been_raised.set()
 
 
-class TestEVA(TestBase):
+class TestEVA(TriblerTestBase):
     def setUp(self):
         super().setUp()
         self.initialize(MockCommunity, 3)

src/tribler/core/components/ipv8/ipv8_component.py

@@ -17,6 +17,7 @@
 from tribler.core.components.component import Component
 from tribler.core.components.ipv8.rendezvous.db.database import RendezvousDatabase
 from tribler.core.components.ipv8.rendezvous.rendezvous_hook import RendezvousHook
+from tribler.core.components.ipv8.tribler_community import args_kwargs_to_community_settings
 from tribler.core.components.key.key_component import KeyComponent
 from tribler.core.utilities.simpledefs import STATEDIR_DB_DIR
 
@@ -122,15 +123,19 @@ def make_bootstrapper(self) -> DispersyBootstrapper:
 
     def _init_peer_discovery_community(self):
         ipv8 = self.ipv8
-        community = DiscoveryCommunity(self.peer, ipv8.endpoint, ipv8.network, max_peers=100)
+        community = DiscoveryCommunity(args_kwargs_to_community_settings(DiscoveryCommunity.settings_class,
+                                                                         [self.peer, ipv8.endpoint, ipv8.network],
+                                                                         {"max_peers": 100}))
         self.initialise_community_by_default(community)
         ipv8.add_strategy(community, RandomChurn(community), INFINITE)
         ipv8.add_strategy(community, PeriodicSimilarity(community), INFINITE)
         self._peer_discovery_community = community
 
     def _init_dht_discovery_community(self):
         ipv8 = self.ipv8
-        community = DHTDiscoveryCommunity(self.peer, ipv8.endpoint, ipv8.network, max_peers=60)
+        community = DHTDiscoveryCommunity(args_kwargs_to_community_settings(DHTDiscoveryCommunity.settings_class,
+                                                                            [self.peer, ipv8.endpoint, ipv8.network],
+                                                                            {"max_peers": 60}))
         self.initialise_community_by_default(community)
         ipv8.add_strategy(community, PingChurn(community), INFINITE)
         self.dht_discovery_community = community
@@ -148,4 +153,4 @@ async def shutdown(self):
         if self.rendevous_hook is not None:
             self.rendevous_hook.shutdown(self.ipv8.network)
         await self._task_manager.shutdown_task_manager()
-        await self.ipv8.stop(stop_loop=False)
+        await self.ipv8.stop()

src/tribler/core/components/ipv8/tribler_community.py

@@ -1,13 +1,22 @@
-from ipv8.community import Community
+from ipv8.community import Community, DEFAULT_MAX_PEERS
 
 from tribler.core.config.tribler_config_section import TriblerConfigSection
 
+def args_kwargs_to_community_settings(settings_class, args, kwargs):
+    return settings_class(my_peer=args[0] if len(args) > 0 else kwargs.pop("my_peer"),
+                          endpoint=args[1] if len(args) > 1 else kwargs.pop("endpoint"),
+                          network=args[2] if len(args) > 2 else kwargs.pop("network"),
+                          max_peers=args[4] if len(args) > 3 else kwargs.pop("max_peers", DEFAULT_MAX_PEERS),
+                          anonymize=args[5] if len(args) > 4 else kwargs.pop("anonymize", True),
+                          **kwargs)
+
 
 class TriblerCommunity(Community):
     """Base class for Tribler communities.
     """
 
     def __init__(self, *args, settings: TriblerConfigSection = None, **kwargs):
-        super().__init__(*args, **kwargs)
+        community_settings = args_kwargs_to_community_settings(self.settings_class, args, kwargs)
+        super().__init__(community_settings)
         self.settings = settings
         self.logger.info(f'Init. Settings: {settings}.')

src/tribler/core/components/knowledge/community/tests/test_knowledge_community.py

@@ -2,19 +2,18 @@
 from unittest.mock import MagicMock, Mock
 
 from ipv8.keyvault.private.libnaclkey import LibNaCLSK
-from ipv8.test.base import TestBase
-from ipv8.test.mocking.ipv8 import MockIPv8
 from pony.orm import db_session
 
 from tribler.core.components.database.db.layers.knowledge_data_access_layer import Operation, ResourceType
+from tribler.core.components.ipv8.adapters_tests import TriblerMockIPv8, TriblerTestBase
 from tribler.core.components.knowledge.community.knowledge_community import KnowledgeCommunity
 from tribler.core.components.knowledge.community.knowledge_payload import StatementOperation
 from tribler.core.components.database.db.tribler_database import TriblerDatabase
 
 REQUEST_INTERVAL_FOR_RANDOM_OPERATIONS = 0.1  # in seconds
 
 
-class TestKnowledgeCommunity(TestBase):
+class TestKnowledgeCommunity(TriblerTestBase):
     def setUp(self):
         super().setUp()
         self.initialize(KnowledgeCommunity, 2)
@@ -23,8 +22,8 @@ async def tearDown(self):
         await super().tearDown()
 
     def create_node(self, *args, **kwargs):
-        return MockIPv8("curve25519", KnowledgeCommunity, db=TriblerDatabase(), key=LibNaCLSK(),
-                        request_interval=REQUEST_INTERVAL_FOR_RANDOM_OPERATIONS)
+        return TriblerMockIPv8("curve25519", KnowledgeCommunity, db=TriblerDatabase(), key=LibNaCLSK(),
+                               request_interval=REQUEST_INTERVAL_FOR_RANDOM_OPERATIONS)
 
     def create_operation(self, subject='1' * 20, obj=''):
         community = self.overlay(0)

src/tribler/core/components/metadata_store/remote_query_community/tests/test_remote_query_community.py

@@ -10,10 +10,10 @@
 
 import pytest
 from ipv8.keyvault.crypto import default_eccrypto
-from ipv8.test.base import TestBase
 from pony.orm import db_session
 from pony.orm.dbapiprovider import OperationalError
 
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 from tribler.core.components.metadata_store.db.orm_bindings.channel_node import NEW
 from tribler.core.components.metadata_store.db.serialization import CHANNEL_THUMBNAIL, CHANNEL_TORRENT, REGULAR_TORRENT
 from tribler.core.components.metadata_store.db.store import MetadataStore
@@ -52,7 +52,7 @@ class BasicRemoteQueryCommunity(RemoteQueryCommunity):
     community_id = unhexlify('eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee')
 
 
-class TestRemoteQueryCommunity(TestBase):
+class TestRemoteQueryCommunity(TriblerTestBase):
     """
     Unit tests for the base RemoteQueryCommunity which do not need a real Session.
     """

src/tribler/core/components/metadata_store/remote_query_community/tests/test_remote_search_by_tags.py

@@ -2,14 +2,14 @@
 from unittest.mock import AsyncMock, Mock, PropertyMock, patch
 
 from ipv8.keyvault.crypto import default_eccrypto
-from ipv8.test.base import TestBase
 from pony.orm import db_session
 
 from tribler.core.components.database.db.layers.knowledge_data_access_layer import KnowledgeDataAccessLayer, \
     ResourceType, SHOW_THRESHOLD
 from tribler.core.components.database.db.layers.tests.test_knowledge_data_access_layer_base import Resource, \
     TestKnowledgeAccessLayerBase
 from tribler.core.components.database.db.tribler_database import TriblerDatabase
+from tribler.core.components.ipv8.adapters_tests import TriblerTestBase
 from tribler.core.components.metadata_store.db.orm_bindings.channel_node import NEW
 from tribler.core.components.metadata_store.db.store import MetadataStore
 from tribler.core.components.metadata_store.remote_query_community.remote_query_community import RemoteQueryCommunity
@@ -21,7 +21,7 @@
 from tribler.core.utilities.unicode import hexlify
 
 
-class TestRemoteSearchByTags(TestBase):
+class TestRemoteSearchByTags(TriblerTestBase):
     """ In this test set we will use only one node's instance as it is sufficient
     for testing remote search by tags
     """

src/tribler/core/components/popularity/community/tests/test_popularity_community.py

@@ -4,10 +4,9 @@
 from unittest.mock import Mock
 
 from ipv8.keyvault.crypto import default_eccrypto
-from ipv8.test.base import TestBase
-from ipv8.test.mocking.ipv8 import MockIPv8
 from pony.orm import db_session
 
+from tribler.core.components.ipv8.adapters_tests import TriblerMockIPv8, TriblerTestBase
 from tribler.core.components.metadata_store.db.store import MetadataStore
 from tribler.core.components.metadata_store.remote_query_community.settings import RemoteQueryCommunitySettings
 from tribler.core.components.popularity.community.popularity_community import PopularityCommunity
@@ -39,7 +38,7 @@ def _generate_checked_torrents(count: int, status: str = None) -> List[HealthInf
     return [_generate_single_checked_torrent(status) for _ in range(count)]
 
 
-class TestPopularityCommunity(TestBase):
+class TestPopularityCommunity(TriblerTestBase):
     NUM_NODES = 2
 
     def setUp(self):
@@ -64,10 +63,10 @@ def create_node(self, *args, **kwargs):
         self.count += 1
 
         rqc_settings = RemoteQueryCommunitySettings()
-        return MockIPv8("curve25519", PopularityCommunity, metadata_store=mds,
-                        torrent_checker=torrent_checker,
-                        rqc_settings=rqc_settings
-                        )
+        return TriblerMockIPv8("curve25519", PopularityCommunity, metadata_store=mds,
+                               torrent_checker=torrent_checker,
+                               rqc_settings=rqc_settings
+                               )
 
     @db_session
     def fill_database(self, metadata_store, last_check_now=False):