[PLAT-255] Remove IPFS (#3526)

* [PLAT-255] Remove IPFS * Remove unused import * Fix typo * Fix lint * Fix lint * Fix tests * Fix mad-dog
AudiusProject · Sep 28, 2022 · 0e1d8bc · 0e1d8bc
1 parent 376f3e2
commit 0e1d8bc
Show file tree

Hide file tree

Showing 49 changed files with 154 additions and 246 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -707,7 +707,6 @@ jobs:
           name: discovery provider tests
           command: |
             cd discovery-provider
-            export audius_ipfs_port=5001
             export audius_redis_url=redis://localhost:6379/0
             export audius_elasticsearch_url=http://localhost:9200
             export audius_delegate_owner_wallet=0x1D9c77BcfBfa66D37390BF2335f0140979a6122B

diff --git a/discovery-provider/compose/.env b/discovery-provider/compose/.env
@@ -5,13 +5,10 @@ audius_web3_eth_provider_url=http://audius_ganache_cli_eth_contracts:8545
 audius_redis_url=redis://redis-server:6379/00
 audius_db_url=postgresql+psycopg2://postgres:postgres@discovery-provider-db:5432/audius_discovery
 audius_db_url_read_replica=postgresql+psycopg2://postgres:postgres@discovery-provider-db:5432/audius_discovery
-audius_ipfs_host=ipfs-node
-audius_ipfs_port=5001
-WAIT_HOSTS=discovery-provider-db:5432,redis-server:6379, ipfs-node:5001
+WAIT_HOSTS=discovery-provider-db:5432,redis-server:6379
 audius_delegate_owner_wallet=0x1D9c77BcfBfa66D37390BF2335f0140979a6122B
 audius_delegate_private_key=0x3873ed01bfb13621f9301487cc61326580614a5b99f3c33cf39c6f9da3a19cad
 audius_discprov_identity_service_url=http://audius-identity-service_identity-service_1:7000
-audius_discprov_user_metadata_service_url=http://cn1_creator-node_1:4000
 audius_discprov_blacklist_block_indexing_interval=5
 audius_eth_contracts_registry=0x26b9c682a87879Ef035c92E4F978c303eC69a84B
 audius_solana_track_listen_count_address=4ZN1TTLp76gGCZ7tzXLfV4cahuHaP6oHFuiVWQS3knTp

diff --git a/discovery-provider/compose/.test.env b/discovery-provider/compose/.test.env
@@ -1,5 +1,4 @@
 COMPOSE_PROJECT_NAME=dn1
 audius_db_port=5432
-audius_ipfs_external_port=5001
 audius_redis_port=5379
 audius_es_port=9200
diff --git a/discovery-provider/compose/docker-compose.backend.yml b/discovery-provider/compose/docker-compose.backend.yml
@@ -18,9 +18,8 @@ services:
       - audius_elasticsearch_search_enabled=true
       - audius_delegate_owner_wallet=${audius_delegate_owner_wallet}
       - audius_delegate_private_key=${audius_delegate_private_key}
-      - audius_ipfs_host=${COMPOSE_PROJECT_NAME}-ipfs-node
       - audius_discprov_dev_mode=true
-      - WAIT_HOSTS=${COMPOSE_PROJECT_NAME}_discovery-provider-db_1:5432,${COMPOSE_PROJECT_NAME}_redis-server_1:6379, ${COMPOSE_PROJECT_NAME}-ipfs-node:5001
+      - WAIT_HOSTS=${COMPOSE_PROJECT_NAME}_discovery-provider-db_1:5432,${COMPOSE_PROJECT_NAME}_redis-server_1:6379
     depends_on:
       discovery-provider-db:
         condition: service_healthy

diff --git a/discovery-provider/compose/docker-compose.ipfs.yml b/discovery-provider/compose/docker-compose.ipfs.yml
diff --git a/discovery-provider/compose/docker-compose.web-server.yml b/discovery-provider/compose/docker-compose.web-server.yml
@@ -1,7 +1,7 @@
 # Compose file to run the discovery flask server on its own
 #
 # The server is pointed at whatever database is set in .env
-# and runs no workers, but does need redis and IPFS to be available
+# and runs no workers
 version: '3'
 services:
   web-server:
@@ -15,10 +15,9 @@ services:
       - audius_db_run_migrations=false
       - FLASK_RUN_PORT=${audius_server_port}
       - audius_redis_url=redis://${COMPOSE_PROJECT_NAME}_redis-server_1:6379/00
-      - audius_ipfs_host=${COMPOSE_PROJECT_NAME}-ipfs-node
       - audius_discprov_dev_mode=true
       - audius_no_workers=true
-      - WAIT_HOSTS=${COMPOSE_PROJECT_NAME}_redis-server_1:6379,${COMPOSE_PROJECT_NAME}-ipfs-node:5001
+      - WAIT_HOSTS=${COMPOSE_PROJECT_NAME}_redis-server_1:6379
     depends_on:
       - redis-server
     volumes:

diff --git a/discovery-provider/compose/env/shellEnv1.sh b/discovery-provider/compose/env/shellEnv1.sh
@@ -1,6 +1,5 @@
 export COMPOSE_PROJECT_NAME="dn1"
 export audius_server_port="5000"
 export audius_db_port="5432"
-export audius_ipfs_external_port="5200"
 export audius_redis_port="5380"
 export audius_es_port="9200"
diff --git a/discovery-provider/compose/env/shellEnv2.sh b/discovery-provider/compose/env/shellEnv2.sh
@@ -1,6 +1,5 @@
 export COMPOSE_PROJECT_NAME="dn2"
 export audius_server_port="5001"
 export audius_db_port="5433"
-export audius_ipfs_external_port="5201"
 export audius_redis_port="5382"
 export audius_es_port="9201"
diff --git a/discovery-provider/compose/env/shellEnv3.sh b/discovery-provider/compose/env/shellEnv3.sh
@@ -1,6 +1,5 @@
 export COMPOSE_PROJECT_NAME="dn3"
 export audius_server_port="5002"
 export audius_db_port="5434"
-export audius_ipfs_external_port="5202"
 export audius_redis_port="5383"
 export audius_es_port="9202"
diff --git a/discovery-provider/compose/env/shellEnv4.sh b/discovery-provider/compose/env/shellEnv4.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn4"
 export audius_server_port="5003"
 export audius_db_port="5435"
-export audius_ipfs_external_port="5203"
 export audius_redis_port="5384"
diff --git a/discovery-provider/compose/env/shellEnv5.sh b/discovery-provider/compose/env/shellEnv5.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn5"
 export audius_server_port="5004"
 export audius_db_port="5436"
-export audius_ipfs_external_port="5204"
 export audius_redis_port="5385"
diff --git a/discovery-provider/compose/env/shellEnv6.sh b/discovery-provider/compose/env/shellEnv6.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn6"
 export audius_server_port="5005"
 export audius_db_port="5437"
-export audius_ipfs_external_port="5205"
 export audius_redis_port="5386"
diff --git a/discovery-provider/compose/env/shellEnv7.sh b/discovery-provider/compose/env/shellEnv7.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn7"
 export audius_server_port="5006"
 export audius_db_port="5438"
-export audius_ipfs_external_port="5206"
 export audius_redis_port="5387"
diff --git a/discovery-provider/compose/env/shellEnv8.sh b/discovery-provider/compose/env/shellEnv8.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn8"
 export audius_server_port="5007"
 export audius_db_port="5439"
-export audius_ipfs_external_port="5207"
 export audius_redis_port="5388"
diff --git a/discovery-provider/compose/env/shellEnv9.sh b/discovery-provider/compose/env/shellEnv9.sh
@@ -1,5 +1,4 @@
 export COMPOSE_PROJECT_NAME="dn9"
 export audius_server_port="5008"
 export audius_db_port="5440"
-export audius_ipfs_external_port="5208"
 export audius_redis_port="5389"
diff --git a/discovery-provider/compose/env/unsetShellEnv.sh b/discovery-provider/compose/env/unsetShellEnv.sh
@@ -2,6 +2,5 @@
 unset COMPOSE_PROJECT_NAME
 unset audius_server_port
 unset audius_db_port
-unset audius_ipfs_external_port
 unset audius_redis_port
 unset audius_es_port
diff --git a/discovery-provider/default_config.ini b/discovery-provider/default_config.ini
@@ -8,7 +8,6 @@ block_processing_window = 20
 block_processing_interval_sec = 1
 peer_refresh_interval = 3000
 identity_service_url = https://identityservice.test
-user_metadata_service_url = ''
 healthy_block_diff = 100
 notifications_max_block_diff = 25
 notifications_max_slot_diff = 200
@@ -61,11 +60,6 @@ engine_args_literal = {
     'connect_args': {'options': '-c timezone=utc'}
   }
 
-[ipfs]
-host = 127.0.0.1
-port = 5001
-gateway_hosts = https://cloudflare-ipfs.com,https://ipfs.io
-
 [cors]
 allow_all = false
 

diff --git a/discovery-provider/integration_tests/queries/test_search.py b/discovery-provider/integration_tests/queries/test_search.py
@@ -101,6 +101,7 @@ def setup_search(app_module):
             handle="",
             wallet="",
             name="user 1",
+            creator_node_endpoint="https://cn.io",
             updated_at=now,
             created_at=now,
         ),
@@ -112,6 +113,7 @@ def setup_search(app_module):
             handle="",
             name="user 2",
             wallet="",
+            creator_node_endpoint="https://cn.io",
             updated_at=now,
             created_at=now,
         ),
@@ -123,6 +125,7 @@ def setup_search(app_module):
             handle="",
             wallet="",
             name="fdwea",
+            creator_node_endpoint="https://cn.io",
             updated_at=now,
             created_at=now,
         ),

diff --git a/discovery-provider/integration_tests/tasks/entity_manager/test_playlist_entity_manager.py b/discovery-provider/integration_tests/tasks/entity_manager/test_playlist_entity_manager.py
@@ -178,7 +178,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=0,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata=test_metadata,
+            metadata=test_metadata,
         )
 
         # validate db records
@@ -439,7 +439,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=0,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata=test_metadata,
+            metadata=test_metadata,
         )
 
         # validate db records

diff --git a/...ery-provider/integration_tests/tasks/entity_manager/test_social_feature_entity_manager.py b/...ery-provider/integration_tests/tasks/entity_manager/test_social_feature_entity_manager.py
@@ -158,7 +158,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=1,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata={},
+            metadata={},
         )
 
         # Verify follows
@@ -326,7 +326,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=1,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata={},
+            metadata={},
         )
 
         # Verify follows

diff --git a/discovery-provider/integration_tests/tasks/entity_manager/test_track_entity_manager.py b/discovery-provider/integration_tests/tasks/entity_manager/test_track_entity_manager.py
@@ -259,7 +259,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=0,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata=test_metadata,
+            metadata=test_metadata,
         )
 
         # validate db records
@@ -494,7 +494,7 @@ def get_events_side_effect(_, tx_receipt):
             block_number=0,
             block_timestamp=1585336422,
             block_hash=0,
-            ipfs_metadata={},
+            metadata={},
         )
 
         # validate db records

diff --git a/discovery-provider/integration_tests/tasks/test_index_operations.py b/discovery-provider/integration_tests/tasks/test_index_operations.py
@@ -2,7 +2,6 @@
 import secrets
 from contextlib import contextmanager
 
-import ipfshttpclient
 import pytest
 import src.utils.multihash
 from chance import chance
@@ -28,10 +27,6 @@ def seed_contract_data(task, contracts, web3):
     user_factory_contract = contracts["user_factory_contract"]
     track_factory_contract = contracts["track_factory_contract"]
 
-    ipfs_peer_host = task.shared_config["ipfs"]["host"]
-    ipfs_peer_port = task.shared_config["ipfs"]["port"]
-    ipfs = ipfshttpclient.connect(f"/dns/{ipfs_peer_host}/tcp/{ipfs_peer_port}/http")
-
     # Retrieve web3 instance from fixture
     chain_id = web3.net.version
 
@@ -79,9 +74,7 @@ def seed_contract_data(task, contracts, web3):
     new_user_args = tx_new_user_info[0].args
     user_id_from_event = int(new_user_args._userId)
 
-    # Add audio file to ipfs node
-    res = ipfs.add(test_file)
-    test_audio_file_hash = res["Hash"]
+    test_audio_file_hash = "Qmb372yhT7bKWxm5VzRpySgFPAhjnQuC6FNStTZww6ULVD"
     test_track_segments = [{"multihash": test_audio_file_hash, "duration": 28060}]
 
     # Create track metadata object
@@ -113,9 +106,7 @@ def seed_contract_data(task, contracts, web3):
     with open(track_metadata_json_file, "w") as f:
         json.dump(track_metadata, f)
 
-    # add track metadata to ipfs
-    metadata_res = ipfs.add(track_metadata_json_file)
-    metadata_hash = metadata_res["Hash"]
+    metadata_hash = "QmQgUzCB38dc1Qb1waQcW75PgBB2fXr4gw2m9JBvGYUdka"
 
     # get track metadata multihash
     metadata_decoded = src.utils.multihash.from_b58_string(metadata_hash)
@@ -253,13 +244,13 @@ def test_index_operations_metadata_fetch_error(
     celery_app, celery_app_contracts, mocker
 ):
     """
-    Confirm indexer throws IndexingError when ipfs metadata fetch throws an error
+    Confirm indexer throws IndexingError when metadata fetch throws an error
     """
     task = celery_app.celery.tasks["update_discovery_provider"]
     db = task.db
     web3 = celery_app_contracts["web3"]
 
-    # patch ipfs metadata event to raise an exception
+    # patch metadata event to raise an exception
     def fetch_metadata_stub(*_, should_fetch_from_replica_set):
         raise Exception("Broken fetch")
 

diff --git a/discovery-provider/integration_tests/tasks/test_index_playlists.py b/discovery-provider/integration_tests/tasks/test_index_playlists.py
@@ -462,7 +462,7 @@ class TestPlaylistTransaction:
         ],
         autospec=True,
     )
-    test_ipfs_metadata = {}
+    test_metadata = {}
 
     with db.scoped_session() as session:
         try:
@@ -481,7 +481,7 @@ class TestPlaylistTransaction:
                 test_block_number,
                 test_block_timestamp,
                 block_hash,
-                test_ipfs_metadata,
+                test_metadata,
             )
             assert len(updated_playlist_ids_set) == 1
             assert (

diff --git a/discovery-provider/integration_tests/tasks/test_index_tracks.py b/discovery-provider/integration_tests/tasks/test_index_tracks.py
@@ -265,7 +265,7 @@ def test_index_tracks(mock_index_task, app):
         track_record = parse_track_event(
             None,  # self - not used
             session,
-            update_task,  # only need the ipfs client for get_metadata
+            update_task,
             entry,  # Contains the event args used for updating
             event_type,  # String that should one of user_event_types_lookup
             track_record,  # User ORM instance
@@ -520,7 +520,7 @@ def test_index_tracks(mock_index_task, app):
         parse_track_event(
             None,  # self - not used
             session,
-            update_task,  # only need the ipfs client for get_metadata
+            update_task,
             entry,  # Contains the event args used for updating
             event_type,  # String that should one of user_event_types_lookup
             track_record,  # User ORM instance
@@ -629,7 +629,7 @@ class TestTrackTransaction:
         autospec=True,
     )
 
-    test_ipfs_metadata = {}
+    test_metadata = {}
 
     with db.scoped_session() as session, challenge_event_bus.use_scoped_dispatch_queue():
         try:
@@ -648,7 +648,7 @@ class TestTrackTransaction:
                 test_block_number,
                 test_block_timestamp,
                 block_hash,
-                test_ipfs_metadata,
+                test_metadata,
             )
             assert len(updated_track_ids_set) == 1
             assert list(updated_track_ids_set)[0] == blessed_track_record.track_id

diff --git a/discovery-provider/integration_tests/tasks/test_index_user_replica_set.py b/discovery-provider/integration_tests/tasks/test_index_user_replica_set.py
@@ -198,7 +198,7 @@ class TestUserReplicaSetTransaction:
         autospec=True,
     )
 
-    test_ipfs_metadata = {}
+    test_metadata = {}
 
     with db.scoped_session() as session:
         try:
@@ -217,7 +217,7 @@ class TestUserReplicaSetTransaction:
                 test_block_number,
                 test_block_timestamp,
                 block_hash,
-                test_ipfs_metadata,
+                test_metadata,
             )
             assert len(updated_user_ids_set) == 1
             assert list(updated_user_ids_set)[0] == blessed_user_record.user_id